0: ch1=8074 mean# 8074 dev# 0 part# 0
1: ch1=8074 ch2=8074 mean# 8074 dev# 0 part# 0
2: ch1=13338 ch2=10884 mean# 12111 dev# 1227 part# 0.1013128561
3: ch1=18148 ch2=14148 mean# 16148 dev# 2000 part# 0.1238543473
4: ch1=20188 ch2=20182 mean# 20185 dev# 3 part# 0.0001486252167
5: ch1=27764 ch2=20680 mean# 24222 dev# 3542 part# 0.1462306994
6: ch1=28222 ch2=28296 mean# 28259 dev# 37 part# 0.001309317386
7: ch1=34120 ch2=30472 mean# 32296 dev# 1824 part# 0.05647758236
8: ch1=37842 ch2=34824 mean# 36333 dev# 1509 part# 0.04153249112
9: ch1=40380 ch2=40360 mean# 40370 dev# 10 part# 0.0002477086946
10: ch1=47458 ch2=41356 mean# 44407 dev# 3051 part# 0.06870538429
11: ch1=49004 ch2=47884 mean# 48444 dev# 560 part# 0.01155973908
12: ch1=53814 ch2=51148 mean# 52481 dev# 1333 part# 0.02539966845
13: ch1=58624 ch2=54412 mean# 56518 dev# 2106 part# 0.03726246506
14: ch1=60572 ch2=60538 mean# 60555 dev# 17 part# 0.0002807365205
15: ch1=68240 ch2=60944 mean# 64592 dev# 3648 part# 0.05647758236
16: ch1=68698 ch2=68560 mean# 68629 dev# 69 part# 0.001005405878
17: ch1=74596 ch2=70736 mean# 72666 dev# 1930 part# 0.0265598767
18: ch1=78318 ch2=75088 mean# 76703 dev# 1615 part# 0.02105523904
19: ch1=80764 ch2=80716 mean# 80740 dev# 24 part# 0.0002972504335
20: ch1=87934 ch2=81620 mean# 84777 dev# 3157 part# 0.03723887375
21: ch1=89480 ch2=88148 mean# 88814 dev# 666 part# 0.007498817754
22: ch1=94290 ch2=91412 mean# 92851 dev# 1439 part# 0.01549794833
23: ch1=99100 ch2=94676 mean# 96888 dev# 2212 part# 0.02283048468
24: ch1=100956 ch2=100894 mean# 100925 dev# 31 part# 0.0003071587813
25: ch1=108716 ch2=101208 mean# 104962 dev# 3754 part# 0.03576532459
26: ch1=109174 ch2=108824 mean# 108999 dev# 175 part# 0.001605519317
27: ch1=113984 ch2=112088 mean# 113036 dev# 948 part# 0.008386708659
28: ch1=119882 ch2=114264 mean# 117073 dev# 2809 part# 0.02399357666
29: ch1=121148 ch2=121072 mean# 121110 dev# 38 part# 0.0003137643465
30: ch1=128410 ch2=121884 mean# 125147 dev# 3263 part# 0.02607333775
31: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
32: ch1=126692 ch2=123602 mean# 125147 dev# 1545 part# 0.01234548171
33: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
34: ch1=123192 ch2=127102 mean# 125147 dev# 1955 part# 0.01562162896
35: ch1=129004 ch2=121290 mean# 125147 dev# 3857 part# 0.03081975597
36: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
37: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
38: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
39: ch1=123686 ch2=126608 mean# 125147 dev# 1461 part# 0.01167427106
40: ch1=128502 ch2=121792 mean# 125147 dev# 3355 part# 0.02680847324
41: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
42: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
43: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
44: ch1=123096 ch2=127198 mean# 125147 dev# 2051 part# 0.01638872686
45: ch1=129092 ch2=121202 mean# 125147 dev# 3945 part# 0.03152292904
46: ch1=121424 ch2=128870 mean# 125147 dev# 3723 part# 0.02974901516
47: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
48: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
49: ch1=123590 ch2=126704 mean# 125147 dev# 1557 part# 0.01244136895
50: ch1=128590 ch2=121704 mean# 125147 dev# 3443 part# 0.0275116463
51: ch1=121878 ch2=128416 mean# 125147 dev# 3269 part# 0.02612128137
52: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
53: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
54: ch1=123000 ch2=127244 mean# 125122 dev# 2122 part# 0.01695944758
55: ch1=128046 ch2=122198 mean# 125122 dev# 2924 part# 0.02336919167
56: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
57: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
58: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
59: ch1=122410 ch2=127834 mean# 125122 dev# 2712 part# 0.02167484535
60: ch1=128636 ch2=121608 mean# 125122 dev# 3514 part# 0.02808458944
61: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
62: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
63: ch1=125096 ch2=125148 mean# 125122 dev# 26 part# 0.0002077971899
64: ch1=122908 ch2=127336 mean# 125122 dev# 2214 part# 0.01769472994
65: ch1=128138 ch2=122106 mean# 125122 dev# 3016 part# 0.02410447403
66: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
67: ch1=126642 ch2=123602 mean# 125122 dev# 1520 part# 0.01214814341
68: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
69: ch1=122322 ch2=127922 mean# 125122 dev# 2800 part# 0.02237815892
70: ch1=128732 ch2=121512 mean# 125122 dev# 3610 part# 0.0288518406
71: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
72: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
73: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
74: ch1=123908 ch2=126336 mean# 125122 dev# 1214 part# 0.00970253033
75: ch1=128234 ch2=122010 mean# 125122 dev# 3112 part# 0.0248717252
76: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
77: ch1=126646 ch2=123598 mean# 125122 dev# 1524 part# 0.01218011221
78: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
79: ch1=123322 ch2=126922 mean# 125122 dev# 1800 part# 0.0143859593
80: ch1=128828 ch2=121416 mean# 125122 dev# 3706 part# 0.02961909177
81: ch1=121840 ch2=128404 mean# 125122 dev# 3282 part# 0.02623039913
82: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
83: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
84: ch1=123820 ch2=126424 mean# 125122 dev# 1302 part# 0.0104058439
85: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
86: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
87: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
88: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
89: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
90: ch1=128920 ch2=121374 mean# 125147 dev# 3773 part# 0.03014854531
91: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
92: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
93: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
94: ch1=123770 ch2=126524 mean# 125147 dev# 1377 part# 0.0110030604
95: ch1=128418 ch2=121876 mean# 125147 dev# 3271 part# 0.02613726258
96: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
97: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
98: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
99: ch1=123180 ch2=127114 mean# 125147 dev# 1967 part# 0.0157175162
100: ch1=126832 ch2=122374 ch3=1088 mean# 83431.33333 dev# 58253.9661 part# 0.6982264788
101: ch1=119248 ch2=125606 ch3=5440 mean# 83431.33333 dev# 55209.25102 part# 0.6617328145
102: ch1=123198 ch2=119712 ch3=7384 mean# 83431.33333 dev# 53792.41415 part# 0.6447507429
103: ch1=118610 ch2=120800 ch3=10884 mean# 83431.33333 dev# 51306.5019 part# 0.6149548359
104: ch1=120786 ch2=118624 ch3=10884 mean# 83431.33333 dev# 51306.30397 part# 0.6149524636
105: ch1=119344 ch2=115714 ch3=15236 mean# 83431.33333 dev# 48244.14881 part# 0.578249764
106: ch1=112258 ch2=118624 ch3=19412 mean# 83431.33333 dev# 45343.04628 part# 0.5434774259
107: ch1=115342 ch2=114272 ch3=20680 mean# 83431.33333 dev# 44374.04347 part# 0.5318630507
108: ch1=112078 ch2=114272 ch3=23944 mean# 83431.33333 dev# 42073.43207 part# 0.504288142
109: ch1=112988 ch2=112096 ch3=25210 mean# 83431.33333 dev# 41170.31015 part# 0.4934634089
110: ch1=111048 ch2=109858 ch3=29388 mean# 83431.33333 dev# 38217.49541 part# 0.4580712531
111: ch1=107722 ch2=111008 ch3=31564 mean# 83431.33333 dev# 36700.26929 part# 0.4398859257
112: ch1=107264 ch2=107744 ch3=35286 mean# 83431.33333 dev# 34044.45566 part# 0.4080535968
113: ch1=108196 ch2=105568 ch3=36530 mean# 83431.33333 dev# 33181.6003 part# 0.3977114948
114: ch1=102298 ch2=107660 ch3=40336 mean# 83431.33333 dev# 30551.52563 part# 0.3661876708
115: ch1=106818 ch2=102108 ch3=41368 mean# 83431.33333 dev# 29805.35783 part# 0.3572441748
116: ch1=99640 ch2=104934 ch3=45720 mean# 83431.33333 dev# 26753.38116 part# 0.3206634737
117: ch1=103404 ch2=99078 ch3=47812 mean# 83431.33333 dev# 25248.5148 part# 0.3026262891
118: ch1=97914 ch2=101220 ch3=51160 mean# 83431.33333 dev# 22859.15754 part# 0.273987681
119: ch1=100090 ch2=97956 ch3=52248 mean# 83431.33333 dev# 22067.15051 part# 0.2644947603
120: ch1=98242 ch2=96082 ch3=55970 mean# 83431.33333 dev# 19438.10727 part# 0.2329832989
121: ch1=92474 ch2=99040 ch3=58780 mean# 83431.33333 dev# 17636.02875 part# 0.2113837578
122: ch1=96196 ch2=93142 ch3=60956 mean# 83431.33333 dev# 15941.29199 part# 0.1910708046
123: ch1=92478 ch2=93596 ch3=64220 mean# 83431.33333 dev# 13592.12952 part# 0.1629139674
124: ch1=91390 ch2=92508 ch3=66396 mean# 83431.33333 dev# 12054.44364 part# 0.1444834112
125: ch1=90132 ch2=90044 ch3=70118 mean# 83431.33333 dev# 9414.016831 part# 0.1128355074
126: ch1=87038 ch2=91416 ch3=71840 mean# 83431.33333 dev# 8388.920365 part# 0.100548799
127: ch1=88126 ch2=87064 ch3=75104 mean# 83431.33333 dev# 5904.253909 part# 0.07076782395
128: ch1=87330 ch2=85976 ch3=76988 mean# 83431.33333 dev# 4589.534278 part# 0.05500971991
129: ch1=83608 ch2=85976 ch3=80710 mean# 83431.33333 dev# 2153.461916 part# 0.02581118903
130: ch1=86820 ch2=81834 ch3=81640 mean# 83431.33333 dev# 2397.457728 part# 0.02873569955
131: ch1=81136 ch2=84254 ch3=84904 mean# 83431.33333 dev# 1644.595459 part# 0.01971196423
132: ch1=85172 ch2=82394 ch3=82728 mean# 83431.33333 dev# 1238.367025 part# 0.01484294899
133: ch1=81358 ch2=84888 ch3=84048 mean# 83431.33333 dev# 1505.641244 part# 0.01804647228
134: ch1=87034 ch2=81624 ch3=81636 mean# 83431.33333 dev# 2547.474741 part# 0.03053378915
135: ch1=78960 ch2=85346 ch3=85988 mean# 83431.33333 dev# 3172.554947 part# 0.03802594085
136: ch1=82564 ch2=83918 ch3=83812 mean# 83431.33333 dev# 614.8221062 part# 0.007369199096
137: ch1=84678 ch2=82716 ch3=82900 mean# 83431.33333 dev# 884.7211739 part# 0.01060418357
138: ch1=84858 ch2=82716 ch3=82720 mean# 83431.33333 dev# 1008.806996 part# 0.01209146439
139: ch1=83312 ch2=83174 ch3=83808 mean# 83431.33333 dev# 272.236825 part# 0.003263004607
140: ch1=80764 ch2=84812 ch3=84718 mean# 83431.33333 dev# 1886.479849 part# 0.02261116746
141: ch1=84402 ch2=83176 ch3=82716 mean# 83431.33333 dev# 711.5922678 part# 0.008529077019
142: ch1=81598 ch2=84892 ch3=83804 mean# 83431.33333 dev# 1370.345293 part# 0.01642482792
143: ch1=86408 ch2=81628 ch3=82258 mean# 83431.33333 dev# 2120.476886 part# 0.02541583361
144: ch1=80666 ch2=84892 ch3=84736 mean# 83431.33333 dev# 1956.422813 part# 0.02344949714
145: ch1=87018 ch2=81712 ch3=81564 mean# 83431.33333 dev# 2536.875944 part# 0.03040675299
146: ch1=81148 ch2=84262 ch3=84884 mean# 83431.33333 dev# 1634.406994 part# 0.01958984627
147: ch1=84236 ch2=83350 ch3=82708 mean# 83431.33333 dev# 626.4488983 part# 0.007508556717
148: ch1=81560 ch2=84854 ch3=83880 mean# 83431.33333 dev# 1381.686248 part# 0.01656075953
149: ch1=87050 ch2=81624 ch3=81620 mean# 83431.33333 dev# 2558.78426 part# 0.03066934397
150: ch1=80064 ch2=85346 ch3=84884 mean# 83431.33333 dev# 2388.522742 part# 0.02862860566
151: ch1=83478 ch2=83478 ch3=83338 mean# 83431.33333 dev# 65.99663291 part# 0.0007910293444
152: ch1=83782 ch2=82720 ch3=83792 mean# 83431.33333 dev# 503.005191 part# 0.006028972221
153: ch1=82694 ch2=83808 ch3=83792 mean# 83431.33333 dev# 521.414316 part# 0.006249622236
154: ch1=84870 ch2=82720 ch3=82704 mean# 83431.33333 dev# 1017.311926 part# 0.01219340367
155: ch1=80518 ch2=84896 ch3=84880 mean# 83431.33333 dev# 2060.048112 part# 0.02469154009
156: ch1=85328 ch2=82720 ch3=82246 mean# 83431.33333 dev# 1355.034399 part# 0.016241313
157: ch1=82698 ch2=83808 ch3=83788 mean# 83431.33333 dev# 518.6092513 part# 0.006216000999
158: ch1=84874 ch2=82720 ch3=82700 mean# 83431.33333 dev# 1020.152058 part# 0.01222744523
159: ch1=80780 ch2=84896 ch3=84618 mean# 83431.33333 dev# 1878.207893 part# 0.02251202058
160: ch1=87212 ch2=81632 ch3=81450 mean# 83431.33333 dev# 2674.367381 part# 0.03205471223
161: ch1=81156 ch2=84266 ch3=84872 mean# 83431.33333 dev# 1627.81353 part# 0.01951081764
162: ch1=84244 ch2=83354 ch3=82696 mean# 83431.33333 dev# 634.3297425 part# 0.007603015763
163: ch1=81754 ch2=84756 ch3=83784 mean# 83431.33333 dev# 1250.67484 part# 0.01499046929
164: ch1=87058 ch2=81628 ch3=81608 mean# 83431.33333 dev# 2564.453591 part# 0.03073729604
165: ch1=81160 ch2=84262 ch3=84872 mean# 83431.33333 dev# 1625.267431 part# 0.01948030034
166: ch1=84248 ch2=83350 ch3=82696 mean# 83431.33333 dev# 636.2061161 part# 0.007625505799
167: ch1=82190 ch2=84320 ch3=83784 mean# 83431.33333 dev# 904.6197482 part# 0.01084268598
168: ch1=85974 ch2=82712 ch3=81608 mean# 83431.33333 dev# 1853.567611 part# 0.02221668451
169: ch1=81164 ch2=84258 ch3=84872 mean# 83431.33333 dev# 1622.72391 part# 0.01944981394
170: ch1=83960 ch2=83346 ch3=82988 mean# 83431.33333 dev# 401.378735 part# 0.004810887216
171: ch1=84798 ch2=82796 ch3=82700 mean# 83431.33333 dev# 967.1736602 part# 0.0115924512
172: ch1=81160 ch2=84258 ch3=84876 mean# 83431.33333 dev# 1625.771065 part# 0.01948633685
173: ch1=85696 ch2=81898 ch3=82700 mean# 83431.33333 dev# 1634.490203 part# 0.0195908436
174: ch1=80886 ch2=84896 ch3=84512 mean# 83431.33333 dev# 1806.6369 part# 0.02165417749
175: ch1=87504 ch2=81632 ch3=81158 mean# 83431.33333 dev# 2886.304365 part# 0.0345949687
176: ch1=81152 ch2=84266 ch3=84876 mean# 83431.33333 dev# 1630.857716 part# 0.01954730496
177: ch1=84240 ch2=83354 ch3=82700 mean# 83431.33333 dev# 631.0759771 part# 0.007564016442
178: ch1=81610 ch2=83808 ch3=84876 mean# 83431.33333 dev# 1359.680681 part# 0.0162970029
179: ch1=88138 ch2=80544 ch3=81612 mean# 83431.33333 dev# 3356.554715 part# 0.04023134452
180: ch1=82240 ch2=83808 ch3=84246 mean# 83431.33333 dev# 861.1687149 part# 0.01032188604
181: ch1=84194 ch2=85572 ch3=80528 mean# 83431.33333 dev# 2128.650485 part# 0.02551380159
182: ch1=87808 ch2=84134 ch3=78352 mean# 83431.33333 dev# 3892.23923 part# 0.046652008
183: ch1=87050 ch2=88156 ch3=75088 mean# 83431.33333 dev# 5916.880785 part# 0.07091916848
184: ch1=89226 ch2=88156 ch3=72912 mean# 83431.33333 dev# 7451.107539 part# 0.08930826395
185: ch1=90314 ch2=89244 ch3=70736 mean# 83431.33333 dev# 8987.578144 part# 0.107724254
186: ch1=91854 ch2=92056 ch3=66384 mean# 83431.33333 dev# 12054.56708 part# 0.1444848908
187: ch1=94662 ch2=90336 ch3=65296 mean# 83431.33333 dev# 12944.65927 part# 0.1551534508
188: ch1=93574 ch2=94688 ch3=62032 mean# 83431.33333 dev# 15138.44663 part# 0.1814479767
189: ch1=95750 ch2=94688 ch3=59856 mean# 83431.33333 dev# 16675.91511 part# 0.1998759273
190: ch1=98464 ch2=95776 ch3=56054 mean# 83431.33333 dev# 19389.77602 part# 0.2324040052
191: ch1=94528 ch2=100266 ch3=55500 mean# 83431.33333 dev# 19888.86954 part# 0.2383860924
192: ch1=101648 ch2=97498 ch3=51148 mean# 83431.33333 dev# 22890.54875 part# 0.2743639331
193: ch1=100106 ch2=101216 ch3=48972 mean# 83431.33333 dev# 24370.6417 part# 0.2921041859
194: ch1=103370 ch2=101216 ch3=45708 mean# 83431.33333 dev# 26688.91577 part# 0.3198907976
195: ch1=102282 ch2=103392 ch3=44620 mean# 83431.33333 dev# 27447.49802 part# 0.3289830921
196: ch1=104188 ch2=105838 ch3=40268 mean# 83431.33333 dev# 30528.51818 part# 0.3659119058
197: ch1=108810 ch2=103392 ch3=38092 mean# 83431.33333 dev# 32135.96159 part# 0.3851785691
198: ch1=107722 ch2=107744 ch3=34828 mean# 83431.33333 dev# 34367.74776 part# 0.4119285452
199: ch1=106634 ch2=109920 ch3=33740 mean# 83431.33333 dev# 35162.67818 part# 0.4214565053
200: ch1=112532 ch2=108374 ch3=29388 mean# 83431.33333 dev# 38252.09057 part# 0.4584859074
201: ch1=111052 ch2=112322 ch3=26920 mean# 83431.33333 dev# 39962.91048 part# 0.4789916316
202: ch1=114254 ch2=111008 ch3=25032 mean# 83431.33333 dev# 41315.82206 part# 0.4952075008
203: ch1=114712 ch2=114902 ch3=20680 mean# 83431.33333 dev# 44371.96113 part# 0.5318380919
204: ch1=114528 ch2=116174 ch3=19592 mean# 83431.33333 dev# 45146.22678 part# 0.5411183662
205: ch1=119338 ch2=115352 ch3=15604 mean# 83431.33333 dev# 47988.76542 part# 0.575188763
206: ch1=114534 ch2=120524 ch3=15236 mean# 83431.33333 dev# 48283.34869 part# 0.57871961
207: ch1=122336 ch2=117074 ch3=10884 mean# 83431.33333 dev# 51343.67109 part# 0.6154003423
208: ch1=119248 ch2=122338 ch3=8708 mean# 83431.33333 dev# 52852.43251 part# 0.6334842127
209: ch1=124054 ch2=121884 ch3=4356 mean# 83431.33333 dev# 55921.72197 part# 0.6702724232
210: ch1=121878 ch2=124060 ch3=4356 mean# 83431.33333 dev# 55921.79979 part# 0.670273356
211: ch1=123102 ch2=126558 ch3=634 mean# 83431.33333 dev# 58563.55402 part# 0.7019371701
212: ch1=128452 ch2=121842 mean# 125147 dev# 3305 part# 0.02640894308
213: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
214: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
215: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
216: ch1=123146 ch2=127148 mean# 125147 dev# 2001 part# 0.0159891967
217: ch1=127954 ch2=122340 mean# 125147 dev# 2807 part# 0.02242962276
218: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
219: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
220: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
221: ch1=123644 ch2=126650 mean# 125147 dev# 1503 part# 0.01200987639
222: ch1=128544 ch2=121750 mean# 125147 dev# 3397 part# 0.02714407856
223: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
224: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
225: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
226: ch1=123054 ch2=127240 mean# 125147 dev# 2093 part# 0.01672433219
227: ch1=128050 ch2=122244 mean# 125147 dev# 2903 part# 0.02319672066
228: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
229: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
230: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
231: ch1=122464 ch2=127830 mean# 125147 dev# 2683 part# 0.02143878799
232: ch1=128640 ch2=121654 mean# 125147 dev# 3493 part# 0.02791117646
233: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
234: ch1=127326 ch2=122968 mean# 125147 dev# 2179 part# 0.01741152405
235: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
236: ch1=122962 ch2=127332 mean# 125147 dev# 2185 part# 0.01745946767
237: ch1=128142 ch2=122152 mean# 125147 dev# 2995 part# 0.02393185614
238: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
239: ch1=126696 ch2=123598 mean# 125147 dev# 1549 part# 0.01237744413
240: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
241: ch1=122376 ch2=127918 mean# 125147 dev# 2771 part# 0.02214196105
242: ch1=128736 ch2=121558 mean# 125147 dev# 3589 part# 0.02867827435
243: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
244: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
245: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
246: ch1=123962 ch2=126332 mean# 125147 dev# 1185 part# 0.009468864615
247: ch1=128238 ch2=122056 mean# 125147 dev# 3091 part# 0.02469895403
248: ch1=122978 ch2=127316 mean# 125147 dev# 2169 part# 0.01733161802
249: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
250: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
251: ch1=123372 ch2=126922 mean# 125147 dev# 1775 part# 0.01418332042
252: ch1=128828 ch2=121466 mean# 125147 dev# 3681 part# 0.02941340983
253: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
254: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
255: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
256: ch1=123870 ch2=126424 mean# 125147 dev# 1277 part# 0.0102040001
257: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
258: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
259: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
260: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
261: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
262: ch1=127832 ch2=122462 mean# 125147 dev# 2685 part# 0.02145476919
263: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
264: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
265: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
266: ch1=123774 ch2=126520 mean# 125147 dev# 1373 part# 0.01097109799
267: ch1=128422 ch2=121872 mean# 125147 dev# 3275 part# 0.02616922499
268: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
269: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
270: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
271: ch1=123184 ch2=127110 mean# 125147 dev# 1963 part# 0.01568555379
272: ch1=127924 ch2=122370 mean# 125147 dev# 2777 part# 0.02218990467
273: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
274: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
275: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
276: ch1=122594 ch2=127700 mean# 125147 dev# 2553 part# 0.02040000959
277: ch1=121152 ch2=129142 mean# 125147 dev# 3995 part# 0.03192245919
278: ch1=114066 ch2=136228 mean# 125147 dev# 11081 part# 0.08854387241
279: ch1=112520 ch2=137774 mean# 125147 dev# 12627 part# 0.1008973447
280: ch1=107710 ch2=142584 mean# 125147 dev# 17437 part# 0.1393321454
281: ch1=102900 ch2=147394 mean# 125147 dev# 22247 part# 0.1777669461
282: ch1=100960 ch2=149334 mean# 125147 dev# 24187 part# 0.193268716
283: ch1=93284 ch2=157010 mean# 125147 dev# 31863 part# 0.254604585
284: ch1=92826 ch2=157468 mean# 125147 dev# 32321 part# 0.2582642812
285: ch1=88016 ch2=162278 mean# 125147 dev# 37131 part# 0.2966990819
286: ch1=82118 ch2=166520 mean# 124319 dev# 42201 part# 0.3394573637
2025-11-28T14:51:09.605010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788911363377394:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:09.660887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788911052696252:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:09.677611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788911052696252:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16064915ns) {Configure:104(16052075ns) {ProcessConfigured:135(84680ns) {StartInitialHandshake:157(82120ns) {DropHandshakes:470(3219ns) {DropIncomingHandshake:442(682ns) DropOutgoingHandshake:457(520ns)} GenerateSessionVirtualId:988(1250ns) PrepareNewSessionHandshake:477(1896ns) {SwitchToState:179(639ns)}}}}}
2025-11-28T14:51:09.760112Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788911052696263:2057] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:09.764608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788911363377404:2057] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:09.766253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577788911363377394:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:09.772785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788911052696263:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:09.773279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788911052696263:2057] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:09.773552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788911052696252:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:51:09.773650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:09.773693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788911052696252:2048] [node 1] ICP22 created new session: [2:7577788911052696266:2048]
2025-11-28T14:51:09.773749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788911052696266:2048] [node 1] ICS09 handshake done sender: [2:7577788911052696263:2057] self: [2:7577788911052696262:0] peer: [1:7577788911363377405:0] socket: 16 qp: -1
2025-11-28T14:51:09.773780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788911052696266:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:09.773844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:09.789166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788911363377404:2057] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:09.792491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577788911363377394:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:09.792604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:09.792653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788911363377394:2048] [node 2] ICP22 created new session: [1:7577788911363377407:2048]
2025-11-28T14:51:09.792706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788911363377407:2048] [node 2] ICS09 handshake done sender: [1:7577788911363377404:2057] self: [1:7577788911363377405:0] peer: [2:7577788911052696262:0] socket: 18 qp: 18
2025-11-28T14:51:09.792738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788911363377407:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:09.792777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:09.854823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:09.855093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577788911052696266:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-28T14:51:09.855133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577788911052696252:2048] [node 1] ICP30 unregister session Session# [2:7577788911052696266:2048] VirtualId# [2:7577788911052696262:0]
2025-11-28T14:51:09.855162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577788911052696266:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:09.855248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:09.855299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:09.858726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788911052696252:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:5079) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:09.864910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788911052696252:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:5079) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:09.865009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577788911052696252:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:5079) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-28T14:51:09.852399Z
2025-11-28T14:51:09.865022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:5079) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-28T14:51:09.852399Z
2025-11-28T14:51:09.876780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577788911052696252:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:10.020574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788909678929118:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:10.032857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788916163804459:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:10.142981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788913973896424:2057] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:10.141190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788916163804470:2057] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:10.144611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577788909678929118:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:10.144947Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1583: Handshake [1:7577788913973896424:2057] [node 2] ICH03 handshake failed, explanation# incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected
2025-11-28T14:51:10.150458Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1583: Handshake [2:7577788916163804470:2057] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:12767) error from peer: connection checker refused to proceed with incoming connection: rejected
2025-11-28T14:51:10.151085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788916163804459:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:12767) error from peer: connection checker refused to proceed with incoming connection: rejected incoming: [0:0:0] held: no
2025-11-28T14:51:10.151151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577788916163804459:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:12767) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:10.151173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:12767) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:10.168211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577788916163804459:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:10.248666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788913527379248:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:10.248722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788913527379249:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:10.268303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788916296062131:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:10.268337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788916296062132:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:10.304875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788914969502819:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:10.304916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788914969502820:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:10.412821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788913585173980:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:10.412874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788913585173981:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:10.412905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788913585173982:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:10.472847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788914165406608:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:10.472895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788914165406609:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:10.472916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788914165406610:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:10.512723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788912842944683:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:10.512761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788912842944684:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:10.512779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788912842944685:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:10.780500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788913527379284:2074] [node 5] ICP01 ready to work
2025-11-28T14:51:10.831513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788913527379295:2081] [node 4] ICP01 ready to work
2025-11-28T14:51:10.857251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788914969502849:2066] [node 6] ICP01 ready to work
2025-11-28T14:51:10.857526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788914969502850:2068] [node 5] ICP01 ready to work
2025-11-28T14:51:10.857755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788914969502851:2070] [node 4] ICP01 ready to work
2025-11-28T14:51:10.915443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788912842944709:2059] [node 4] ICP01 ready to work
2025-11-28T14:51:10.904667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788916296062152:2058] [node 6] ICP01 ready to work
2025-11-28T14:51:10.941307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788914165406609:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(163757987ns) {Configure:104(155095611ns) {ProcessConfigured:135(86380ns) {StartInitialHandshake:157(84187ns) {DropHandshakes:470(3442ns) {DropIncomingHandshake:442(797ns) DropOutgoingHandshake:457(661ns)} GenerateSessionVirtualId:988(803ns) PrepareNewSessionHandshake:477(2572ns) {SwitchToState:179(1116ns)}}}}}
2025-11-28T14:51:10.915824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788913527379297:2082] [node 6] ICP01 ready to work
2025-11-28T14:51:10.905057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788916296062153:2063] [node 4] ICP01 ready to work
2025-11-28T14:51:10.905400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788916296062155:2066] [node 5] ICP01 ready to work
2025-11-28T14:51:10.964588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788912842944711:2062] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.073653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788918460373931:2064] [node 6] ICP01 ready to work
2025-11-28T14:51:11.074011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788918460373932:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.041352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788914969502849:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(181936483ns) {Configure:104(181927147ns) {ProcessConfigured:135(61620ns) {StartInitialHandshake:157(59383ns) {DropHandshakes:470(5852ns) {DropIncomingHandshake:442(1151ns) DropOutgoingHandshake:457(453ns)} GenerateSessionVirtualId:988(622ns) PrepareNewSessionHandshake:477(2110ns) {SwitchToState:179(915ns)}}}}}
2025-11-28T14:51:11.084720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788913585173980:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(287449020ns) {Configure:104(287445269ns) {ProcessConfigured:135(23992419ns) {StartInitialHandshake:157(23988973ns) {DropHandshakes:470(3592ns) {DropIncomingHandshake:442(1024ns) DropOutgoingHandshake:457(519ns)} GenerateSessionVirtualId:988(1060ns) PrepareNewSessionHandshake:477(2551ns) {SwitchToState:179(1107ns)}}}}}
2025-11-28T14:51:11.089073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788918460373930:2069] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-11-28T14:51:11.048725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788919264470168:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39451
2025-11-28T14:51:11.166014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788912842944685:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(185255586ns) {Configure:104(185251117ns) {ProcessConfigured:135(73016ns) {StartInitialHandshake:157(70736ns) {DropHandshakes:470(3107ns) {DropIncomingHandshake:442(785ns) DropOutgoingHandshake:457(659ns)} GenerateSessionVirtualId:988(1140ns) PrepareNewSessionHandshake:477(1754ns) {SwitchToState:179(575ns)}}}}}
2025-11-28T14:51:11.184402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788913527379284:2074] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(178597658ns) {Configure:104(178581774ns) {ProcessConfigured:135(46079923ns) {StartInitialHandshake:157(46076435ns) {DropHandshakes:470(2376ns) {DropIncomingHandshake:442(362ns) DropOutgoingHandshake:457(337ns)} GenerateSessionVirtualId:988(1336ns) PrepareNewSessionHandshake:477(3445ns) {SwitchToState:179(1794ns)}}}}}
2025-11-28T14:51:11.224649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788913585173982:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(139708027ns) {Configure:104(139703501ns) {ProcessConfigured:135(31260277ns) {StartInitialHandshake:157(31256510ns) {DropHandshakes:470(3070ns) {DropIncomingHandshake:442(812ns) DropOutgoingHandshake:457(427ns)} GenerateSessionVirtualId:988(931ns) PrepareNewSessionHandshake:477(7049ns) {SwitchToState:179(4724ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.244831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788912842944684:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(76573317ns) {Configure:104(76569252ns) {ProcessConfigured:135(69449ns) {StartInitialHandshake:157(67350ns) {DropHandshakes:470(3352ns) {DropIncomingHandshake:442(740ns) DropOutgoingHandshake:457(467ns)} GenerateSessionVirtualId:988(868ns) PrepareNewSessionHandshake:477(2086ns) {SwitchToState:179(959ns)}}}}}
2025-11-28T14:51:11.246631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788917137912040:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:11.246777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788917137912041:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:11.384778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788913527379297:2082] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28644566ns) {Configure:104(28642187ns) {ProcessConfigured:135(19892441ns) {StartInitialHandshake:157(19890092ns) {DropHandshakes:470(9845ns) {DropIncomingHandshake:442(790ns) DropOutgoingHandshake:457(611ns)} GenerateSessionVirtualId:988(579ns) PrepareNewSessionHandshake:477(3280ns) {SwitchToState:179(737ns)}}}}}
2025-11-28T14:51:11.409457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788916296062152:2058] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(503660764ns) {Configure:104(503656149ns) {ProcessConfigured:135(77064ns) {StartInitialHandshake:157(74403ns) {DropHandshakes:470(3530ns) {DropIncomingHandshake:442(706ns) DropOutgoingHandshake:457(606ns)} GenerateSessionVirtualId:988(1324ns) PrepareNewSessionHandshake:477(2088ns) {SwitchToState:179(655ns)}}}}}
2025-11-28T14:51:11.410722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788913585173981:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(185814538ns) {Configure:104(185809666ns) {ProcessConfigured:135(67330ns) {StartInitialHandshake:157(65157ns) {DropHandshakes:470(2764ns) {DropIncomingHandshake:442(546ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:477(1159ns) {SwitchToState:179(368ns)}}}}}
2025-11-28T14:51:11.424662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788917880141345:2069] [node 5] ICP01 ready to work
2025-11-28T14:51:11.425108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788917880141346:2070] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.434000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788917880141347:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48690
2025-11-28T14:51:11.414558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788914165406624:2061] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.481149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788916296062132:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(71417435ns) {Configure:104(71383846ns) {ProcessConfigured:135(23886146ns) {StartInitialHandshake:157(23881964ns) {DropHandshakes:470(3198ns) {DropIncomingHandshake:442(770ns) DropOutgoingHandshake:457(533ns)} GenerateSessionVirtualId:988(1158ns) PrepareNewSessionHandshake:477(2552ns) {SwitchToState:179(1224ns)}}}}}
2025-11-28T14:51:11.482356Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788919264470165:2090] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.521574Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788916296062155:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(38353491ns) {Configure:104(38349676ns) {ProcessConfigured:135(91337ns) {StartInitialHandshake:157(87599ns) {DropHandshakes:470(3103ns) {DropIncomingHandshake:442(842ns) DropOutgoingHandshake:457(473ns)} GenerateSessionVirtualId:988(1344ns) PrepareNewSessionHandshake:477(1963ns) {SwitchToState:179(605ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.526396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788917822346623:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25708
2025-11-28T14:51:11.530057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788917137912030:2094] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.552969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788918460373930:2069] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.573409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788916296062154:2068] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13659
2025-11-28T14:51:11.575335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788920591029502:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13659
2025-11-28T14:51:11.575494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788920591029503:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13659
2025-11-28T14:51:11.575590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788920591029504:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13659
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.580688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788917137912059:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:11.596701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788919264470163:2089] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.622607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788917137912040:2101] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.622812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788912842944685:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:11.597511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788919264470168:2092] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.647613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788914969502834:2061] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.678138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788919264470163:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.678454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788919264470196:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39451
2025-11-28T14:51:11.678584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788919264470197:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39451
2025-11-28T14:51:11.678676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788919264470198:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39451
2025-11-28T14:51:11.678768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788919264470199:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39451
2025-11-28T14:51:11.680270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788919264470163:2089] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:11.687482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788914969502849:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:11.687565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:11.687626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788914969502849:2066] [node 6] ICP22 created new session: [3:7577788919264470202:2066]
2025-11-28T14:51:11.687667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788919264470202:2066] [node 6] ICS09 handshake done sender: [3:7577788919264470163:2089] self: [3:7577788919264470162:0] peer: [6:7577788917137912060:0] socket: 42 qp: -1
2025-11-28T14:51:11.687757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788919264470202:2066] [node 6] ICS10 traffic start
2025-11-28T14:51:11.687805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:11.693095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788918460373934:2070] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.716120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788913527379292:2090] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.767224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788917880141312:2082] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.767987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788917880141347:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.780726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788917880141344:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.792872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788917880141315:2084] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.793891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788917880141355:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48690
2025-11-28T14:51:11.794006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788917880141356:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48690
2025-11-28T14:51:11.796757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788919264470198:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.813209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788914165406622:2060] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.813855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788918460373983:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-11-28T14:51:11.813962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788918460373984:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-11-28T14:51:11.814097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788914165406610:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:11.829831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788913527379294:2091] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.830444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788917822346623:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.847714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788917822346598:2095] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.852822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788918460373984:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.852941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788914165406608:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:11.856621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788917822346598:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.857077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788917822346598:2095] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.864895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788918460373987:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-11-28T14:51:11.865178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788913527379284:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:11.865245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:11.865294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788913527379284:2074] [node 5] ICP22 created new session: [1:7577788917822346626:2074]
2025-11-28T14:51:11.865334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788917822346626:2074] [node 5] ICS09 handshake done sender: [1:7577788917822346598:2095] self: [1:7577788917822346596:0] peer: [5:7577788918460373986:0] socket: 55 qp: -1
2025-11-28T14:51:11.865360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788917822346626:2074] [node 5] ICS10 traffic start
2025-11-28T14:51:11.865394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:11.866654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788918460373984:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:11.866834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788914165406608:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:11.866875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:11.866911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788914165406608:2048] [node 1] ICP22 created new session: [5:7577788918460373988:2048]
2025-11-28T14:51:11.866943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788918460373988:2048] [node 1] ICS09 handshake done sender: [5:7577788918460373984:2117] self: [5:7577788918460373986:0] peer: [1:7577788917822346596:0] socket: 64 qp: -1
2025-11-28T14:51:11.866968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788918460373988:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:11.866993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.868885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788917880141358:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48690
2025-11-28T14:51:11.868943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788917880141359:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48690
2025-11-28T14:51:11.873376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788917137912028:2093] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.873967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788917137912041:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:11.885935Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788917137912032:2095] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.886761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788917137912061:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:11.896610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788912842944711:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:11.896886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788917137912040:2101] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:11.897068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788912842944685:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:11.897149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:11.897186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788912842944685:2050] [node 3] ICP22 created new session: [6:7577788917137912063:2050]
2025-11-28T14:51:11.897223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788917137912063:2050] [node 3] ICS09 handshake done sender: [6:7577788917137912040:2101] self: [6:7577788917137912060:0] peer: [3:7577788919264470162:0] socket: 47 qp: -1
2025-11-28T14:51:11.897255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788917137912063:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:11.897287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:11.901987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788920591029495:2111] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.902561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788916296062154:2068] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.916639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788917137912059:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.916862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788912842944684:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:11.926584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788920591029497:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:11.927261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788920591029503:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:11.927359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788920591029504:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.019013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788920591029499:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.026727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788920591029502:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.027633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788920591029495:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.028058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577788916296062131:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:12.028356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788924885996806:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13659
2025-11-28T14:51:12.032600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788920591029495:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:12.032811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788917880141355:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.039796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788916296062152:2058] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.039878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:12.039936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788916296062152:2058] [node 6] ICP22 created new session: [2:7577788924885996812:2058]
2025-11-28T14:51:12.039971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788924885996812:2058] [node 6] ICS09 handshake done sender: [2:7577788920591029495:2111] self: [2:7577788920591029494:0] peer: [6:7577788917137912066:0] socket: 68 qp: -1
2025-11-28T14:51:12.039998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788924885996812:2058] [node 6] ICS10 traffic start
2025-11-28T14:51:12.040026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:12.048593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788913585173981:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:12.051612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788920591029499:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.050923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788918460373938:2072] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:12.069972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788917880141359:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.068785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788918460373983:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.068879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788918460373934:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.082769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788922755341287:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.083428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788922175108658:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48690
2025-11-28T14:51:12.084844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788920591029499:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:12.088519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788917822346612:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:12.089204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788913527379292:2090] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.084699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788918460373934:2070] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:12.108569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788913527379292:2090] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:12.152550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577788918460373931:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.152660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:12.152727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788918460373931:2064] [node 6] ICP22 created new session: [5:7577788922755341290:2064]
2025-11-28T14:51:12.152777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788922755341290:2064] [node 6] ICS09 handshake done sender: [5:7577788918460373934:2070] self: [5:7577788918460373933:0] peer: [6:7577788917137912062:0] socket: 46 qp: -1
2025-11-28T14:51:12.152812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788922755341290:2064] [node 6] ICS10 traffic start
2025-11-28T14:51:12.152850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:12.166571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788919264470167:2091] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:12.167243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788919264470165:2090] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.167383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788919264470197:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.156608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788913527379248:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.160499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:12.168551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788913527379248:2048] [node 2] ICP22 created new session: [1:7577788922117313926:2048]
2025-11-28T14:51:12.168599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788922117313926:2048] [node 2] ICS09 handshake done sender: [1:7577788913527379292:2090] self: [1:7577788913527379291:0] peer: [2:7577788924885996809:0] socket: 53 qp: -1
2025-11-28T14:51:12.168631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788922117313926:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:12.168691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:12.172748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788917880141358:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.172849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788917880141356:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.172906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788917880141355:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.173551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788919264470196:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.193010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788922755341292:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-11-28T14:51:12.173622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788919264470199:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.174110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788914969502819:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:12.174164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577788914969502849:2066] [node 6] ICP18 handshake (actor [3:7577788919264470196:2117]) is too old
2025-11-28T14:51:12.174207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788914969502820:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:12.236171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788919264470165:2090] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:12.234525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788913585173981:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.234617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:12.234661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788913585173981:2049] [node 2] ICP22 created new session: [4:7577788922175108659:2049]
2025-11-28T14:51:12.234704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788922175108659:2049] [node 2] ICS09 handshake done sender: [4:7577788917880141355:2116] self: [4:7577788922175108656:0] peer: [2:7577788920591029498:0] socket: 79 qp: -1
2025-11-28T14:51:12.234738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788922175108659:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:12.234780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:12.235348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788913585173980:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:12.235401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788913585173982:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:12.237647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788914969502850:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.237712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:12.237754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788914969502850:2068] [node 5] ICP22 created new session: [3:7577788923559437506:2068]
2025-11-28T14:51:12.237799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788923559437506:2068] [node 5] ICS09 handshake done sender: [3:7577788919264470165:2090] self: [3:7577788919264470164:0] peer: [5:7577788918460373985:0] socket: 43 qp: -1
2025-11-28T14:51:12.237832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788923559437506:2068] [node 5] ICS10 traffic start
2025-11-28T14:51:12.237863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.239420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788923559437505:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39451
2025-11-28T14:51:12.236657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788913527379294:2091] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.240838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788913527379294:2091] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.241610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788913527379249:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.241654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:12.241683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788913527379249:2049] [node 3] ICP22 created new session: [1:7577788922117313931:2049]
2025-11-28T14:51:12.241712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788922117313931:2049] [node 3] ICS09 handshake done sender: [1:7577788913527379294:2091] self: [1:7577788913527379293:0] peer: [3:7577788923559437503:0] socket: 54 qp: -1
2025-11-28T14:51:12.241736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788922117313931:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:12.241777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:12.242259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788922117313930:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25708
2025-11-28T14:51:12.242835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788918460373930:2069] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.243644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788914165406610:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7066) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788918460373930:2069] held: no
2025-11-28T14:51:12.243715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788914165406610:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.243741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:12.243780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788914165406610:2050] [node 3] ICP22 created new session: [5:7577788922755341293:2050]
2025-11-28T14:51:12.243813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788922755341293:2050] [node 3] ICS09 handshake done sender: [5:7577788918460373930:2069] self: [5:7577788918460373985:0] peer: [3:7577788919264470164:0] socket: 44 qp: -1
2025-11-28T14:51:12.254085Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788917137912037:2099] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:12.243834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788922755341293:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:12.243857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:12.244551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788922755341295:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.264598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788919264470167:2091] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.257335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788917822346612:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.264167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788917822346612:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:12.266260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788919264470197:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:12.293419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788912842944711:2062] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788917137912041:2102] held: no
2025-11-28T14:51:12.293869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788921432879363:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:12.330180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788913527379295:2081] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.330258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:12.330331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788913527379295:2081] [node 4] ICP22 created new session: [1:7577788922117313934:2081]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.300789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788914969502819:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.300846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:12.300892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788914969502819:2048] [node 1] ICP22 created new session: [3:7577788923559437510:2048]
2025-11-28T14:51:12.300944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788923559437510:2048] [node 1] ICS09 handshake done sender: [3:7577788919264470197:2118] self: [3:7577788923559437503:0] peer: [1:7577788913527379293:0] socket: 74 qp: -1
2025-11-28T14:51:12.300972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788923559437510:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:12.301006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:12.336811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788919264470167:2091] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:12.330378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788922117313934:2081] [node 4] ICS09 handshake done sender: [1:7577788917822346612:2112] self: [1:7577788917822346611:0] peer: [4:7577788922175108661:0] socket: 83 qp: -1
2025-11-28T14:51:12.330426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788922117313934:2081] [node 4] ICS10 traffic start
2025-11-28T14:51:12.330470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:12.339332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788914969502851:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.337415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788921432879364:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:12.341170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577788913527379249:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:12.339440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:12.339484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788914969502851:2070] [node 4] ICP22 created new session: [3:7577788923559437512:2070]
2025-11-28T14:51:12.339557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788923559437512:2070] [node 4] ICS09 handshake done sender: [3:7577788919264470167:2091] self: [3:7577788919264470166:0] peer: [4:7577788922175108662:0] socket: 78 qp: -1
2025-11-28T14:51:12.339591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788923559437512:2070] [node 4] ICS10 traffic start
2025-11-28T14:51:12.339629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.344608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788921432879365:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:12.344830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788917137912059:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:12.347771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788912842944684:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.347836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:12.347872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788912842944684:2049] [node 2] ICP22 created new session: [6:7577788921432879367:2049]
2025-11-28T14:51:12.347910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788921432879367:2049] [node 2] ICS09 handshake done sender: [6:7577788917137912059:2117] self: [6:7577788917137912066:0] peer: [2:7577788920591029494:0] socket: 71 qp: -1
2025-11-28T14:51:12.347942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788921432879367:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:12.347976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:12.341284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788920591029501:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:12.354737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788917137912041:2102] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:12.417201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788922755341287:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.405923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788916296062153:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.429095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788912842944711:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.429189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:12.429244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788912842944711:2062] [node 5] ICP22 created new session: [6:7577788921432879369:2062]
2025-11-28T14:51:12.429289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788921432879369:2062] [node 5] ICS09 handshake done sender: [6:7577788917137912041:2102] self: [6:7577788917137912062:0] peer: [5:7577788918460373933:0] socket: 48 qp: -1
2025-11-28T14:51:12.429321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788921432879369:2062] [node 5] ICS10 traffic start
2025-11-28T14:51:12.429354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:12.406012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:12.432306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788917880141349:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.406061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788916296062153:2063] [node 4] ICP22 created new session: [2:7577788924885996816:2063]
2025-11-28T14:51:12.406118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788924885996816:2063] [node 4] ICS09 handshake done sender: [2:7577788920591029499:2113] self: [2:7577788920591029498:0] peer: [4:7577788922175108656:0] socket: 70 qp: -1
2025-11-28T14:51:12.406147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788924885996816:2063] [node 4] ICS10 traffic start
2025-11-28T14:51:12.406186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:12.416706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788920591029497:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.416828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788924885996806:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.455986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788920591029501:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.456419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788924885996818:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13659
2025-11-28T14:51:12.424939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788914165406609:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.481157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577788916296062152:2058] [node 6] ICP18 handshake (actor [2:7577788924885996806:2119]) is too old
2025-11-28T14:51:12.482412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788913585173982:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7066) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577788917880141356:2117] held: no
2025-11-28T14:51:12.482517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788913585173980:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27748) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577788917880141358:2118] held: no
2025-11-28T14:51:12.483094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788922175108664:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48690
2025-11-28T14:51:12.467274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788922755341295:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.489219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788920591029497:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.494055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788917822346618:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:12.495577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788921432879365:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.495676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788912842944683:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:12.496215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788920591029503:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:12.496511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788916296062132:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.496569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:12.496608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788916296062132:2049] [node 3] ICP22 created new session: [2:7577788924885996822:2049]
2025-11-28T14:51:12.496660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788924885996822:2049] [node 3] ICS09 handshake done sender: [2:7577788920591029497:2112] self: [2:7577788920591029496:0] peer: [3:7577788923559437504:0] socket: 69 qp: -1
2025-11-28T14:51:12.496705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788924885996822:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:12.496773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:12.497429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788920591029501:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:12.500712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577788916296062131:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.500763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:12.500799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788916296062131:2048] [node 1] ICP22 created new session: [2:7577788924885996824:2048]
2025-11-28T14:51:12.500835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788924885996824:2048] [node 1] ICS09 handshake done sender: [2:7577788920591029503:2116] self: [2:7577788924885996809:0] peer: [1:7577788913527379291:0] socket: 66 qp: -1
2025-11-28T14:51:12.500863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788924885996824:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:12.500892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:12.495413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788918460373932:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:12.495676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788922755341297:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-11-28T14:51:12.497946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788922755341287:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:12.498092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788914165406609:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.498133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:12.498177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788914165406609:2049] [node 2] ICP22 created new session: [5:7577788922755341299:2049]
2025-11-28T14:51:12.498234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788922755341299:2049] [node 2] ICS09 handshake done sender: [5:7577788922755341287:2119] self: [5:7577788922755341296:0] peer: [2:7577788920591029500:0] socket: 92 qp: -1
2025-11-28T14:51:12.498265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788922755341299:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:12.498292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.502348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788922175108665:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48690
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.502844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788917880141349:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.503083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788917880141358:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:12.503369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788921432879372:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:12.503734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788917880141356:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.503907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788913585173980:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.503946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:12.503977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788913585173980:2048] [node 1] ICP22 created new session: [4:7577788922175108668:2048]
2025-11-28T14:51:12.504012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788922175108668:2048] [node 1] ICS09 handshake done sender: [4:7577788917880141358:2118] self: [4:7577788922175108661:0] peer: [1:7577788917822346611:0] socket: 85 qp: -1
2025-11-28T14:51:12.504042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788922175108668:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:12.504070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:12.504538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788913585173982:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.504584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:12.504627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788913585173982:2050] [node 3] ICP22 created new session: [4:7577788922175108670:2050]
2025-11-28T14:51:12.504658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788922175108670:2050] [node 3] ICS09 handshake done sender: [4:7577788917880141356:2117] self: [4:7577788922175108662:0] peer: [3:7577788919264470166:0] socket: 80 qp: -1
2025-11-28T14:51:12.504690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788922175108670:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:12.504719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:12.505004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788922755341301:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-11-28T14:51:12.513533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788916296062155:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.513599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:12.513639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788916296062155:2066] [node 5] ICP22 created new session: [2:7577788924885996826:2066]
2025-11-28T14:51:12.513676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788924885996826:2066] [node 5] ICS09 handshake done sender: [2:7577788920591029501:2114] self: [2:7577788920591029500:0] peer: [5:7577788922755341296:0] socket: 91 qp: -1
2025-11-28T14:51:12.513706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788924885996826:2066] [node 5] ICS10 traffic start
2025-11-28T14:51:12.513756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:12.514906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788917880141349:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.517021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788917822346618:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.517664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788922117313936:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25708
2025-11-28T14:51:12.517828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788922117313937:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25708
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.536981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788923559437514:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39451
2025-11-28T14:51:12.550339Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577788913527379248:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:12.561467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788917822346618:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:12.562581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788913527379297:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.562631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:12.562670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788913527379297:2082] [node 6] ICP22 created new session: [1:7577788922117313939:2082]
2025-11-28T14:51:12.562713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788922117313939:2082] [node 6] ICS09 handshake done sender: [1:7577788917822346618:2114] self: [1:7577788917822346615:0] peer: [6:7577788921432879371:0] socket: 63 qp: -1
2025-11-28T14:51:12.562760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788922117313939:2082] [node 6] ICS10 traffic start
2025-11-28T14:51:12.562814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:12.561960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577788914969502820:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23349) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577788919264470199:2120] held: no
2025-11-28T14:51:12.562055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788919264470199:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:12.563572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788914969502820:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.563610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:12.563652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788914969502820:2049] [node 2] ICP22 created new session: [3:7577788923559437515:2049]
2025-11-28T14:51:12.563690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788923559437515:2049] [node 2] ICS09 handshake done sender: [3:7577788919264470199:2120] self: [3:7577788923559437504:0] peer: [2:7577788920591029496:0] socket: 76 qp: -1
2025-11-28T14:51:12.563721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788923559437515:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:12.563748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:12.562320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788922755341295:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:12.565237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788918460373932:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.565398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:12.565500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788918460373932:2068] [node 4] ICP22 created new session: [5:7577788922755341302:2068]
2025-11-28T14:51:12.565751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788922755341302:2068] [node 4] ICS09 handshake done sender: [5:7577788922755341295:2121] self: [5:7577788922755341298:0] peer: [4:7577788917880141348:0] socket: 59 qp: -1
2025-11-28T14:51:12.565783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788922755341302:2068] [node 4] ICS10 traffic start
2025-11-28T14:51:12.566137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:12.585857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788917880141351:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.586575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788917880141345:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.586644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:12.586694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788917880141345:2069] [node 5] ICP22 created new session: [4:7577788922175108672:2069]
2025-11-28T14:51:12.586732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788922175108672:2069] [node 5] ICS09 handshake done sender: [4:7577788917880141349:2114] self: [4:7577788917880141348:0] peer: [5:7577788922755341298:0] socket: 40 qp: -1
2025-11-28T14:51:12.586764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788922175108672:2069] [node 5] ICS10 traffic start
2025-11-28T14:51:12.586796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:12.623031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788917137912039:2100] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:12.623827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788921432879372:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.624048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788921432879373:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:12.628609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:12.628870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577788924885996824:2048] [node 1] ICS01 socket: 66 reason# EndOfStream
2025-11-28T14:51:12.628922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577788916296062131:2048] [node 1] ICP30 unregister session Session# [2:7577788924885996824:2048] VirtualId# [2:7577788924885996809:0]
2025-11-28T14:51:12.628961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577788924885996824:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.629045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:12.629087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:12.629931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788912842944709:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:12.630163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788921432879365:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:12.630466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788917880141351:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:12.630896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788917880141351:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:12.631100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788917880141346:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:12.631173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:12.631218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788917880141346:2070] [node 6] ICP22 created new session: [4:7577788922175108675:2070]
2025-11-28T14:51:12.631257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788922175108675:2070] [node 6] ICS09 handshake done sender: [4:7577788917880141351:2115] self: [4:7577788917880141350:0] peer: [6:7577788921432879374:0] socket: 39 qp: -1
2025-11-28T14:51:12.631286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788922175108675:2070] [node 6] ICS10 traffic start
2025-11-28T14:51:12.631321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:12.644954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:12.645149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788918460373988:2048] [node 1] ICS01 socket: 64 reason# EndOfStream
2025-11-28T14:51:12.645188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788914165406608:2048] [node 1] ICP30 unregister session Session# [5:7577788918460373988:2048] VirtualId# [5:7577788918460373986:0]
2025-11-28T14:51:12.645215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788918460373988:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.645285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:12.645325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:12.640555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788912842944683:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.640659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:12.640716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788912842944683:2048] [node 1] ICP22 created new session: [6:7577788921432879376:2048]
2025-11-28T14:51:12.640771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788921432879376:2048] [node 1] ICS09 handshake done sender: [6:7577788921432879365:2121] self: [6:7577788921432879371:0] peer: [1:7577788917822346615:0] socket: 102 qp: -1
2025-11-28T14:51:12.640809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788921432879376:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:12.640853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:12.644597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788912842944709:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12990) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788921432879372:2122] held: no
2025-11-28T14:51:12.648467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:12.649821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788923559437510:2048] [node 1] ICS01 socket: 74 reason# EndOfStream
2025-11-28T14:51:12.649868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788914969502819:2048] [node 1] ICP30 unregister session Session# [3:7577788923559437510:2048] VirtualId# [3:7577788923559437503:0]
2025-11-28T14:51:12.649891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788923559437510:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.649958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:12.649996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:12.648855Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:12.649078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788922175108668:2048] [node 1] ICS01 socket: 85 reason# EndOfStream
2025-11-28T14:51:12.649125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788913585173980:2048] [node 1] ICP30 unregister session Session# [4:7577788922175108668:2048] VirtualId# [4:7577788922175108661:0]
2025-11-28T14:51:12.649156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788922175108668:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.649242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:12.649280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:12.664604Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:12.664828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788923559437515:2049] [node 2] ICS01 socket: 76 reason# EndOfStream
2025-11-28T14:51:12.664868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788914969502820:2049] [node 2] ICP30 unregister session Session# [3:7577788923559437515:2049] VirtualId# [3:7577788923559437504:0]
2025-11-28T14:51:12.664895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788923559437515:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.664987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:12.665023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:12.665459Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:12.665662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788921432879376:2048] [node 1] ICS01 socket: 102 reason# EndOfStream
2025-11-28T14:51:12.665686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788912842944683:2048] [node 1] ICP30 unregister session Session# [6:7577788921432879376:2048] VirtualId# [6:7577788921432879371:0]
2025-11-28T14:51:12.665709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788921432879376:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.665726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:12.665765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:12.672557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:12.672712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788922175108659:2049] [node 2] ICS01 socket: 79 reason# EndOfStream
2025-11-28T14:51:12.672736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788913585173981:2049] [node 2] ICP30 unregister session Session# [4:7577788922175108659:2049] VirtualId# [4:7577788922175108656:0]
2025-11-28T14:51:12.672752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788922175108659:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.672809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:12.672835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:12.673082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788921432879375:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27243
2025-11-28T14:51:12.675452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:12.684212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:12.684463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788921432879367:2049] [node 2] ICS01 socket: 71 reason# EndOfStream
2025-11-28T14:51:12.684525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788912842944684:2049] [node 2] ICP30 unregister session Session# [6:7577788921432879367:2049] VirtualId# [6:7577788917137912066:0]
2025-11-28T14:51:12.684552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788921432879367:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.684621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:12.684658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:12.689875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788921432879372:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:12.680447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788922755341299:2049] [node 2] ICS01 socket: 92 reason# EndOfStream
2025-11-28T14:51:12.683853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788914165406609:2049] [node 2] ICP30 unregister session Session# [5:7577788922755341299:2049] VirtualId# [5:7577788922755341296:0]
2025-11-28T14:51:12.683900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788922755341299:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.684013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:12.684064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:12.691626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788912842944709:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:12.691687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:12.691732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788912842944709:2059] [node 4] ICP22 created new session: [6:7577788921432879378:2059]
2025-11-28T14:51:12.691782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788921432879378:2059] [node 4] ICS09 handshake done sender: [6:7577788921432879372:2122] self: [6:7577788921432879374:0] peer: [4:7577788917880141350:0] socket: 106 qp: -1
2025-11-28T14:51:12.691810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788921432879378:2059] [node 4] ICS10 traffic start
2025-11-28T14:51:12.691840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:12.696551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:12.696791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788922755341293:2050] [node 3] ICS01 socket: 44 reason# EndOfStream
2025-11-28T14:51:12.696836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788914165406610:2050] [node 3] ICP30 unregister session Session# [5:7577788922755341293:2050] VirtualId# [5:7577788918460373985:0]
2025-11-28T14:51:12.696863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788922755341293:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.696952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:12.696996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:12.697327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:12.697529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788922175108670:2050] [node 3] ICS01 socket: 80 reason# EndOfStream
2025-11-28T14:51:12.697567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788913585173982:2050] [node 3] ICP30 unregister session Session# [4:7577788922175108670:2050] VirtualId# [4:7577788922175108662:0]
2025-11-28T14:51:12.697589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788922175108670:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.697648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:12.697680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:12.698239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:12.698432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788917137912063:2050] [node 3] ICS01 socket: 47 reason# EndOfStream
2025-11-28T14:51:12.698467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788912842944685:2050] [node 3] ICP30 unregister session Session# [6:7577788917137912063:2050] VirtualId# [6:7577788917137912060:0]
2025-11-28T14:51:12.698488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788917137912063:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.698542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:12.698586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:12.711633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:12.711871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788921432879378:2059] [node 4] ICS01 socket: 106 reason# EndOfStream
2025-11-28T14:51:12.711920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788912842944709:2059] [node 4] ICP30 unregister session Session# [6:7577788921432879378:2059] VirtualId# [6:7577788921432879374:0]
2025-11-28T14:51:12.711950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788921432879378:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.712057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:12.712099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:12.720565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:12.720823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788921432879369:2062] [node 5] ICS01 socket: 48 reason# EndOfStream
2025-11-28T14:51:12.720859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788912842944711:2062] [node 5] ICP30 unregister session Session# [6:7577788921432879369:2062] VirtualId# [6:7577788917137912062:0]
2025-11-28T14:51:12.720892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788921432879369:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:12.720970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:12.721010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:12.951978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788924950364891:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:12.952021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788924950364892:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:12.984000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788921682571718:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:12.984039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788921682571719:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:13.017513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788929310960966:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:13.017553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788929310960967:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:13.100760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788926316116184:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:13.100809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788926316116185:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:13.100834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788926316116186:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:13.220877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788927516633198:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:13.220930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788927516633199:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:13.220958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788927516633200:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:13.533282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788926316116197:2059] [node 6] ICP01 ready to work
2025-11-28T14:51:13.601742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788926316116205:2064] [node 5] ICP01 ready to work
2025-11-28T14:51:13.661477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788929310960992:2062] [node 5] ICP01 ready to work
2025-11-28T14:51:13.416835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788928512874560:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:13.665719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788927516633216:2062] [node 6] ICP01 ready to work
2025-11-28T14:51:13.663425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788929310960995:2068] [node 6] ICP01 ready to work
2025-11-28T14:51:13.717742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788927516633222:2061] [node 4] ICP01 ready to work
2025-11-28T14:51:13.416888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788928512874561:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:13.416910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788928512874562:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:13.919770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788929245332250:2099] [node 4] ICP01 ready to work
2025-11-28T14:51:13.926068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788926316116185:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(302723054ns) {Configure:104(302718227ns) {ProcessConfigured:135(38541187ns) {StartInitialHandshake:157(38537658ns) {DropHandshakes:470(21240923ns) {DropIncomingHandshake:442(751ns) DropOutgoingHandshake:457(1840ns)} GenerateSessionVirtualId:988(2599ns) PrepareNewSessionHandshake:477(4001ns) {SwitchToState:179(1693ns)}}}}}
2025-11-28T14:51:13.929867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788925977539077:2064] [node 6] ICP01 ready to work
2025-11-28T14:51:13.964932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788925977539078:2067] [node 5] ICP01 ready to work
2025-11-28T14:51:13.960712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788929245332251:2101] [node 6] ICP01 ready to work
2025-11-28T14:51:13.621087Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788928512874562:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(89853107ns) {Configure:104(89847694ns) {ProcessConfigured:135(17102431ns) {StartInitialHandshake:157(17098800ns) {DropHandshakes:470(4149ns) {DropIncomingHandshake:442(1027ns) DropOutgoingHandshake:457(1259ns)} GenerateSessionVirtualId:988(974ns) PrepareNewSessionHandshake:477(4775ns) {SwitchToState:179(2327ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:13.987708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788929245332252:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4163
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:13.975659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788929310961035:2060] [node 4] ICP01 ready to work
2025-11-28T14:51:13.977007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788925977539079:2065] [node 4] ICP01 ready to work
2025-11-28T14:51:13.626768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788928512874582:2060] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:13.990739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788929245332253:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4163
2025-11-28T14:51:13.994223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788927516633198:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(117472383ns) {Configure:104(117466772ns) {ProcessConfigured:135(12943542ns) {StartInitialHandshake:157(12941754ns) {DropHandshakes:470(1010ns) {DropIncomingHandshake:442(185ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(373ns) PrepareNewSessionHandshake:477(2268ns) {SwitchToState:179(1057ns)}}}}}
2025-11-28T14:51:13.898661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788928512874561:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(270420461ns) {Configure:104(270410015ns) {ProcessConfigured:135(75014ns) {StartInitialHandshake:157(69683ns) {DropHandshakes:470(3091ns) {DropIncomingHandshake:442(796ns) DropOutgoingHandshake:457(449ns)} GenerateSessionVirtualId:988(852ns) PrepareNewSessionHandshake:477(1835ns) {SwitchToState:179(715ns)}}}}}
2025-11-28T14:51:14.060746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788926316116197:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(475513503ns) {Configure:104(475509799ns) {ProcessConfigured:135(28292420ns) {StartInitialHandshake:157(28288911ns) {DropHandshakes:470(3081ns) {DropIncomingHandshake:442(723ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(907ns) PrepareNewSessionHandshake:477(4263ns) {SwitchToState:179(2124ns)}}}}}
2025-11-28T14:51:13.953079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788928512874634:2074] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.021041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788929310960967:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(356429059ns) {Configure:104(356426038ns) {ProcessConfigured:135(25855141ns) {StartInitialHandshake:157(25852867ns) {DropHandshakes:470(2441ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(512ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:477(5027ns) {SwitchToState:179(1899ns)}}}}}
2025-11-28T14:51:14.022072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788929310961038:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18208
2025-11-28T14:51:14.078999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788933540299561:2100] [node 5] ICP01 ready to work
2025-11-28T14:51:14.022322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788929310961039:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18208
2025-11-28T14:51:14.095961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788928512874578:2063] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.076783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788927516633200:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(140921102ns) {Configure:104(140915604ns) {ProcessConfigured:135(11512143ns) {StartInitialHandshake:157(11509076ns) {DropHandshakes:470(3122ns) {DropIncomingHandshake:442(766ns) DropOutgoingHandshake:457(453ns)} GenerateSessionVirtualId:988(1001ns) PrepareNewSessionHandshake:477(3912ns) {SwitchToState:179(1606ns)}}}}}
2025-11-28T14:51:14.138425Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788921682571719:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(143229175ns) {Configure:104(143224985ns) {ProcessConfigured:135(83879ns) {StartInitialHandshake:157(81348ns) {DropHandshakes:470(3693ns) {DropIncomingHandshake:442(743ns) DropOutgoingHandshake:457(425ns)} GenerateSessionVirtualId:988(994ns) PrepareNewSessionHandshake:477(2004ns) {SwitchToState:179(807ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.078185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788927516633267:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21269
2025-11-28T14:51:14.078326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788927516633268:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21269
2025-11-28T14:51:14.078550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788931811600570:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21269
2025-11-28T14:51:14.144357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788927516633222:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(149872421ns) {Configure:104(149863600ns) {ProcessConfigured:135(15719397ns) {StartInitialHandshake:157(15716481ns) {DropHandshakes:470(3200ns) {DropIncomingHandshake:442(786ns) DropOutgoingHandshake:457(454ns)} GenerateSessionVirtualId:988(1378ns) PrepareNewSessionHandshake:477(2235ns) {SwitchToState:179(852ns)}}}}}
2025-11-28T14:51:14.168770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788933540299561:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50845125ns) {Configure:104(50833916ns) {ProcessConfigured:135(49730009ns) {StartInitialHandshake:157(49725833ns) {DropHandshakes:470(2768ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(576ns)} GenerateSessionVirtualId:988(689ns) PrepareNewSessionHandshake:477(2014ns) {SwitchToState:179(401ns)}}}}}
2025-11-28T14:51:14.120970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788929310961035:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(128883581ns) {Configure:104(128879560ns) {ProcessConfigured:135(239836ns) {StartInitialHandshake:157(238079ns) {DropHandshakes:470(998ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(173ns)} GenerateSessionVirtualId:988(592ns) PrepareNewSessionHandshake:477(8670ns) {SwitchToState:179(660ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.188969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788933605928343:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18208
2025-11-28T14:51:14.214674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788926316116200:2062] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.156887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788930272506384:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31054
2025-11-28T14:51:14.157084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788930272506385:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31054
2025-11-28T14:51:14.157207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788930272506386:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31054
2025-11-28T14:51:14.217561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788921682571718:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(161591841ns) {Configure:104(161587860ns) {ProcessConfigured:135(6194496ns) {StartInitialHandshake:157(6191115ns) {DropHandshakes:470(4198ns) {DropIncomingHandshake:442(1451ns) DropOutgoingHandshake:457(780ns)} GenerateSessionVirtualId:988(1839ns) PrepareNewSessionHandshake:477(3738ns) {SwitchToState:179(2040ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.301135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788929310961038:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.295731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788928512874576:2062] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.300703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788932807841934:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53117
2025-11-28T14:51:14.300826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788932807841935:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53117
2025-11-28T14:51:14.300907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788932807841936:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53117
2025-11-28T14:51:14.323386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788929310960997:2075] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.395484Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788926316116254:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.425497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788930611083558:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37736
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.459382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788929245332257:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.460243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788933540299563:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4163
2025-11-28T14:51:14.460355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788933540299564:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4163
2025-11-28T14:51:14.465305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788929310961039:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.465234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788929245332253:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.471382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788929245332252:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.513812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788927516633220:2067] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.514477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788931811600570:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.514538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788927516633268:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.550586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788927516633271:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.551243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788927516633200:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:14.551295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788927516633222:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:14.552599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788926316116254:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.553196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788926316116254:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:14.563326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788926316116205:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:14.563397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.563442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788926316116205:2064] [node 5] ICP22 created new session: [4:7577788930611083560:2064]
2025-11-28T14:51:14.563483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788930611083560:2064] [node 5] ICS09 handshake done sender: [4:7577788926316116254:2111] self: [4:7577788926316116253:0] peer: [5:7577788931811600576:0] socket: 50 qp: -1
2025-11-28T14:51:14.563521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788930611083560:2064] [node 5] ICS10 traffic start
2025-11-28T14:51:14.563553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:14.578830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788931811600577:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21269
2025-11-28T14:51:14.581838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788927516633268:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:14.582782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788927516633222:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:14.582830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:14.582874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788927516633222:2061] [node 4] ICP22 created new session: [5:7577788931811600578:2061]
2025-11-28T14:51:14.582913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788931811600578:2061] [node 4] ICS09 handshake done sender: [5:7577788927516633268:2112] self: [5:7577788931811600576:0] peer: [4:7577788926316116253:0] socket: 51 qp: -1
2025-11-28T14:51:14.582939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788931811600578:2061] [node 4] ICS10 traffic start
2025-11-28T14:51:14.582967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:14.604120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788928512874585:2066] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.609218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788927516633267:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.636213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788926316116252:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.717845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788930272506379:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.718430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788930272506386:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.718528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788930272506384:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.768638Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577788928512874560:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(39900737ns) {HandleCleanupEventQueue:832(39897953ns) {CleanupEventQueue:841(29170ns) {DropSessionEvent:512(24534ns)} ScheduleCleanupEventQueue:822(39863515ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.787817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788931811600582:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21269
2025-11-28T14:51:14.785925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788929310961037:2112] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.786616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788929310960997:2075] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.786902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788933605928343:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.787209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788929310960966:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:14.800643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788932807841934:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.800814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788928512874562:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.804942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788929310960967:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:14.805079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788929310961037:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.805197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788933605928345:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18208
2025-11-28T14:51:14.805320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788933605928346:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18208
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.820711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788930272506379:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.832706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788932807841938:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53117
2025-11-28T14:51:14.836542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788930272506379:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:14.836913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788921682571719:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:14.836983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:14.837028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788921682571719:2049] [node 3] ICP22 created new session: [2:7577788930272506395:2049]
2025-11-28T14:51:14.837066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788930272506395:2049] [node 3] ICS09 handshake done sender: [2:7577788930272506379:2111] self: [2:7577788930272506378:0] peer: [3:7577788933605928349:0] socket: 76 qp: -1
2025-11-28T14:51:14.837095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788930272506395:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:14.837130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:14.834570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788929310960997:2075] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:14.843419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788929310961037:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:14.851421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788927516633218:2066] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.872905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788929310960992:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:14.872961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:14.873001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788929310960992:2062] [node 5] ICP22 created new session: [3:7577788933605928352:2062]
2025-11-28T14:51:14.850155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788932807841934:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.881539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788927516633218:2066] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.881649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788931811600570:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:14.881910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788927516633200:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:14.881961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:14.881991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788927516633200:2050] [node 3] ICP22 created new session: [5:7577788931811600585:2050]
2025-11-28T14:51:14.882040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788931811600585:2050] [node 3] ICS09 handshake done sender: [5:7577788931811600570:2116] self: [5:7577788931811600575:0] peer: [3:7577788929310960996:0] socket: 66 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.871670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788932807841935:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.891413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788930272506381:2112] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.882070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788931811600585:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:14.871719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788928512874562:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.873038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788933605928352:2062] [node 5] ICS09 handshake done sender: [3:7577788929310960997:2075] self: [3:7577788929310960996:0] peer: [5:7577788931811600575:0] socket: 60 qp: -1
2025-11-28T14:51:14.892044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788930272506398:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31054
2025-11-28T14:51:14.882114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:14.967939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577788926316116186:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(29029516ns) {HandleCleanupEventQueue:832(29027022ns) {CleanupEventQueue:841(29024794ns) {DropSessionEvent:512(20476ns) DropSessionEvent:512(21125002ns) DropSessionEvent:512(2888ns) DropSessionEvent:512(3367ns) DropSessionEvent:512(3698ns) DropSessionEvent:512(3475ns) DropSessionEvent:512(2945ns) DropSessionEvent:512(3159ns) DropSessionEvent:512(2585ns) DropSessionEvent:512(7848253ns)} ScheduleCleanupEventQueue:822(444ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.871774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:14.967438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788930272506381:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.871820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788928512874562:2050] [node 3] ICP22 created new session: [6:7577788932807841939:2050]
2025-11-28T14:51:14.882934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788931811600583:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21269
2025-11-28T14:51:14.873064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788933605928352:2062] [node 5] ICS10 traffic start
2025-11-28T14:51:14.981035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788930272506381:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.023905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788929245332259:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.883201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788927516633218:2066] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:14.992729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788930611083563:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37736
2025-11-28T14:51:15.020867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788934906050860:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37736
2025-11-28T14:51:14.871857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788932807841939:2050] [node 3] ICS09 handshake done sender: [6:7577788932807841934:2115] self: [6:7577788932807841937:0] peer: [3:7577788929310961036:0] socket: 79 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:14.873096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:14.873557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788929310960995:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:14.873602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:14.873649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788929310960995:2068] [node 6] ICP22 created new session: [3:7577788933605928354:2068]
2025-11-28T14:51:14.873698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788933605928354:2068] [node 6] ICS09 handshake done sender: [3:7577788929310961037:2112] self: [3:7577788929310961036:0] peer: [6:7577788932807841937:0] socket: 61 qp: -1
2025-11-28T14:51:14.873722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788933605928354:2068] [node 6] ICS10 traffic start
2025-11-28T14:51:14.873758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:14.964859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788929310960994:2074] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:14.967022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788933605928351:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18208
2025-11-28T14:51:14.985065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788933605928343:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:15.012649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788929310960967:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:15.012736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:15.012789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788929310960967:2049] [node 2] ICP22 created new session: [3:7577788937900895654:2049]
2025-11-28T14:51:15.012857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788937900895654:2049] [node 2] ICS09 handshake done sender: [3:7577788933605928343:2117] self: [3:7577788933605928349:0] peer: [2:7577788930272506378:0] socket: 82 qp: -1
2025-11-28T14:51:15.012891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788937900895654:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:15.012928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:14.883384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577788927516633216:2062] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.072626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788925977539077:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:15.072693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:15.072733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788925977539077:2064] [node 6] ICP22 created new session: [2:7577788934567473697:2064]
2025-11-28T14:51:15.072773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788934567473697:2064] [node 6] ICS09 handshake done sender: [2:7577788930272506381:2112] self: [2:7577788930272506380:0] peer: [6:7577788932807841944:0] socket: 77 qp: -1
2025-11-28T14:51:15.072806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788934567473697:2064] [node 6] ICS10 traffic start
2025-11-28T14:51:15.072838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:14.883419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:14.883450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788927516633216:2062] [node 6] ICP22 created new session: [5:7577788931811600587:2062]
2025-11-28T14:51:14.883484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788931811600587:2062] [node 6] ICS09 handshake done sender: [5:7577788927516633218:2066] self: [5:7577788927516633217:0] peer: [6:7577788932807841941:0] socket: 65 qp: -1
2025-11-28T14:51:14.883504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788931811600587:2062] [node 6] ICS10 traffic start
2025-11-28T14:51:14.883525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:15.073727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788937900895656:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18208
2025-11-28T14:51:14.871887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788932807841939:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:15.048859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788933540299564:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.083100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788929310961039:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:15.050042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788930611083558:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.074340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788926316116184:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:15.083305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577788926316116184:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24140875ns) {IncomingHandshake:261(17678585ns) {DropIncomingHandshake:442(213ns) PrepareNewSessionHandshake:477(896ns) {SwitchToState:179(354ns)} GenerateSessionVirtualId:988(797ns) IssueIncomingHandshakeReply:193(9224390ns) {SwitchToState:179(962ns)}}}
2025-11-28T14:51:14.871932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:15.049064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788929245332257:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.049200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788933540299563:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.049531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788933540299566:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4163
2025-11-28T14:51:15.064997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788929245332257:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:15.121221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788924950364892:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:15.132639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:15.133091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788924950364892:2049] [node 3] ICP22 created new session: [1:7577788937835266868:2049]
2025-11-28T14:51:15.133001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788929310960966:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:15.133149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788937835266868:2049] [node 3] ICS09 handshake done sender: [1:7577788929245332257:2113] self: [1:7577788929245332256:0] peer: [3:7577788933605928348:0] socket: 56 qp: -1
2025-11-28T14:51:15.133084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:15.133185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788937835266868:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:15.133245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:15.133157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788929310960966:2048] [node 1] ICP22 created new session: [3:7577788937900895657:2048]
2025-11-28T14:51:15.133573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788929245332259:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.133218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788937900895657:2048] [node 1] ICS09 handshake done sender: [3:7577788929310961039:2114] self: [3:7577788933605928348:0] peer: [1:7577788929245332256:0] socket: 58 qp: -1
2025-11-28T14:51:15.133252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788937900895657:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:15.133330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:14.880779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788928512874582:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:14.956625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788928512874582:2060] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5459) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788932807841935:2116] held: no
2025-11-28T14:51:14.956856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788932807841936:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:14.957324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788932807841943:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53117
2025-11-28T14:51:14.960942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788928512874561:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:14.961253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788932807841935:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:14.965588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788928512874582:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:14.965633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:14.965673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788928512874582:2060] [node 5] ICP22 created new session: [6:7577788932807841946:2060]
2025-11-28T14:51:14.965716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788932807841946:2060] [node 5] ICS09 handshake done sender: [6:7577788932807841935:2116] self: [6:7577788932807841941:0] peer: [5:7577788927516633217:0] socket: 80 qp: -1
2025-11-28T14:51:14.965742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788932807841946:2060] [node 5] ICS10 traffic start
2025-11-28T14:51:14.965771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:15.068738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788932807841948:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53117
2025-11-28T14:51:15.156708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788929245332255:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.202469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788936106567885:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21269
2025-11-28T14:51:15.192695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788929245332259:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:15.200814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788929245332250:2099] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:15.200884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:15.200933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788929245332250:2099] [node 4] ICP22 created new session: [1:7577788937835266872:2099]
2025-11-28T14:51:15.200981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788937835266872:2099] [node 4] ICS09 handshake done sender: [1:7577788929245332259:2114] self: [1:7577788929245332258:0] peer: [4:7577788934906050861:0] socket: 57 qp: -1
2025-11-28T14:51:15.201019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788937835266872:2099] [node 4] ICS10 traffic start
2025-11-28T14:51:15.201062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:15.217396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788931811600569:2115] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:15.218382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788933605928346:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.218496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577788929310960992:2062] [node 5] ICP18 handshake (actor [3:7577788933605928346:2119]) is too old
2025-11-28T14:51:15.267658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788926316116256:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.268299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788926316116184:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577788930611083558:2115] held: no
2025-11-28T14:51:15.280707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788933605928345:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.289316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788934906050862:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37736
2025-11-28T14:51:15.297695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788930611083558:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:15.298180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788926316116184:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:15.298234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:15.298277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788926316116184:2048] [node 1] ICP22 created new session: [4:7577788934906050864:2048]
2025-11-28T14:51:15.298317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788934906050864:2048] [node 1] ICS09 handshake done sender: [4:7577788930611083558:2115] self: [4:7577788934906050861:0] peer: [1:7577788929245332258:0] socket: 64 qp: -1
2025-11-28T14:51:15.298344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788934906050864:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:15.298374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:15.324780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788930272506383:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.353906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788930272506385:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.354023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788934567473696:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31054
2025-11-28T14:51:15.357299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788930272506383:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.357586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577788921682571718:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.358259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788934906050866:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37736
2025-11-28T14:51:15.358471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788930272506383:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:15.356703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788931811600583:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.356868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788927516633199:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:15.358575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788929245332255:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.360982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788936106567887:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21269
2025-11-28T14:51:15.361426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788931811600583:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:15.361635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788927516633199:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:15.361697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:15.361732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788927516633199:2049] [node 2] ICP22 created new session: [5:7577788936106567888:2049]
2025-11-28T14:51:15.361767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788936106567888:2049] [node 2] ICS09 handshake done sender: [5:7577788931811600583:2120] self: [5:7577788936106567886:0] peer: [2:7577788930272506382:0] socket: 97 qp: -1
2025-11-28T14:51:15.361796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788936106567888:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:15.361828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:15.368503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788928512874636:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:15.369291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788932807841936:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:15.369617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788930611083563:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.364539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788929245332255:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:15.372649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788925977539078:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:15.372728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:15.372774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788925977539078:2067] [node 5] ICP22 created new session: [2:7577788934567473703:2067]
2025-11-28T14:51:15.372814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788934567473703:2067] [node 5] ICS09 handshake done sender: [2:7577788930272506383:2113] self: [2:7577788930272506382:0] peer: [5:7577788936106567886:0] socket: 95 qp: -1
2025-11-28T14:51:15.372843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788934567473703:2067] [node 5] ICS10 traffic start
2025-11-28T14:51:15.372903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:15.384942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788928512874561:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:15.385005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:15.385047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788928512874561:2049] [node 2] ICP22 created new session: [6:7577788937102809247:2049]
2025-11-28T14:51:15.385087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788937102809247:2049] [node 2] ICS09 handshake done sender: [6:7577788932807841936:2117] self: [6:7577788932807841944:0] peer: [2:7577788930272506380:0] socket: 81 qp: -1
2025-11-28T14:51:15.385118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788937102809247:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:15.385176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:15.385771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788937102809245:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53117
2025-11-28T14:51:15.385902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788937102809246:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53117
2025-11-28T14:51:15.499121Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788933605928342:2116] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:15.508622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788934906050860:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.508785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788926316116186:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:15.516591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788933605928342:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.520664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788933605928342:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.528809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788934906050869:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37736
2025-11-28T14:51:15.533706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788930272506377:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:15.553845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788934906050860:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:15.534606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788934567473702:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31054
2025-11-28T14:51:15.535318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788930272506385:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:15.535490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577788921682571718:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:15.535547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:15.535595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788921682571718:2048] [node 1] ICP22 created new session: [2:7577788934567473705:2048]
2025-11-28T14:51:15.535632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788934567473705:2048] [node 1] ICS09 handshake done sender: [2:7577788930272506385:2115] self: [2:7577788934567473701:0] peer: [1:7577788929245332254:0] socket: 73 qp: -1
2025-11-28T14:51:15.535665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788934567473705:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:15.535706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:15.544632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788929310961035:2060] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:15.552512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:15.553065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788929310961035:2060] [node 4] ICP22 created new session: [3:7577788937900895661:2060]
2025-11-28T14:51:15.553111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788937900895661:2060] [node 4] ICS09 handshake done sender: [3:7577788933605928342:2116] self: [3:7577788933605928341:0] peer: [4:7577788934906050868:0] socket: 103 qp: -1
2025-11-28T14:51:15.553141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788937900895661:2060] [node 4] ICS10 traffic start
2025-11-28T14:51:15.553201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:15.553455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577788929310961035:2060] [node 4] ICP35 event processing took too much time PendingConnection:279(24114089ns) {HandleHandshakeStatus:329(24110862ns) {DropIncomingHandshake:442(20295ns) DropOutgoingHandshake:457(577ns) SwitchToState:179(980ns) ProcessPendingSessionEvents:495(972ns)}}
2025-11-28T14:51:15.604726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788926316116186:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:15.604815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:15.604866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788926316116186:2050] [node 3] ICP22 created new session: [4:7577788934906050870:2050]
2025-11-28T14:51:15.604906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788934906050870:2050] [node 3] ICS09 handshake done sender: [4:7577788934906050860:2117] self: [4:7577788934906050868:0] peer: [3:7577788933605928341:0] socket: 104 qp: -1
2025-11-28T14:51:15.604937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788934906050870:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:15.604968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:15.605110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577788926316116186:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:15.617153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788928512874617:2097] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:15.624716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788930272506398:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.624853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577788925977539077:2064] [node 6] ICP18 handshake (actor [2:7577788930272506398:2118]) is too old
2025-11-28T14:51:15.709702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788930611083557:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:15.717969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:15.718211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788937900895657:2048] [node 1] ICS01 socket: 58 reason# EndOfStream
2025-11-28T14:51:15.718263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788929310960966:2048] [node 1] ICP30 unregister session Session# [3:7577788937900895657:2048] VirtualId# [3:7577788933605928348:0]
2025-11-28T14:51:15.718290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788937900895657:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.718338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:15.718370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:15.718733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788936106567885:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.718951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] socket: Connection reset by peer
2025-11-28T14:51:15.719151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577788934567473705:2048] [node 1] ICS07 socket disconnect 73 reason# ECONNRESET
2025-11-28T14:51:15.719176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577788934567473705:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:51:15.719203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:15.719274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577788934567473705:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:15.718919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788927516633198:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:15.720578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788937102809246:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.720832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788928512874634:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:15.724892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788930272506388:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:15.728593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788921682571718:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17168) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:15.728628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577788934567473705:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:15.725585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788930611083557:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.729301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788934906050866:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:15.729534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788926316116185:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:15.732858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788930611083557:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:15.745089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788928512874560:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.745202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788930272506388:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:15.776948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788937102809250:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53117
2025-11-28T14:51:15.777798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788928512874634:2074] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26771) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788937102809246:2122] held: no
2025-11-28T14:51:15.777853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788937102809246:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:15.778046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788928512874634:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:15.778099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:15.778136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788928512874634:2074] [node 4] ICP22 created new session: [6:7577788937102809251:2074]
2025-11-28T14:51:15.778171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788937102809251:2074] [node 4] ICS09 handshake done sender: [6:7577788937102809246:2122] self: [6:7577788937102809249:0] peer: [4:7577788926316116260:0] socket: 111 qp: -1
2025-11-28T14:51:15.778197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788937102809251:2074] [node 4] ICS10 traffic start
2025-11-28T14:51:15.778224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:15.750307Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788921682571718:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17168) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:15.750360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577788934567473705:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:15.750398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577788921682571718:2048] [node 1] ICP30 unregister session Session# [2:7577788934567473705:2048] VirtualId# [2:7577788934567473701:0]
2025-11-28T14:51:15.750436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:15.750481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577788921682571718:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:15.750495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:15.751042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788930272506388:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:15.751221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788925977539079:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:15.751269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:15.751312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788925977539079:2065] [node 4] ICP22 created new session: [2:7577788934567473712:2065]
2025-11-28T14:51:15.751356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788934567473712:2065] [node 4] ICS09 handshake done sender: [2:7577788930272506388:2117] self: [2:7577788930272506387:0] peer: [4:7577788934906050873:0] socket: 87 qp: -1
2025-11-28T14:51:15.752480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788934567473712:2065] [node 4] ICS10 traffic start
2025-11-28T14:51:15.752517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:15.772448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577788921682571718:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:15.736622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:15.737161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788934906050864:2048] [node 1] ICS01 socket: 64 reason# EndOfStream
2025-11-28T14:51:15.737200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788926316116184:2048] [node 1] ICP30 unregister session Session# [4:7577788934906050864:2048] VirtualId# [4:7577788934906050861:0]
2025-11-28T14:51:15.737233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788934906050864:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.737319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:15.737359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:15.752514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788926316116197:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:15.752576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:15.752610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788926316116197:2059] [node 6] ICP22 created new session: [4:7577788934906050874:2059]
2025-11-28T14:51:15.752650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788934906050874:2059] [node 6] ICS09 handshake done sender: [4:7577788930611083557:2114] self: [4:7577788926316116260:0] peer: [6:7577788937102809249:0] socket: 42 qp: -1
2025-11-28T14:51:15.752693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788934906050874:2059] [node 6] ICS10 traffic start
2025-11-28T14:51:15.752717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:15.760860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788934906050876:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37736
2025-11-28T14:51:15.785137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788926316116185:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20089) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577788934906050866:2119] held: no
2025-11-28T14:51:15.785350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788934906050866:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:15.793576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788927516633198:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17168) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788936106567885:2121] held: no
2025-11-28T14:51:15.792772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788926316116185:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:15.792841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:15.793018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788926316116185:2049] [node 2] ICP22 created new session: [4:7577788934906050877:2049]
2025-11-28T14:51:15.793050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788934906050877:2049] [node 2] ICS09 handshake done sender: [4:7577788934906050866:2119] self: [4:7577788934906050873:0] peer: [2:7577788930272506387:0] socket: 106 qp: -1
2025-11-28T14:51:15.793077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788934906050877:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:15.793123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:15.836488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:15.836782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788937900895654:2049] [node 2] ICS01 socket: 82 reason# EndOfStream
2025-11-28T14:51:15.836822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788929310960967:2049] [node 2] ICP30 unregister session Session# [3:7577788937900895654:2049] VirtualId# [3:7577788933605928349:0]
2025-11-28T14:51:15.836854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788937900895654:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.836966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:15.837011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:15.840983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:15.841202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788936106567888:2049] [node 2] ICS01 socket: 97 reason# EndOfStream
2025-11-28T14:51:15.841235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788927516633199:2049] [node 2] ICP30 unregister session Session# [5:7577788936106567888:2049] VirtualId# [5:7577788936106567886:0]
2025-11-28T14:51:15.841257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788936106567888:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.841332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:15.841371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:15.840760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:15.840989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788937102809247:2049] [node 2] ICS01 socket: 81 reason# EndOfStream
2025-11-28T14:51:15.841022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788928512874561:2049] [node 2] ICP30 unregister session Session# [6:7577788937102809247:2049] VirtualId# [6:7577788932807841944:0]
2025-11-28T14:51:15.841068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788937102809247:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.841154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:15.841193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:15.841271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:15.841437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788934906050877:2049] [node 2] ICS01 socket: 106 reason# EndOfStream
2025-11-28T14:51:15.841469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788926316116185:2049] [node 2] ICP30 unregister session Session# [4:7577788934906050877:2049] VirtualId# [4:7577788934906050873:0]
2025-11-28T14:51:15.841487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788934906050877:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.841530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:15.841564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:15.882300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:15.882519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788934906050870:2050] [node 3] ICS01 socket: 104 reason# EndOfStream
2025-11-28T14:51:15.882567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788926316116186:2050] [node 3] ICP30 unregister session Session# [4:7577788934906050870:2050] VirtualId# [4:7577788934906050868:0]
2025-11-28T14:51:15.882596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788934906050870:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.882692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:15.882741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:15.887706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:15.893191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:15.896508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788931811600585:2050] [node 3] ICS01 socket: 66 reason# EndOfStream
2025-11-28T14:51:15.896836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788932807841939:2050] [node 3] ICS01 socket: 79 reason# EndOfStream
2025-11-28T14:51:15.896892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788928512874562:2050] [node 3] ICP30 unregister session Session# [6:7577788932807841939:2050] VirtualId# [6:7577788932807841937:0]
2025-11-28T14:51:15.896922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788932807841939:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.896961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:15.897063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:15.904529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788927516633200:2050] [node 3] ICP30 unregister session Session# [5:7577788931811600585:2050] VirtualId# [5:7577788931811600575:0]
2025-11-28T14:51:15.904601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788931811600585:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.904649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:15.904753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:15.912535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:15.912791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788937102809251:2074] [node 4] ICS01 socket: 111 reason# EndOfStream
2025-11-28T14:51:15.912826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788928512874634:2074] [node 4] ICP30 unregister session Session# [6:7577788937102809251:2074] VirtualId# [6:7577788937102809249:0]
2025-11-28T14:51:15.912857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788937102809251:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.912945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:15.912984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:15.913319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:15.913507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788931811600578:2061] [node 4] ICS01 socket: 51 reason# EndOfStream
2025-11-28T14:51:15.913537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788927516633222:2061] [node 4] ICP30 unregister session Session# [5:7577788931811600578:2061] VirtualId# [5:7577788931811600576:0]
2025-11-28T14:51:15.913557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788931811600578:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.913600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:15.913631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:15.940538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:15.940772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788932807841946:2060] [node 5] ICS01 socket: 80 reason# EndOfStream
2025-11-28T14:51:15.940816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788928512874582:2060] [node 5] ICP30 unregister session Session# [6:7577788932807841946:2060] VirtualId# [6:7577788932807841941:0]
2025-11-28T14:51:15.940852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788932807841946:2060] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:15.940956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:15.941003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:16.400732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788941611426944:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:16.400767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788941611426945:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:16.485417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788941767440570:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:16.485449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788941767440571:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:16.532969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788941621907085:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:16.533016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788941621907086:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:16.566432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788942451438549:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:16.566473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788942451438550:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:16.566497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788942451438551:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:16.644803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788942670984043:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:16.644853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788942670984044:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:16.644891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788942670984045:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:16.724774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788939275870778:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:16.724824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788939275870779:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:16.724852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788939275870780:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:17.013618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788945916874405:2069] [node 4] ICP01 ready to work
2025-11-28T14:51:17.142108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788946062407912:2088] [node 6] ICP01 ready to work
2025-11-28T14:51:17.073380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788943570838087:2059] [node 4] ICP01 ready to work
2025-11-28T14:51:17.153361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788946746405872:2063] [node 6] ICP01 ready to work
2025-11-28T14:51:17.139876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788945916874414:2072] [node 5] ICP01 ready to work
2025-11-28T14:51:17.180477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788945906394293:2088] [node 6] ICP01 ready to work
2025-11-28T14:51:17.203143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788945906394297:2092] [node 4] ICP01 ready to work
2025-11-28T14:51:17.315865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788946062407929:2085] [node 4] ICP01 ready to work
2025-11-28T14:51:17.334231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788939275870780:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(101105655ns) {Configure:104(101100711ns) {ProcessConfigured:135(61401ns) {StartInitialHandshake:157(59098ns) {DropHandshakes:470(3306ns) {DropIncomingHandshake:442(814ns) DropOutgoingHandshake:457(492ns)} GenerateSessionVirtualId:988(1377ns) PrepareNewSessionHandshake:477(2101ns) {SwitchToState:179(849ns)}}}}}
2025-11-28T14:51:17.384684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788946746405910:2060] [node 5] ICP01 ready to work
2025-11-28T14:51:17.322244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788942670984045:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(194491837ns) {Configure:104(194488064ns) {ProcessConfigured:135(86532ns) {StartInitialHandshake:157(83160ns) {DropHandshakes:470(3022ns) {DropIncomingHandshake:442(799ns) DropOutgoingHandshake:457(526ns)} GenerateSessionVirtualId:988(816ns) PrepareNewSessionHandshake:477(1976ns) {SwitchToState:179(642ns)}}}}}
2025-11-28T14:51:17.401827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788942670984043:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(287544016ns) {Configure:104(287539406ns) {ProcessConfigured:135(80432ns) {StartInitialHandshake:157(78187ns) {DropHandshakes:470(3087ns) {DropIncomingHandshake:442(670ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(1325ns) PrepareNewSessionHandshake:477(2252ns) {SwitchToState:179(581ns)}}}}}
2025-11-28T14:51:17.305116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788945906394305:2091] [node 5] ICP01 ready to work
2025-11-28T14:51:17.408781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788943570838142:2063] [node 5] ICP01 ready to work
2025-11-28T14:51:17.376430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788946062407930:2086] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.417947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788942670984044:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(301089578ns) {Configure:104(301085364ns) {ProcessConfigured:135(89400ns) {StartInitialHandshake:157(87036ns) {DropHandshakes:470(3458ns) {DropIncomingHandshake:442(962ns) DropOutgoingHandshake:457(469ns)} GenerateSessionVirtualId:988(1716ns) PrepareNewSessionHandshake:477(2298ns) {SwitchToState:179(647ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.416597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788942451438550:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(155863808ns) {Configure:104(155860001ns) {ProcessConfigured:135(36581497ns) {StartInitialHandshake:157(36577798ns) {DropHandshakes:470(16303599ns) {DropIncomingHandshake:442(842ns) DropOutgoingHandshake:457(3462ns)} GenerateSessionVirtualId:988(2687ns) PrepareNewSessionHandshake:477(5181ns) {SwitchToState:179(2841ns)}}}}}
2025-11-28T14:51:17.457518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788942451438551:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(72536998ns) {Configure:104(72531534ns) {ProcessConfigured:135(83500ns) {StartInitialHandshake:157(81029ns) {DropHandshakes:470(3598ns) {DropIncomingHandshake:442(1254ns) DropOutgoingHandshake:457(620ns)} GenerateSessionVirtualId:988(1489ns) PrepareNewSessionHandshake:477(2154ns) {SwitchToState:179(999ns)}}}}}
2025-11-28T14:51:17.457683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788946746405871:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41261
2025-11-28T14:51:17.468182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788946746405865:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41261
2025-11-28T14:51:17.471471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788942451438549:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(391614510ns) {Configure:104(391600964ns) {ProcessConfigured:135(112831ns) {StartInitialHandshake:157(109652ns) {DropHandshakes:470(3085ns) {DropIncomingHandshake:442(902ns) DropOutgoingHandshake:457(502ns)} GenerateSessionVirtualId:988(1605ns) PrepareNewSessionHandshake:477(2014ns) {SwitchToState:179(963ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.521020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788946965951410:2084] [node 4] ICP01 ready to work
2025-11-28T14:51:17.486341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788943570838142:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(33368117ns) {Configure:104(33362112ns) {ProcessConfigured:135(87081ns) {StartInitialHandshake:157(84703ns) {DropHandshakes:470(3321ns) {DropIncomingHandshake:442(680ns) DropOutgoingHandshake:457(515ns)} GenerateSessionVirtualId:988(1214ns) PrepareNewSessionHandshake:477(2902ns) {SwitchToState:179(1220ns)}}}}}
2025-11-28T14:51:17.524138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788946965951411:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32588
2025-11-28T14:51:17.576574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788946965951415:2088] [node 6] ICP01 ready to work
2025-11-28T14:51:17.549217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788943570838150:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28527
2025-11-28T14:51:17.432958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788941767440571:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(290510756ns) {Configure:104(290505302ns) {ProcessConfigured:135(104187ns) {StartInitialHandshake:157(100378ns) {DropHandshakes:470(3386ns) {DropIncomingHandshake:442(909ns) DropOutgoingHandshake:457(535ns)} GenerateSessionVirtualId:988(1257ns) PrepareNewSessionHandshake:477(2288ns) {SwitchToState:179(955ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.613464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788941611426945:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(394225314ns) {Configure:104(394219100ns) {ProcessConfigured:135(77739ns) {StartInitialHandshake:157(75260ns) {DropHandshakes:470(3529ns) {DropIncomingHandshake:442(1236ns) DropOutgoingHandshake:457(456ns)} GenerateSessionVirtualId:988(1240ns) PrepareNewSessionHandshake:477(1991ns) {SwitchToState:179(734ns)}}}}}
2025-11-28T14:51:17.616242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788945906394303:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57120
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.617404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788945906394297:2092] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(256410615ns) {Configure:104(256404710ns) {ProcessConfigured:135(34545ns) {StartInitialHandshake:157(33587ns) {DropHandshakes:470(1536ns) {DropIncomingHandshake:442(232ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(975ns) {SwitchToState:179(547ns)}}}}}
2025-11-28T14:51:17.617718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788945906394313:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57120
2025-11-28T14:51:17.465083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788946062407930:2086] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31845794ns) {Configure:104(31835917ns) {ProcessConfigured:135(110267ns) {StartInitialHandshake:157(104994ns) {DropHandshakes:470(2738ns) {DropIncomingHandshake:442(511ns) DropOutgoingHandshake:457(492ns)} GenerateSessionVirtualId:988(1377ns) PrepareNewSessionHandshake:477(1668ns) {SwitchToState:179(651ns)}}}}}
2025-11-28T14:51:17.467539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788946062407938:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36673
2025-11-28T14:51:17.467604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788946062407939:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36673
2025-11-28T14:51:17.467664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788946062407940:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36673
2025-11-28T14:51:17.575091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788946062407929:2085] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(149259520ns) {Configure:104(149251892ns) {ProcessConfigured:135(78858ns) {StartInitialHandshake:157(75986ns) {DropHandshakes:470(6055ns) {DropIncomingHandshake:442(1080ns) DropOutgoingHandshake:457(460ns)} GenerateSessionVirtualId:988(1523ns) PrepareNewSessionHandshake:477(1805ns) {SwitchToState:179(666ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.621949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788946965951416:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32588
2025-11-28T14:51:17.549319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788943570838151:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28527
2025-11-28T14:51:17.597672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788939275870778:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(245023714ns) {Configure:104(245012811ns) {ProcessConfigured:135(88633ns) {StartInitialHandshake:157(85601ns) {DropHandshakes:470(3894ns) {DropIncomingHandshake:442(1030ns) DropOutgoingHandshake:457(651ns)} GenerateSessionVirtualId:988(1057ns) PrepareNewSessionHandshake:477(2937ns) {SwitchToState:179(1078ns)}}}}}
2025-11-28T14:51:17.656661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788941767440570:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(260518765ns) {Configure:104(260514846ns) {ProcessConfigured:135(47960069ns) {StartInitialHandshake:157(45243976ns) {DropHandshakes:470(3834ns) {DropIncomingHandshake:442(563ns) DropOutgoingHandshake:457(607ns)} GenerateSessionVirtualId:988(1741ns) PrepareNewSessionHandshake:477(2966ns) {SwitchToState:179(1573ns)}}}}}
2025-11-28T14:51:17.662950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788946746405871:2073] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.654306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788943570838089:2060] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.628708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788945916874414:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(482608496ns) {Configure:104(482602270ns) {ProcessConfigured:135(35678818ns) {StartInitialHandshake:157(35675396ns) {DropHandshakes:470(3189ns) {DropIncomingHandshake:442(829ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(1051ns) PrepareNewSessionHandshake:477(3549ns) {SwitchToState:179(1629ns)}}}}}
2025-11-28T14:51:17.652227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788945916874452:2077] [node 6] ICP01 ready to work
2025-11-28T14:51:17.680799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788941621907086:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(600040179ns) {Configure:104(565083718ns) {ProcessConfigured:135(17619300ns) {StartInitialHandshake:157(17615909ns) {DropHandshakes:470(3184ns) {DropIncomingHandshake:442(839ns) DropOutgoingHandshake:457(594ns)} GenerateSessionVirtualId:988(801ns) PrepareNewSessionHandshake:477(4600ns) {SwitchToState:179(2515ns)}}}}}
2025-11-28T14:51:17.683648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788945916874455:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23639
2025-11-28T14:51:17.683849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788945916874456:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23639
2025-11-28T14:51:17.689247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788945916874457:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23639
2025-11-28T14:51:17.689549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788945916874458:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23639
2025-11-28T14:51:17.689705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788945916874459:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23639
2025-11-28T14:51:17.688904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788945906394293:2088] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(194189377ns) {Configure:104(194184099ns) {ProcessConfigured:135(3940812ns) {StartInitialHandshake:157(3938412ns) {DropHandshakes:470(2090ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(373ns)} GenerateSessionVirtualId:988(698ns) PrepareNewSessionHandshake:477(3005ns) {SwitchToState:179(1540ns)}}}}}
2025-11-28T14:51:17.792508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788946746405909:2108] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:17.837678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788946746405874:2074] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:17.840552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788943570838150:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:17.840749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788943570838087:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:17.854560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788946746405874:2074] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:17.855141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788946746405874:2074] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.865831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788943570838155:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28527
2025-11-28T14:51:17.855414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788946746405872:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:17.866044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:17.875713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788946746405872:2063] [node 6] ICP22 created new session: [4:7577788946746405927:2063]
2025-11-28T14:51:17.879182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788946746405927:2063] [node 6] ICS09 handshake done sender: [4:7577788946746405874:2074] self: [4:7577788946746405873:0] peer: [6:7577788943570838154:0] socket: 61 qp: -1
2025-11-28T14:51:17.879239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788946746405927:2063] [node 6] ICS10 traffic start
2025-11-28T14:51:17.879278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:17.879861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577788946746405872:2063] [node 6] ICP35 event processing took too much time PendingConnection:279(24248430ns) {HandleHandshakeStatus:329(24244786ns) {DropIncomingHandshake:442(12784ns) DropOutgoingHandshake:457(1169ns) SwitchToState:179(1104ns) ProcessPendingSessionEvents:495(136771ns) {ForwardSessionEventToSession:612(56895ns) ForwardSessionEventToSession:612(15720ns) ForwardSessionEventToSession:612(2604ns) ForwardSessionEventToSession:612(7526ns) ForwardSessionEventToSession:612(1789ns) ForwardSessionEventToSession:612(1732ns) ForwardSessionEventToSession:612(7904ns) ForwardSessionEventToSession:612(2617ns) ForwardSessionEventToSession:612(1949ns) ForwardSessionEventToSession:612(20134ns)}}}
2025-11-28T14:51:17.880439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788943570838150:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:17.880616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788943570838087:2059] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.880815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:17.880855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788943570838087:2059] [node 4] ICP22 created new session: [6:7577788943570838156:2059]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.897590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788943570838156:2059] [node 4] ICS09 handshake done sender: [6:7577788943570838150:2114] self: [6:7577788943570838154:0] peer: [4:7577788946746405873:0] socket: 73 qp: -1
2025-11-28T14:51:17.897640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788943570838156:2059] [node 4] ICS10 traffic start
2025-11-28T14:51:17.897715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:17.898235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577788943570838087:2059] [node 4] ICP35 event processing took too much time PendingConnection:279(17380210ns) {HandleHandshakeStatus:329(17376404ns) {DropIncomingHandshake:442(974ns) DropOutgoingHandshake:457(4049ns) SwitchToState:179(625ns) ProcessPendingSessionEvents:495(96002ns) {ForwardSessionEventToSession:612(49127ns) ForwardSessionEventToSession:612(3413ns) ForwardSessionEventToSession:612(5058ns) ForwardSessionEventToSession:612(8080ns) ForwardSessionEventToSession:612(4758ns) ForwardSessionEventToSession:612(2236ns) ForwardSessionEventToSession:612(2046ns) ForwardSessionEventToSession:612(7606ns) ForwardSessionEventToSession:612(2159ns) ForwardSessionEventToSession:612(1997ns)}}}
2025-11-28T14:51:17.888765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788946965951406:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:17.889596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788946965951419:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32588
2025-11-28T14:51:17.897373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788946965951420:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32588
2025-11-28T14:51:17.952384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788943570838138:2107] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:17.965435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788946965951421:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32588
2025-11-28T14:51:17.999063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788945916874407:2070] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.006066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788945916874407:2070] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.006472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788945916874407:2070] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.005642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788946746405865:2067] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.005780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788942451438551:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:18.033326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788951041373227:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41261
2025-11-28T14:51:18.060661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788946746405865:2067] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:18.060960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788945916874405:2069] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:18.061009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:18.061056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788945916874405:2069] [node 4] ICP22 created new session: [3:7577788950211841762:2069]
2025-11-28T14:51:18.061097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788950211841762:2069] [node 4] ICS09 handshake done sender: [3:7577788945916874407:2070] self: [3:7577788945916874406:0] peer: [4:7577788951041373226:0] socket: 49 qp: -1
2025-11-28T14:51:18.061128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788950211841762:2069] [node 4] ICS10 traffic start
2025-11-28T14:51:18.061161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:18.063231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788942451438551:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:18.063279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:18.063325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788942451438551:2050] [node 3] ICP22 created new session: [4:7577788951041373228:2050]
2025-11-28T14:51:18.063370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788951041373228:2050] [node 3] ICS09 handshake done sender: [4:7577788946746405865:2067] self: [4:7577788951041373226:0] peer: [3:7577788945916874406:0] socket: 50 qp: -1
2025-11-28T14:51:18.063398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788951041373228:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:18.063440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:18.071340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788946965951403:2109] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.077271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788945906394313:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.140484Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788941621907105:2067] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.173465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788945906394303:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.191992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788946062407937:2112] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.228900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788945906394310:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.229821Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577788945906394297:2092] [node 4] ICP35 event processing took too much time PendingConnection:279(18191272ns) {HandleCleanupEventQueue:832(18189401ns) {CleanupEventQueue:841(18177772ns) {DropSessionEvent:512(18169970ns)} ScheduleCleanupEventQueue:822(9492ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.230722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788946062407940:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.230796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788946062407939:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.230899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788946062407938:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.231129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788946062407948:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36673
2025-11-28T14:51:18.231917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788943570838151:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.232027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788939275870779:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:18.233459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788946965951416:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.233592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788942670984043:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:18.231630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788950357375245:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36673
2025-11-28T14:51:18.248497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788946062407937:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.248974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788951260918719:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32588
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.234158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788945906394310:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.255079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788946062407937:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:18.252475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788946965951416:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:18.263544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788942670984043:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:18.245062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788945906394310:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:18.257152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788947865805456:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28527
2025-11-28T14:51:18.264268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788943570838151:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:18.264418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788939275870779:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:18.264456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:18.264487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788939275870779:2049] [node 2] ICP22 created new session: [6:7577788947865805457:2049]
2025-11-28T14:51:18.264518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788947865805457:2049] [node 2] ICS09 handshake done sender: [6:7577788943570838151:2115] self: [6:7577788947865805455:0] peer: [2:7577788946062407936:0] socket: 74 qp: -1
2025-11-28T14:51:18.264550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788947865805457:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:18.264573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:18.263606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:18.263649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788942670984043:2048] [node 1] ICP22 created new session: [5:7577788951260918720:2048]
2025-11-28T14:51:18.263692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788951260918720:2048] [node 1] ICS09 handshake done sender: [5:7577788946965951416:2115] self: [5:7577788951260918718:0] peer: [1:7577788945906394309:0] socket: 80 qp: -1
2025-11-28T14:51:18.263740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788951260918720:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:18.263774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:18.270089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788945906394312:2113] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.270555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788945906394305:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:18.270609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:18.270653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788945906394305:2091] [node 5] ICP22 created new session: [1:7577788950201361620:2091]
2025-11-28T14:51:18.270695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788950201361620:2091] [node 5] ICS09 handshake done sender: [1:7577788945906394310:2112] self: [1:7577788945906394309:0] peer: [5:7577788951260918718:0] socket: 78 qp: -1
2025-11-28T14:51:18.270723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788950201361620:2091] [node 5] ICS10 traffic start
2025-11-28T14:51:18.270751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.289248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788950201361622:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57120
2025-11-28T14:51:18.292568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788946062407912:2088] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.292651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:18.297615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788951041373230:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41261
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.297790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788951041373231:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41261
2025-11-28T14:51:18.292711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788946062407912:2088] [node 6] ICP22 created new session: [2:7577788950357375250:2088]
2025-11-28T14:51:18.292755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788950357375250:2088] [node 6] ICS09 handshake done sender: [2:7577788946062407937:2112] self: [2:7577788946062407936:0] peer: [6:7577788947865805455:0] socket: 69 qp: -1
2025-11-28T14:51:18.292787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788950357375250:2088] [node 6] ICS10 traffic start
2025-11-28T14:51:18.292825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:18.293541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788946062407948:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.327966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577788941767440570:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:18.332735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788951041373232:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41261
2025-11-28T14:51:18.336551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788945906394312:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.368463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788945916874455:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.338114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788946965951399:2106] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.345032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788945906394312:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:18.388013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788946746405917:2114] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.354805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788943570838136:2106] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.388623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788941611426944:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:18.388698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:18.388743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788941611426944:2048] [node 2] ICP22 created new session: [1:7577788950201361625:2048]
2025-11-28T14:51:18.388787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788950201361625:2048] [node 2] ICS09 handshake done sender: [1:7577788945906394312:2113] self: [1:7577788945906394311:0] peer: [2:7577788950357375253:0] socket: 79 qp: -1
2025-11-28T14:51:18.388822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788950201361625:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:18.388854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.411271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788945906394308:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.408787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788947865805460:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28527
2025-11-28T14:51:18.464488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788945916874457:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.464633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788945916874456:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.464719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788945916874459:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.464856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577788945916874405:2069] [node 4] ICP18 handshake (actor [3:7577788945916874457:2115]) is too old
2025-11-28T14:51:18.464959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788941621907085:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.501173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577788942670984045:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(20366303ns) {HandleCleanupEventQueue:832(20364292ns) {CleanupEventQueue:841(20362138ns) {DropSessionEvent:512(10678ns) DropSessionEvent:512(20311754ns) DropSessionEvent:512(4240ns) DropSessionEvent:512(3911ns) DropSessionEvent:512(4286ns) DropSessionEvent:512(3588ns) DropSessionEvent:512(4872ns) DropSessionEvent:512(4842ns) DropSessionEvent:512(3728ns)} ScheduleCleanupEventQueue:822(455ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.500872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788950201361628:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57120
2025-11-28T14:51:18.493805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788946062407933:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.528772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788945906394308:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.563222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788945906394308:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.567968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788945916874458:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.581126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788941621907086:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:18.581244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788950211841768:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23639
2025-11-28T14:51:18.585548Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788945916874454:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.586211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788945916874459:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:18.572730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577788943570838142:2063] [node 5] ICP35 event processing took too much time PendingConnection:279(32620129ns) {HandleCleanupEventQueue:832(32618197ns) {CleanupEventQueue:841(32617186ns) {DropSessionEvent:512(11542ns) DropSessionEvent:512(3738ns) DropSessionEvent:512(3224ns) DropSessionEvent:512(2947ns) DropSessionEvent:512(25161790ns) DropSessionEvent:512(11189ns) DropSessionEvent:512(3562ns) DropSessionEvent:512(7396356ns) DropSessionEvent:512(9101ns)} ScheduleCleanupEventQueue:822(82ns)}}
2025-11-28T14:51:18.563004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577788942451438549:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23778580ns) {HandleCleanupEventQueue:832(23776857ns) {CleanupEventQueue:841(23774797ns) {DropSessionEvent:512(9930ns) DropSessionEvent:512(15961912ns) DropSessionEvent:512(4479ns) DropSessionEvent:512(4078ns) DropSessionEvent:512(3680ns) DropSessionEvent:512(2662ns) DropSessionEvent:512(7769233ns) DropSessionEvent:512(3410ns) DropSessionEvent:512(3330ns)} ScheduleCleanupEventQueue:822(422ns)}}
2025-11-28T14:51:18.583427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788950357375254:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36673
2025-11-28T14:51:18.608592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788941621907085:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:18.608691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:18.608742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788941621907085:2048] [node 1] ICP22 created new session: [3:7577788950211841770:2048]
2025-11-28T14:51:18.608788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788950211841770:2048] [node 1] ICS09 handshake done sender: [3:7577788945916874459:2117] self: [3:7577788950211841767:0] peer: [1:7577788945906394307:0] socket: 86 qp: -1
2025-11-28T14:51:18.608825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788950211841770:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:18.608880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:18.621032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788945916874454:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.603533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577788946965951410:2084] [node 4] ICP35 event processing took too much time PendingConnection:279(16657782ns) {HandleCleanupEventQueue:832(16654170ns) {CleanupEventQueue:841(16651848ns) {DropSessionEvent:512(11379ns) DropSessionEvent:512(4547ns) DropSessionEvent:512(3971ns) DropSessionEvent:512(3408ns) DropSessionEvent:512(3502ns) DropSessionEvent:512(7204973ns) DropSessionEvent:512(4041ns) DropSessionEvent:512(30184ns) DropSessionEvent:512(3443ns) DropSessionEvent:512(4422ns)} ScheduleCleanupEventQueue:822(488ns)}}
2025-11-28T14:51:18.603834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788946965951419:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.603987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788942670984045:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:18.609818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577788942670984044:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19141234ns) {HandleCleanupEventQueue:832(19139145ns) {CleanupEventQueue:841(19138144ns) {DropSessionEvent:512(12806ns) DropSessionEvent:512(4131ns) DropSessionEvent:512(3887ns) DropSessionEvent:512(3197ns) DropSessionEvent:512(3379ns) DropSessionEvent:512(19106329ns)} ScheduleCleanupEventQueue:822(34ns)}}
2025-11-28T14:51:18.620887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788941611426945:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:18.636992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:18.640782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788941611426945:2049] [node 3] ICP22 created new session: [1:7577788950201361630:2049]
2025-11-28T14:51:18.642540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788950201361630:2049] [node 3] ICS09 handshake done sender: [1:7577788945906394308:2111] self: [1:7577788945906394307:0] peer: [3:7577788950211841767:0] socket: 77 qp: -1
2025-11-28T14:51:18.642581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788950201361630:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:18.642626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:18.642811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577788941611426945:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(21896735ns) {HandleHandshakeStatus:329(21893045ns) {DropIncomingHandshake:442(20287ns) DropOutgoingHandshake:457(522ns) SwitchToState:179(791ns) ProcessPendingSessionEvents:495(2011ns)}}
2025-11-28T14:51:18.642391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788945916874454:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:18.643666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788945916874414:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:18.643725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:18.643764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788945916874414:2072] [node 5] ICP22 created new session: [3:7577788950211841774:2072]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.641513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788946062407933:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.641639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788946062407948:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:18.643804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788950211841774:2072] [node 5] ICS09 handshake done sender: [3:7577788945916874454:2112] self: [3:7577788945916874453:0] peer: [5:7577788951260918723:0] socket: 88 qp: -1
2025-11-28T14:51:18.665427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788942670984045:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22364) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788946965951419:2117] held: no
2025-11-28T14:51:18.643826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788950211841774:2072] [node 5] ICS10 traffic start
2025-11-28T14:51:18.668650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577788941767440570:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.696924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788946746405919:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.668717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:18.668759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788941767440570:2048] [node 1] ICP22 created new session: [2:7577788950357375256:2048]
2025-11-28T14:51:18.668798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788950357375256:2048] [node 1] ICS09 handshake done sender: [2:7577788946062407948:2118] self: [2:7577788950357375253:0] peer: [1:7577788945906394311:0] socket: 87 qp: -1
2025-11-28T14:51:18.668825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788950357375256:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:18.668849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:18.643857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:18.695423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788947865805461:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28527
2025-11-28T14:51:18.724516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788946965951420:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.748832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577788941767440570:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(56001893ns) {HandleHandshakeStatus:329(55997674ns) {DropIncomingHandshake:442(1019ns) DropOutgoingHandshake:457(4374ns) SwitchToState:179(11358ns) ProcessPendingSessionEvents:495(55652205ns) {ForwardSessionEventToSession:612(48447010ns) ForwardSessionEventToSession:612(7195109ns)}}}
2025-11-28T14:51:18.721079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788950211841776:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23639
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.749889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788951260918724:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32588
2025-11-28T14:51:18.713397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788947865805462:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28527
2025-11-28T14:51:18.748924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788943570838149:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.750827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788946062407933:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:18.751327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788945916874458:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:18.751511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788941621907086:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:18.751563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:18.751600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788941621907086:2049] [node 2] ICP22 created new session: [3:7577788950211841777:2049]
2025-11-28T14:51:18.751637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788950211841777:2049] [node 2] ICS09 handshake done sender: [3:7577788945916874458:2116] self: [3:7577788950211841769:0] peer: [2:7577788946062407932:0] socket: 85 qp: -1
2025-11-28T14:51:18.751666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788950211841777:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:18.751690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:18.804438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788946746405919:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.796055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788946965951410:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:18.796208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788946965951421:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.796423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788946965951419:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:18.812700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788942670984045:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:18.812748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:18.812783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788942670984045:2050] [node 3] ICP22 created new session: [5:7577788951260918727:2050]
2025-11-28T14:51:18.812826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788951260918727:2050] [node 3] ICS09 handshake done sender: [5:7577788946965951419:2117] self: [5:7577788951260918723:0] peer: [3:7577788945916874453:0] socket: 53 qp: -1
2025-11-28T14:51:18.812855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788951260918727:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.809240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788946746405919:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:18.828002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788946746405910:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:18.808666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788941767440571:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:18.832526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:18.832590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788946746405910:2060] [node 5] ICP22 created new session: [4:7577788951041373235:2060]
2025-11-28T14:51:18.832633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788951041373235:2060] [node 5] ICS09 handshake done sender: [4:7577788946746405919:2115] self: [4:7577788946746405918:0] peer: [5:7577788951260918726:0] socket: 91 qp: -1
2025-11-28T14:51:18.832666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788951041373235:2060] [node 5] ICS10 traffic start
2025-11-28T14:51:18.832719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:18.812886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:18.820810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788951260918729:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32588
2025-11-28T14:51:18.808734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:18.813458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788941767440571:2049] [node 3] ICP22 created new session: [2:7577788950357375258:2049]
2025-11-28T14:51:18.813510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788950357375258:2049] [node 3] ICS09 handshake done sender: [2:7577788946062407933:2110] self: [2:7577788946062407932:0] peer: [3:7577788950211841769:0] socket: 67 qp: -1
2025-11-28T14:51:18.813541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788950357375258:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:18.813589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:18.835270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788946062407935:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.836435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788946965951411:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.836491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788946965951420:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:18.861598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788946062407935:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.862145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788946062407935:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:18.868558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788946062407930:2086] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:18.868677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:18.868730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788946062407930:2086] [node 5] ICP22 created new session: [2:7577788950357375261:2086]
2025-11-28T14:51:18.868787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788950357375261:2086] [node 5] ICS09 handshake done sender: [2:7577788946062407935:2111] self: [2:7577788946062407934:0] peer: [5:7577788951260918733:0] socket: 68 qp: -1
2025-11-28T14:51:18.868832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788950357375261:2086] [node 5] ICS10 traffic start
2025-11-28T14:51:18.868867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:18.860552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788946965951410:2084] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:18.880485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788945916874461:2118] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.860613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:18.860654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788946965951410:2084] [node 4] ICP22 created new session: [5:7577788951260918731:2084]
2025-11-28T14:51:18.860704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788951260918731:2084] [node 4] ICS09 handshake done sender: [5:7577788946965951420:2118] self: [5:7577788951260918726:0] peer: [4:7577788946746405918:0] socket: 92 qp: -1
2025-11-28T14:51:18.860736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788951260918731:2084] [node 4] ICS10 traffic start
2025-11-28T14:51:18.860768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:18.861155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788942670984044:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:18.869717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788951260918734:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32588
2025-11-28T14:51:18.889287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788942670984044:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16783) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788946965951411:2113] held: no
2025-11-28T14:51:18.889349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788946965951411:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:18.889498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788942670984044:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:18.889539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:18.889574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788942670984044:2049] [node 2] ICP22 created new session: [5:7577788951260918735:2049]
2025-11-28T14:51:18.889608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788951260918735:2049] [node 2] ICS09 handshake done sender: [5:7577788946965951411:2113] self: [5:7577788951260918733:0] peer: [2:7577788946062407934:0] socket: 71 qp: -1
2025-11-28T14:51:18.889633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788951260918735:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:18.889664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:18.894014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788950357375245:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:18.900502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577788941767440571:2049] [node 3] ICP18 handshake (actor [2:7577788950357375245:2119]) is too old
2025-11-28T14:51:18.930688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788943570838153:2116] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:18.982810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788946965951413:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:19.016456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788951041373230:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:19.016634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577788946746405910:2060] [node 5] ICP18 handshake (actor [4:7577788951041373230:2118]) is too old
2025-11-28T14:51:19.032827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788951041373231:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:19.033022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788942451438549:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:19.055001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788946062407946:2116] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:19.055778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788951041373232:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:19.055908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788942451438550:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:19.056317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788946062407946:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:19.072882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788946062407946:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:19.095762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788946062407929:2085] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:19.095866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:19.095927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788946062407929:2085] [node 4] ICP22 created new session: [2:7577788954652342560:2085]
2025-11-28T14:51:19.095975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788954652342560:2085] [node 4] ICS09 handshake done sender: [2:7577788946062407946:2116] self: [2:7577788946062407945:0] peer: [4:7577788955336340534:0] socket: 64 qp: -1
2025-11-28T14:51:19.096014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788954652342560:2085] [node 4] ICS10 traffic start
2025-11-28T14:51:19.096048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:19.088999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788955336340535:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41261
2025-11-28T14:51:19.095772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788951041373232:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:19.117276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788942451438550:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:19.117347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:19.117391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788942451438550:2049] [node 2] ICP22 created new session: [4:7577788955336340536:2049]
2025-11-28T14:51:19.117446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788955336340536:2049] [node 2] ICS09 handshake done sender: [4:7577788951041373232:2120] self: [4:7577788955336340534:0] peer: [2:7577788946062407945:0] socket: 105 qp: -1
2025-11-28T14:51:19.117477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788955336340536:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:19.117515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:19.117711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577788942451438550:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:19.118021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577788946062407929:2085] [node 4] ICP35 event processing took too much time PendingConnection:279(22210755ns) {HandleHandshakeStatus:329(22207786ns) {DropIncomingHandshake:442(25109ns) DropOutgoingHandshake:457(631ns) SwitchToState:179(1013ns) ProcessPendingSessionEvents:495(1072ns)}}
2025-11-28T14:51:19.127529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788947865805460:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:19.188776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:19.189046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577788950357375256:2048] [node 1] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-28T14:51:19.189069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577788950357375256:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.189141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:19.189184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577788950357375256:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:19.127677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788939275870778:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:19.194193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788941767440570:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8415) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:19.194229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577788950357375256:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:19.195116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788941767440570:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8415) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:19.195203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577788950357375256:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:19.195248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577788941767440570:2048] [node 1] ICP30 unregister session Session# [2:7577788950357375256:2048] VirtualId# [2:7577788950357375253:0]
2025-11-28T14:51:19.195297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:19.195333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577788941767440570:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8415) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:19.195343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8415) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:19.192420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:19.192613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788950211841770:2048] [node 1] ICS01 socket: 86 reason# EndOfStream
2025-11-28T14:51:19.192656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788941621907085:2048] [node 1] ICP30 unregister session Session# [3:7577788950211841770:2048] VirtualId# [3:7577788950211841767:0]
2025-11-28T14:51:19.192698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788950211841770:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.192765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:19.192801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:19.201771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:19.202010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788951260918720:2048] [node 1] ICS01 socket: 80 reason# EndOfStream
2025-11-28T14:51:19.202049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788942670984043:2048] [node 1] ICP30 unregister session Session# [5:7577788951260918720:2048] VirtualId# [5:7577788951260918718:0]
2025-11-28T14:51:19.202086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788951260918720:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.202163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:19.202212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:19.204520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577788941767440570:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:19.208873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788939275870778:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8415) Socket error# Connection reset by peer state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788947865805460:2119] held: no
2025-11-28T14:51:19.242911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788947865805462:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:19.243055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788943570838142:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:19.241988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788946965951418:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:19.244937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788946965951418:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:19.225504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788946746405924:2116] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:19.246707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788942451438549:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8415) Socket error# Broken pipe state# SendExRequestData processed# 0 remain# 183 incoming: [4:7577788951041373231:2119] held: no
2025-11-28T14:51:19.246115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:19.246306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788950211841777:2049] [node 2] ICS01 socket: 85 reason# EndOfStream
2025-11-28T14:51:19.246339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788941621907086:2049] [node 2] ICP30 unregister session Session# [3:7577788950211841777:2049] VirtualId# [3:7577788950211841769:0]
2025-11-28T14:51:19.246367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788950211841777:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.246431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:19.246477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:19.248521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788946965951418:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:19.257263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788952160772761:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28527
2025-11-28T14:51:19.257790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788947865805462:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:19.258063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788943570838142:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:19.258113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:19.258149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788943570838142:2063] [node 5] ICP22 created new session: [6:7577788952160772762:2063]
2025-11-28T14:51:19.258188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788952160772762:2063] [node 5] ICS09 handshake done sender: [6:7577788947865805462:2121] self: [6:7577788952160772760:0] peer: [5:7577788946965951417:0] socket: 115 qp: -1
2025-11-28T14:51:19.258221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788952160772762:2063] [node 5] ICS10 traffic start
2025-11-28T14:51:19.258253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:19.263150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577788946965951415:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:19.263233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:19.268993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788946965951415:2088] [node 6] ICP22 created new session: [5:7577788955555886034:2088]
2025-11-28T14:51:19.269057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788955555886034:2088] [node 6] ICS09 handshake done sender: [5:7577788946965951418:2116] self: [5:7577788946965951417:0] peer: [6:7577788952160772760:0] socket: 54 qp: -1
2025-11-28T14:51:19.269095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788955555886034:2088] [node 6] ICS10 traffic start
2025-11-28T14:51:19.269152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:19.269403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577788946965951415:2088] [node 6] ICP35 event processing took too much time PendingConnection:279(20585799ns) {HandleHandshakeStatus:329(20582604ns) {DropIncomingHandshake:442(25792ns) DropOutgoingHandshake:457(710ns) SwitchToState:179(772ns) ProcessPendingSessionEvents:495(1101ns)}}
2025-11-28T14:51:19.281482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788947865805461:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:19.281613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788939275870780:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:19.282464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788952160772765:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28527
2025-11-28T14:51:19.283035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788947865805461:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:19.283202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788939275870780:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:19.283253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:19.283286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788939275870780:2050] [node 3] ICP22 created new session: [6:7577788952160772766:2050]
2025-11-28T14:51:19.283330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788952160772766:2050] [node 3] ICS09 handshake done sender: [6:7577788947865805461:2120] self: [6:7577788952160772764:0] peer: [3:7577788945916874462:0] socket: 114 qp: -1
2025-11-28T14:51:19.283357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788952160772766:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:19.283388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:19.280464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788945916874463:2119] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:19.281916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788945916874463:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:19.282659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788945916874463:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:19.284225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788945916874452:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:19.284283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:19.284317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788945916874452:2077] [node 6] ICP22 created new session: [3:7577788954506809076:2077]
2025-11-28T14:51:19.284347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788954506809076:2077] [node 6] ICS09 handshake done sender: [3:7577788945916874463:2119] self: [3:7577788945916874462:0] peer: [6:7577788952160772764:0] socket: 112 qp: -1
2025-11-28T14:51:19.284383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788954506809076:2077] [node 6] ICS10 traffic start
2025-11-28T14:51:19.284409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:19.292511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:19.292740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788951260918735:2049] [node 2] ICS01 socket: 71 reason# EndOfStream
2025-11-28T14:51:19.292776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788942670984044:2049] [node 2] ICP30 unregister session Session# [5:7577788951260918735:2049] VirtualId# [5:7577788951260918733:0]
2025-11-28T14:51:19.292802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788951260918735:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.292879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:19.292920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:19.292519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:19.292742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788955336340536:2049] [node 2] ICS01 socket: 105 reason# EndOfStream
2025-11-28T14:51:19.292779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788942451438550:2049] [node 2] ICP30 unregister session Session# [4:7577788955336340536:2049] VirtualId# [4:7577788955336340534:0]
2025-11-28T14:51:19.292804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788955336340536:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.292882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:19.292918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:19.293477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:19.293679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788947865805457:2049] [node 2] ICS01 socket: 74 reason# EndOfStream
2025-11-28T14:51:19.293719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788939275870779:2049] [node 2] ICP30 unregister session Session# [6:7577788947865805457:2049] VirtualId# [6:7577788947865805455:0]
2025-11-28T14:51:19.293740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788947865805457:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.293768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:19.293820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:19.316224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:19.316462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788951260918727:2050] [node 3] ICS01 socket: 53 reason# EndOfStream
2025-11-28T14:51:19.316497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788942670984045:2050] [node 3] ICP30 unregister session Session# [5:7577788951260918727:2050] VirtualId# [5:7577788951260918723:0]
2025-11-28T14:51:19.316530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788951260918727:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.316616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:19.316656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:19.316757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:19.316914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788952160772766:2050] [node 3] ICS01 socket: 114 reason# EndOfStream
2025-11-28T14:51:19.316941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788939275870780:2050] [node 3] ICP30 unregister session Session# [6:7577788952160772766:2050] VirtualId# [6:7577788952160772764:0]
2025-11-28T14:51:19.316959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788952160772766:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.317005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:19.317035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:19.332555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:19.332770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788951260918731:2084] [node 4] ICS01 socket: 92 reason# EndOfStream
2025-11-28T14:51:19.332828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788946965951410:2084] [node 4] ICP30 unregister session Session# [5:7577788951260918731:2084] VirtualId# [5:7577788951260918726:0]
2025-11-28T14:51:19.332860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788951260918731:2084] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.332949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:19.332988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:19.334833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:19.335016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788943570838156:2059] [node 4] ICS01 socket: 73 reason# EndOfStream
2025-11-28T14:51:19.335064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788943570838087:2059] [node 4] ICP30 unregister session Session# [6:7577788943570838156:2059] VirtualId# [6:7577788943570838154:0]
2025-11-28T14:51:19.335093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788943570838156:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.335190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:19.335235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:19.378851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:19.379074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788952160772762:2063] [node 5] ICS01 socket: 115 reason# EndOfStream
2025-11-28T14:51:19.379106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788943570838142:2063] [node 5] ICP30 unregister session Session# [6:7577788952160772762:2063] VirtualId# [6:7577788952160772760:0]
2025-11-28T14:51:19.379139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788952160772762:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:19.379241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:19.379280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:19.470567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788952233556086:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:19.470604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788952233556087:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:19.552813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788953884413283:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:19.552851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788953884413284:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:19.612819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788955282087242:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:19.612859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788955282087243:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:19.704048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788954461573644:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:19.711612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788954461573646:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:19.711811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788954461573645:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:19.804166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788953311695049:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:19.804231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788953311695050:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:19.804263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788953311695051:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:20.185577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788959577054554:2060] [node 5] ICP01 ready to work
2025-11-28T14:51:20.185982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788959577054555:2062] [node 6] ICP01 ready to work
2025-11-28T14:51:20.237549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788957606662361:2061] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.288242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788956528523442:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.321593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788957606662361:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(83733058ns) {Configure:104(83728865ns) {ProcessConfigured:135(102191ns) {StartInitialHandshake:157(99337ns) {DropHandshakes:470(3178ns) {DropIncomingHandshake:442(736ns) DropOutgoingHandshake:457(604ns)} GenerateSessionVirtualId:988(1172ns) PrepareNewSessionHandshake:477(2177ns) {SwitchToState:179(761ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.008784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788954214965672:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:20.356754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788956528523443:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
2025-11-28T14:51:20.301380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788955282087242:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(134731462ns) {Configure:104(134728293ns) {ProcessConfigured:135(20652794ns) {StartInitialHandshake:157(20649860ns) {DropHandshakes:470(3145ns) {DropIncomingHandshake:442(855ns) DropOutgoingHandshake:457(661ns)} GenerateSessionVirtualId:988(1089ns) PrepareNewSessionHandshake:477(3624ns) {SwitchToState:179(940ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.373131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788956528523459:2085] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.008822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788954214965673:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:20.549789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788954461573645:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(277139339ns) {Configure:104(277133148ns) {ProcessConfigured:135(65400ns) {StartInitialHandshake:157(62776ns) {DropHandshakes:470(3020ns) {DropIncomingHandshake:442(746ns) DropOutgoingHandshake:457(598ns)} GenerateSessionVirtualId:988(984ns) PrepareNewSessionHandshake:477(2000ns) {SwitchToState:179(724ns)}}}}}
2025-11-28T14:51:20.550189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788958756541006:2072] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.476681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788959577054597:2067] [node 4] ICP01 ready to work
2025-11-28T14:51:20.460416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788958179380639:2086] [node 4] ICP01 ready to work
2025-11-28T14:51:20.375654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788956528523460:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.008849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788954214965674:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:20.561590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788958179380643:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32106
2025-11-28T14:51:20.605201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788958756541012:2070] [node 5] ICP01 ready to work
2025-11-28T14:51:20.577459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788957606662414:2059] [node 4] ICP01 ready to work
2025-11-28T14:51:20.405379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788958509932992:2067] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.412472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788956528523458:2086] [node 4] ICP01 ready to work
2025-11-28T14:51:20.492959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788959577054554:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(306716204ns) {Configure:104(306711546ns) {ProcessConfigured:135(110876ns) {StartInitialHandshake:157(108671ns) {DropHandshakes:470(3091ns) {DropIncomingHandshake:442(834ns) DropOutgoingHandshake:457(556ns)} GenerateSessionVirtualId:988(1061ns) PrepareNewSessionHandshake:477(2009ns) {SwitchToState:179(687ns)}}}}}
2025-11-28T14:51:20.650855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788958756541014:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56686
2025-11-28T14:51:20.542197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788959577054603:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6000
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.592628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788958179380651:2080] [node 6] ICP01 ready to work
2025-11-28T14:51:20.446154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788956528523470:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
2025-11-28T14:51:20.528539Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788954214965673:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(243690648ns) {Configure:104(243680183ns) {ProcessConfigured:135(20083867ns) {StartInitialHandshake:157(20080597ns) {DropHandshakes:470(2957ns) {DropIncomingHandshake:442(710ns) DropOutgoingHandshake:457(619ns)} GenerateSessionVirtualId:988(993ns) PrepareNewSessionHandshake:477(4690ns) {SwitchToState:179(1145ns)}}}}}
2025-11-28T14:51:20.661674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788958509933038:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:20.661889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788958509933039:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:20.662012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788958509933040:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:20.662121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788958509933041:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:20.662218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788958509933042:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:20.697141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788954461573646:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(488252358ns) {Configure:104(488244821ns) {ProcessConfigured:135(86137ns) {StartInitialHandshake:157(83756ns) {DropHandshakes:470(3118ns) {DropIncomingHandshake:442(788ns) DropOutgoingHandshake:457(476ns)} GenerateSessionVirtualId:988(1542ns) PrepareNewSessionHandshake:477(1939ns) {SwitchToState:179(837ns)}}}}}
2025-11-28T14:51:20.592816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788958179380645:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32106
2025-11-28T14:51:20.592975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788958179380646:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32106
2025-11-28T14:51:20.595057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788958179380652:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32106
2025-11-28T14:51:20.616521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788958179380644:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32106
2025-11-28T14:51:20.712445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788958179380658:2079] [node 5] ICP01 ready to work
2025-11-28T14:51:20.768973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788953884413284:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(496209797ns) {Configure:104(496206143ns) {ProcessConfigured:135(116150ns) {StartInitialHandshake:157(113045ns) {DropHandshakes:470(4705ns) {DropIncomingHandshake:442(879ns) DropOutgoingHandshake:457(668ns)} GenerateSessionVirtualId:988(1271ns) PrepareNewSessionHandshake:477(2033ns) {SwitchToState:179(926ns)}}}}}
2025-11-28T14:51:20.722072Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788958756540954:2061] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:20.716761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788958509933043:2060] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.728719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788958509932992:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(323048156ns) {Configure:104(323037802ns) {ProcessConfigured:135(16046554ns) {StartInitialHandshake:157(16042447ns) {DropHandshakes:470(3406ns) {DropIncomingHandshake:442(704ns) DropOutgoingHandshake:457(678ns)} GenerateSessionVirtualId:988(1058ns) PrepareNewSessionHandshake:477(4087ns) {SwitchToState:179(1623ns)}}}}}
2025-11-28T14:51:20.809669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788957606662414:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(180904530ns) {Configure:104(180892833ns) {ProcessConfigured:135(65170ns) {StartInitialHandshake:157(63303ns) {DropHandshakes:470(2328ns) {DropIncomingHandshake:442(672ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(623ns) PrepareNewSessionHandshake:477(1976ns) {SwitchToState:179(757ns)}}}}}
2025-11-28T14:51:20.828947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788958179380658:2079] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(59716547ns) {Configure:104(59711498ns) {ProcessConfigured:135(36247497ns) {StartInitialHandshake:157(36243602ns) {DropHandshakes:470(15930281ns) {DropIncomingHandshake:442(882ns) DropOutgoingHandshake:457(3571ns)} GenerateSessionVirtualId:988(3012ns) PrepareNewSessionHandshake:477(3263ns) {SwitchToState:179(1547ns)}}}}}
2025-11-28T14:51:20.781576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788956528523458:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(292887667ns) {Configure:104(292880004ns) {ProcessConfigured:135(91192ns) {StartInitialHandshake:157(88530ns) {DropHandshakes:470(3588ns) {DropIncomingHandshake:442(985ns) DropOutgoingHandshake:457(509ns)} GenerateSessionVirtualId:988(976ns) PrepareNewSessionHandshake:477(8393ns) {SwitchToState:179(566ns)}}}}}
2025-11-28T14:51:20.777191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788956528523471:2082] [node 5] ICP01 ready to work
2025-11-28T14:51:20.805050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788956528523442:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:20.804792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788958756541019:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56686
2025-11-28T14:51:20.833513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788954214965674:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(447754440ns) {Configure:104(447750556ns) {ProcessConfigured:135(98690ns) {StartInitialHandshake:157(95588ns) {DropHandshakes:470(3584ns) {DropIncomingHandshake:442(1176ns) DropOutgoingHandshake:457(595ns)} GenerateSessionVirtualId:988(1352ns) PrepareNewSessionHandshake:477(2239ns) {SwitchToState:179(715ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:20.857086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788956528523474:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
2025-11-28T14:51:20.876802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788958509933043:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(100080151ns) {Configure:104(94549440ns) {ProcessConfigured:135(15285985ns) {StartInitialHandshake:157(15282654ns) {DropHandshakes:470(3434ns) {DropIncomingHandshake:442(1033ns) DropOutgoingHandshake:457(678ns)} GenerateSessionVirtualId:988(684ns) PrepareNewSessionHandshake:477(4654ns) {SwitchToState:179(1676ns)}}}}}
2025-11-28T14:51:20.904320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788957606662373:2072] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:20.952686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788956528523443:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:20.962191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788959577054564:2069] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:20.985348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788956528523460:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:20.996624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788959577054603:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:20.993541Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788956528523445:2109] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:20.996793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788955282087242:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:21.008593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788956528523445:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.009217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788956528523445:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.036606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788952233556087:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:21.037707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:21.037926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788952233556087:2049] [node 3] ICP22 created new session: [1:7577788960823490777:2049]
2025-11-28T14:51:21.037976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788960823490777:2049] [node 3] ICS09 handshake done sender: [1:7577788956528523445:2109] self: [1:7577788956528523444:0] peer: [3:7577788959577054618:0] socket: 64 qp: -1
2025-11-28T14:51:21.038007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788960823490777:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:21.038058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:21.038622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577788952233556087:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(28961721ns) {HandleHandshakeStatus:329(28958615ns) {DropIncomingHandshake:442(24160ns) DropOutgoingHandshake:457(495ns) SwitchToState:179(622ns) ProcessPendingSessionEvents:495(172782ns) {ForwardSessionEventToSession:612(77938ns) ForwardSessionEventToSession:612(10138ns) ForwardSessionEventToSession:612(2017ns) ForwardSessionEventToSession:612(30135ns) ForwardSessionEventToSession:612(2605ns) ForwardSessionEventToSession:612(10630ns) ForwardSessionEventToSession:612(1813ns) ForwardSessionEventToSession:612(1521ns) ForwardSessionEventToSession:612(7716ns) ForwardSessionEventToSession:612(19254ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.044868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788963051508316:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56686
2025-11-28T14:51:21.037548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788963872021915:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6000
2025-11-28T14:51:21.068609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577788955282087242:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577788959577054603:2105] held: no
2025-11-28T14:51:21.068849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788959577054603:2105] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:21.084482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788958179380643:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.100909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788955282087242:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:21.100996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:21.101044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788955282087242:2048] [node 1] ICP22 created new session: [3:7577788963872021916:2048]
2025-11-28T14:51:21.101090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788963872021916:2048] [node 1] ICS09 handshake done sender: [3:7577788959577054603:2105] self: [3:7577788959577054618:0] peer: [1:7577788956528523444:0] socket: 72 qp: -1
2025-11-28T14:51:21.101123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788963872021916:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:21.101179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.070665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788957606662371:2071] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.133660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788961901629719:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47620
2025-11-28T14:51:21.163316Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788956528523447:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.233325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788958179380645:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.233467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577788953884413283:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:21.234566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788962474347961:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32106
2025-11-28T14:51:21.233735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788956528523447:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.234200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788956528523447:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:21.236819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788952233556086:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:21.236874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:21.236922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788952233556086:2048] [node 2] ICP22 created new session: [1:7577788960823490781:2048]
2025-11-28T14:51:21.238217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788958179380645:2112] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:21.238463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577788953884413283:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:21.238532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.236964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788960823490781:2048] [node 2] ICS09 handshake done sender: [1:7577788956528523447:2110] self: [1:7577788956528523446:0] peer: [2:7577788962474347960:0] socket: 65 qp: -1
2025-11-28T14:51:21.236999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788960823490781:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:21.237030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:21.238570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788953884413283:2048] [node 1] ICP22 created new session: [2:7577788962474347962:2048]
2025-11-28T14:51:21.238615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788962474347962:2048] [node 1] ICS09 handshake done sender: [2:7577788958179380645:2112] self: [2:7577788962474347960:0] peer: [1:7577788956528523446:0] socket: 78 qp: -1
2025-11-28T14:51:21.238641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788962474347962:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:21.238665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:21.248479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788961901629720:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47620
2025-11-28T14:51:21.262489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788958756541005:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.338448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788959577054568:2071] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.344603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788958179380644:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.344774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788958179380646:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.367123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788958509933038:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.367266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788958509933043:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:21.366467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788957606662379:2075] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.368732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788957606662379:2075] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.395622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788962804900348:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:21.395739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788957606662379:2075] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.456860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788963872021919:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6000
2025-11-28T14:51:21.428837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788958509933038:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:21.428965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577788957606662361:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:21.429010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:21.429049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788957606662361:2061] [node 6] ICP22 created new session: [5:7577788961901629723:2061]
2025-11-28T14:51:21.460587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788961901629723:2061] [node 6] ICS09 handshake done sender: [5:7577788957606662379:2075] self: [5:7577788957606662378:0] peer: [6:7577788962804900347:0] socket: 61 qp: -1
2025-11-28T14:51:21.476480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788961901629723:2061] [node 6] ICS10 traffic start
2025-11-28T14:51:21.476538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:21.476981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577788957606662361:2061] [node 6] ICP35 event processing took too much time PendingConnection:279(80791643ns) {HandleHandshakeStatus:329(48419059ns) {DropIncomingHandshake:442(1007ns) DropOutgoingHandshake:457(665ns) SwitchToState:179(709ns) ProcessPendingSessionEvents:495(100852ns) {ForwardSessionEventToSession:612(69054ns) ForwardSessionEventToSession:612(5210ns) ForwardSessionEventToSession:612(2909ns) ForwardSessionEventToSession:612(2232ns) ForwardSessionEventToSession:612(7594ns) ForwardSessionEventToSession:612(2390ns) ForwardSessionEventToSession:612(1637ns)}}}
2025-11-28T14:51:21.483555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788958509933043:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:21.483662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:21.483708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788958509933043:2060] [node 5] ICP22 created new session: [6:7577788962804900349:2060]
2025-11-28T14:51:21.483746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788962804900349:2060] [node 5] ICS09 handshake done sender: [6:7577788958509933038:2112] self: [6:7577788962804900347:0] peer: [5:7577788957606662378:0] socket: 85 qp: -1
2025-11-28T14:51:21.483781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788962804900349:2060] [node 5] ICS10 traffic start
2025-11-28T14:51:21.483833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.490078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788963872021920:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6000
2025-11-28T14:51:21.498738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788959577054557:2063] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.538459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788958756541008:2111] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.591284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788958179380656:2117] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.644215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788958509932997:2072] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.644970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788958509933040:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.645097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788958509933041:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.645186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788958509933042:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.645430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788954214965674:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:21.645548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788958509932992:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:21.645586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788954214965673:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:21.647944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788958756541008:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.666520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788959577054557:2063] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.648584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788958179380652:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.672476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788958179380656:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.721349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788959577054557:2063] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:21.668929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788962804900355:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:21.720857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788962804900356:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:21.721606Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788958756541008:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.756435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788959577054555:2062] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.760962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788961901629725:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47620
2025-11-28T14:51:21.730154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788958179380656:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:21.748942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788963051508318:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.753830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788956528523464:2122] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.758569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788958179380651:2080] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.781754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788956528523464:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.773277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788958509933039:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.760430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788958756541006:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:21.768571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:21.782131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788956528523464:2122] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:21.788486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:21.788563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788958756541006:2072] [node 6] ICP22 created new session: [4:7577788963051508320:2072]
2025-11-28T14:51:21.773394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788958509933040:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:21.788636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788963051508320:2072] [node 6] ICS09 handshake done sender: [4:7577788958756541008:2111] self: [4:7577788958756541007:0] peer: [6:7577788962804900353:0] socket: 76 qp: -1
2025-11-28T14:51:21.788684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788963051508320:2072] [node 6] ICS10 traffic start
2025-11-28T14:51:21.788740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:21.789152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577788958756541006:2072] [node 6] ICP35 event processing took too much time PendingConnection:279(40525260ns) {HandleHandshakeStatus:329(40522136ns) {DropIncomingHandshake:442(1573ns) DropOutgoingHandshake:457(1080ns) SwitchToState:179(1195ns) ProcessPendingSessionEvents:495(1771ns)}}
2025-11-28T14:51:21.773530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788958509933041:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:21.773681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788962804900357:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:21.773962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788954214965674:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:21.774025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:21.774077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788954214965674:2050] [node 3] ICP22 created new session: [6:7577788962804900358:2050]
2025-11-28T14:51:21.774130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788962804900358:2050] [node 3] ICS09 handshake done sender: [6:7577788958509933040:2114] self: [6:7577788962804900352:0] peer: [3:7577788959577054556:0] socket: 88 qp: -1
2025-11-28T14:51:21.774173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788962804900358:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:21.774213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:21.774747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788958509932992:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:21.774791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:21.774821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788958509932992:2067] [node 4] ICP22 created new session: [6:7577788962804900360:2067]
2025-11-28T14:51:21.774852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788962804900360:2067] [node 4] ICS09 handshake done sender: [6:7577788958509933041:2115] self: [6:7577788962804900353:0] peer: [4:7577788958756541007:0] socket: 89 qp: -1
2025-11-28T14:51:21.774882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788962804900360:2067] [node 4] ICS10 traffic start
2025-11-28T14:51:21.774906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:21.775070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788954214965672:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:21.775169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788958509933042:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:21.781000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788954214965673:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:21.781060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:21.781106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788954214965673:2049] [node 2] ICP22 created new session: [6:7577788962804900363:2049]
2025-11-28T14:51:21.781147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788962804900363:2049] [node 2] ICS09 handshake done sender: [6:7577788958509933042:2116] self: [6:7577788962804900354:0] peer: [2:7577788958179380655:0] socket: 90 qp: -1
2025-11-28T14:51:21.781174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788962804900363:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:21.781215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:21.783166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788962804900365:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20326
2025-11-28T14:51:21.783545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788958509933039:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:21.783658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788954214965672:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:21.783716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:21.783759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788954214965672:2048] [node 1] ICP22 created new session: [6:7577788962804900366:2048]
2025-11-28T14:51:21.783796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788962804900366:2048] [node 1] ICS09 handshake done sender: [6:7577788958509933039:2113] self: [6:7577788962804900362:0] peer: [1:7577788956528523463:0] socket: 87 qp: -1
2025-11-28T14:51:21.783816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788962804900366:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:21.783843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:21.782298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788956528523459:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:21.782350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:21.782384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788956528523459:2085] [node 6] ICP22 created new session: [1:7577788960823490785:2085]
2025-11-28T14:51:21.782424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788960823490785:2085] [node 6] ICS09 handshake done sender: [1:7577788956528523464:2122] self: [1:7577788956528523463:0] peer: [6:7577788962804900362:0] socket: 66 qp: -1
2025-11-28T14:51:21.782457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788960823490785:2085] [node 6] ICS10 traffic start
2025-11-28T14:51:21.782488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:21.768679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788959577054555:2062] [node 6] ICP22 created new session: [3:7577788963872021923:2062]
2025-11-28T14:51:21.768743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788963872021923:2062] [node 6] ICS09 handshake done sender: [3:7577788959577054557:2063] self: [3:7577788959577054556:0] peer: [6:7577788962804900352:0] socket: 69 qp: -1
2025-11-28T14:51:21.768797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788963872021923:2062] [node 6] ICS10 traffic start
2025-11-28T14:51:21.768855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:21.769229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577788959577054555:2062] [node 6] ICP35 event processing took too much time PendingConnection:279(44354357ns) {HandleHandshakeStatus:329(44349196ns) {DropIncomingHandshake:442(1811ns) DropOutgoingHandshake:457(1273ns) SwitchToState:179(1291ns) ProcessPendingSessionEvents:495(1626ns)}}
2025-11-28T14:51:21.840493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:21.844427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788958179380651:2080] [node 6] ICP22 created new session: [2:7577788962474347968:2080]
2025-11-28T14:51:21.858751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788962474347968:2080] [node 6] ICS09 handshake done sender: [2:7577788958179380656:2117] self: [2:7577788958179380655:0] peer: [6:7577788962804900354:0] socket: 84 qp: -1
2025-11-28T14:51:21.858790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788962474347968:2080] [node 6] ICS10 traffic start
2025-11-28T14:51:21.858847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:21.859088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577788958179380651:2080] [node 6] ICP35 event processing took too much time PendingConnection:279(100455087ns) {HandleHandshakeStatus:329(100452452ns) {DropIncomingHandshake:442(39387776ns) DropOutgoingHandshake:457(1083ns) SwitchToState:179(1022ns) ProcessPendingSessionEvents:495(652ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.878774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788958179380648:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.876629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788958509932988:2065] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.892442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788958756541014:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:21.892640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788954461573645:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:21.885222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788963872021925:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6000
2025-11-28T14:51:21.908430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788958179380648:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.927227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788958179380648:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:21.932127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788958179380639:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:21.927567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788963051508323:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56686
2025-11-28T14:51:21.932564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788958756541014:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:21.940887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788954461573645:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:21.940941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:21.931596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788958509933045:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:21.932199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:21.936496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788958179380639:2086] [node 4] ICP22 created new session: [2:7577788962474347972:2086]
2025-11-28T14:51:21.960497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788962474347972:2086] [node 4] ICS09 handshake done sender: [2:7577788958179380648:2114] self: [2:7577788958179380647:0] peer: [4:7577788963051508322:0] socket: 82 qp: -1
2025-11-28T14:51:21.947051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788958756541019:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.980604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788961901629726:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47620
2025-11-28T14:51:21.972952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788962474347972:2086] [node 4] ICS10 traffic start
2025-11-28T14:51:21.973034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:21.979317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577788958179380639:2086] [node 4] ICP35 event processing took too much time PendingConnection:279(50628771ns) {HandleHandshakeStatus:329(50625407ns) {DropIncomingHandshake:442(17193ns) DropOutgoingHandshake:457(571ns) SwitchToState:179(543ns) ProcessPendingSessionEvents:495(1563ns)}}
2025-11-28T14:51:21.948523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788954461573645:2049] [node 2] ICP22 created new session: [4:7577788963051508324:2049]
2025-11-28T14:51:21.948583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788963051508324:2049] [node 2] ICS09 handshake done sender: [4:7577788958756541014:2113] self: [4:7577788963051508322:0] peer: [2:7577788958179380647:0] socket: 86 qp: -1
2025-11-28T14:51:21.948620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788963051508324:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:21.948692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:21.949196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577788958756541006:2072] [node 6] ICP18 handshake (actor [4:7577788958756541019:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:21.985048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788963872021926:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6000
2025-11-28T14:51:22.008540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788958179380650:2115] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:22.009702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788963051508316:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.009849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788954461573644:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:22.023174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788959577054612:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:22.029019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788961901629719:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.029154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788953311695051:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:22.029680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788959577054612:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.040785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788961901629720:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.040937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788953311695049:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:22.043270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:22.043512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577788962474347962:2048] [node 1] ICS01 socket: 78 reason# EndOfStream
2025-11-28T14:51:22.043548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577788953884413283:2048] [node 1] ICP30 unregister session Session# [2:7577788962474347962:2048] VirtualId# [2:7577788962474347960:0]
2025-11-28T14:51:22.043593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577788962474347962:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.043659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:22.043701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:22.045236Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:22.045438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788963872021916:2048] [node 1] ICS01 socket: 72 reason# EndOfStream
2025-11-28T14:51:22.045474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788955282087242:2048] [node 1] ICP30 unregister session Session# [3:7577788963872021916:2048] VirtualId# [3:7577788959577054618:0]
2025-11-28T14:51:22.045497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788963872021916:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.045556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:22.045587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:22.048228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788959577054612:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:22.048125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788966196597024:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47620
2025-11-28T14:51:22.058412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788961901629719:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:22.060803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788957606662387:2081] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:22.058539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788959577054554:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:22.061846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788963872021919:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.062193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:22.062245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788959577054554:2060] [node 5] ICP22 created new session: [3:7577788968166989224:2060]
2025-11-28T14:51:22.062300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788968166989224:2060] [node 5] ICS09 handshake done sender: [3:7577788959577054612:2114] self: [3:7577788959577054611:0] peer: [5:7577788966196597023:0] socket: 60 qp: -1
2025-11-28T14:51:22.062337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788968166989224:2060] [node 5] ICS10 traffic start
2025-11-28T14:51:22.062393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:22.062636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577788959577054554:2060] [node 5] ICP18 handshake (actor [3:7577788963872021919:2117]) is too old
2025-11-28T14:51:22.064038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788953311695051:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:22.064109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:22.064146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788953311695051:2050] [node 3] ICP22 created new session: [5:7577788966196597026:2050]
2025-11-28T14:51:22.064186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788966196597026:2050] [node 3] ICS09 handshake done sender: [5:7577788961901629719:2115] self: [5:7577788966196597023:0] peer: [3:7577788959577054611:0] socket: 99 qp: -1
2025-11-28T14:51:22.064217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788966196597026:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:22.064257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:22.069859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:22.070104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788962804900366:2048] [node 1] ICS01 socket: 87 reason# EndOfStream
2025-11-28T14:51:22.070137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788954214965672:2048] [node 1] ICP30 unregister session Session# [6:7577788962804900366:2048] VirtualId# [6:7577788962804900362:0]
2025-11-28T14:51:22.070174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788962804900366:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.070228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:22.070261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:22.072025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788967346475624:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56686
2025-11-28T14:51:22.077386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788953311695049:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788961901629720:2116] held: no
2025-11-28T14:51:22.077038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788954461573644:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577788963051508316:2117] held: no
2025-11-28T14:51:22.098364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788958509933048:2118] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:22.144456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788963051508318:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.099379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788963872021920:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.145661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788954461573646:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:22.146824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788967346475626:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56686
2025-11-28T14:51:22.147897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788963051508318:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:22.148048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788954461573646:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:22.148083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:22.148118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788954461573646:2050] [node 3] ICP22 created new session: [4:7577788967346475627:2050]
2025-11-28T14:51:22.148160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788967346475627:2050] [node 3] ICS09 handshake done sender: [4:7577788963051508318:2118] self: [4:7577788967346475625:0] peer: [3:7577788959577054613:0] socket: 117 qp: -1
2025-11-28T14:51:22.148187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788967346475627:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:22.148210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:22.099835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577788959577054555:2062] [node 6] ICP18 handshake (actor [3:7577788963872021920:2118]) is too old
2025-11-28T14:51:22.176782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788958756541016:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:22.225645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788961901629726:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.225782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788953311695050:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:22.121001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788959577054614:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:22.146039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788959577054614:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.146496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788959577054614:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:22.147102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788959577054597:2067] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:22.147158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:22.147194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788959577054597:2067] [node 4] ICP22 created new session: [3:7577788968166989227:2067]
2025-11-28T14:51:22.147236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788968166989227:2067] [node 4] ICS09 handshake done sender: [3:7577788959577054614:2115] self: [3:7577788959577054613:0] peer: [4:7577788967346475625:0] socket: 116 qp: -1
2025-11-28T14:51:22.147261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788968166989227:2067] [node 4] ICS10 traffic start
2025-11-28T14:51:22.147290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:22.227346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788963872021925:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.227448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788955282087243:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:22.236600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788961901629725:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:22.236819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788957606662414:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:22.240616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788958756541016:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:22.242223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788966196597031:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47620
2025-11-28T14:51:22.242546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788958756541016:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:22.243113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788958756541012:2070] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:22.243167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:22.243207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788958756541012:2070] [node 5] ICP22 created new session: [4:7577788967346475630:2070]
2025-11-28T14:51:22.243246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788967346475630:2070] [node 5] ICS09 handshake done sender: [4:7577788958756541016:2114] self: [4:7577788958756541015:0] peer: [5:7577788966196597030:0] socket: 118 qp: -1
2025-11-28T14:51:22.243270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788967346475630:2070] [node 5] ICS10 traffic start
2025-11-28T14:51:22.243301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:22.244413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788961901629725:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:22.244787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788957606662414:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:22.244854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:22.244890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788957606662414:2059] [node 4] ICP22 created new session: [5:7577788966196597032:2059]
2025-11-28T14:51:22.244926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788966196597032:2059] [node 4] ICS09 handshake done sender: [5:7577788961901629725:2117] self: [5:7577788966196597030:0] peer: [4:7577788958756541015:0] socket: 119 qp: -1
2025-11-28T14:51:22.244952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788966196597032:2059] [node 4] ICS10 traffic start
2025-11-28T14:51:22.244988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:22.256807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577788955282087243:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577788963872021925:2119] held: no
2025-11-28T14:51:22.262565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:22.264696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788953311695050:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788961901629726:2118] held: no
2025-11-28T14:51:22.256549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:22.256782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788962804900363:2049] [node 2] ICS01 socket: 90 reason# EndOfStream
2025-11-28T14:51:22.256826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788954214965673:2049] [node 2] ICP30 unregister session Session# [6:7577788962804900363:2049] VirtualId# [6:7577788962804900354:0]
2025-11-28T14:51:22.256855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788962804900363:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.256946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:22.256984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:22.272433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788963051508324:2049] [node 2] ICS01 socket: 86 reason# EndOfStream
2025-11-28T14:51:22.272477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788954461573645:2049] [node 2] ICP30 unregister session Session# [4:7577788963051508324:2049] VirtualId# [4:7577788963051508322:0]
2025-11-28T14:51:22.272509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788963051508324:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.272615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:22.272658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:22.280610Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788958756541018:2115] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:22.304561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:22.304764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788966196597026:2050] [node 3] ICS01 socket: 99 reason# EndOfStream
2025-11-28T14:51:22.304795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788953311695051:2050] [node 3] ICP30 unregister session Session# [5:7577788966196597026:2050] VirtualId# [5:7577788966196597023:0]
2025-11-28T14:51:22.304823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788966196597026:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.304908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:22.304949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:22.305278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:22.305430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788967346475627:2050] [node 3] ICS01 socket: 117 reason# EndOfStream
2025-11-28T14:51:22.305456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788954461573646:2050] [node 3] ICP30 unregister session Session# [4:7577788967346475627:2050] VirtualId# [4:7577788967346475625:0]
2025-11-28T14:51:22.305476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788967346475627:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.305531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:22.305561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:22.308503Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:22.308742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788962804900358:2050] [node 3] ICS01 socket: 88 reason# EndOfStream
2025-11-28T14:51:22.309158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788954214965674:2050] [node 3] ICP30 unregister session Session# [6:7577788962804900358:2050] VirtualId# [6:7577788962804900352:0]
2025-11-28T14:51:22.309194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788962804900358:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.309270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:22.309304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:22.314975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788957606662422:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:22.321076Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:22.321283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788966196597032:2059] [node 4] ICS01 socket: 119 reason# EndOfStream
2025-11-28T14:51:22.321319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788957606662414:2059] [node 4] ICP30 unregister session Session# [5:7577788966196597032:2059] VirtualId# [5:7577788966196597030:0]
2025-11-28T14:51:22.321351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788966196597032:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.321430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:22.321475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:22.321552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:22.321730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788962804900360:2067] [node 4] ICS01 socket: 89 reason# EndOfStream
2025-11-28T14:51:22.321760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788958509932992:2067] [node 4] ICP30 unregister session Session# [6:7577788962804900360:2067] VirtualId# [6:7577788962804900353:0]
2025-11-28T14:51:22.321778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788962804900360:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.321833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:22.321861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:22.344539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:22.344866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577788962804900349:2060] [node 5] ICS07 socket disconnect 85 reason# EndOfStream
2025-11-28T14:51:22.344923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788962804900349:2060] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:22.345094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:22.345173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577788962804900349:2060] [node 5] ICS15 start handshake
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:22.408313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788964574805822:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:22.408358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788964574805823:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:22.440683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788965446159313:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:22.440726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788965446159314:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:22.516889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788965743107813:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:22.516932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788965743107814:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:22.556799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788966867029257:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:22.556839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788966867029258:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:22.556860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788966867029259:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:22.584744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788966354943506:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:22.584786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788966354943507:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:22.584809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788966354943508:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:22.634850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788965634394570:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:22.634885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788965634394571:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:22.634908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788965634394572:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:23.004631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788965743107857:2090] [node 4] ICP01 ready to work
2025-11-28T14:51:22.992778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788964574805867:2090] [node 6] ICP01 ready to work
2025-11-28T14:51:23.032604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788969741126655:2090] [node 4] ICP01 ready to work
2025-11-28T14:51:23.077916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788969929361879:2059] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.110378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788969929361879:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32112591ns) {Configure:104(1717674ns) {ProcessConfigured:135(94687ns) {StartInitialHandshake:157(92331ns) {DropHandshakes:470(3427ns) {DropIncomingHandshake:442(848ns) DropOutgoingHandshake:457(658ns)} GenerateSessionVirtualId:988(1118ns) PrepareNewSessionHandshake:477(2320ns) {SwitchToState:179(709ns)}}}}}
2025-11-28T14:51:23.058233Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788966354943506:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(65268512ns) {Configure:104(47317864ns) {ProcessConfigured:135(102129ns) {StartInitialHandshake:157(99294ns) {DropHandshakes:470(3553ns) {DropIncomingHandshake:442(824ns) DropOutgoingHandshake:457(571ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:477(2357ns) {SwitchToState:179(1062ns)}}}}}
2025-11-28T14:51:23.165328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788970649910846:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5726
2025-11-28T14:51:23.214065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788964574805823:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(193194873ns) {Configure:104(193191228ns) {ProcessConfigured:135(31139472ns) {StartInitialHandshake:157(31134265ns) {DropHandshakes:470(4482ns) {DropIncomingHandshake:442(1317ns) DropOutgoingHandshake:457(540ns)} GenerateSessionVirtualId:988(2133ns) PrepareNewSessionHandshake:477(8490ns) {SwitchToState:179(2737ns)}}}}}
2025-11-28T14:51:23.252834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788966867029258:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(323526193ns) {Configure:104(323517229ns) {ProcessConfigured:135(85049ns) {StartInitialHandshake:157(81100ns) {DropHandshakes:470(3173ns) {DropIncomingHandshake:442(845ns) DropOutgoingHandshake:457(570ns)} GenerateSessionVirtualId:988(1573ns) PrepareNewSessionHandshake:477(1740ns) {SwitchToState:179(700ns)}}}}}
2025-11-28T14:51:23.216861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788969741126664:2088] [node 5] ICP01 ready to work
2025-11-28T14:51:23.245000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788970038075172:2089] [node 6] ICP01 ready to work
2025-11-28T14:51:23.280894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788970038075174:2088] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.310074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788965743107813:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(231010010ns) {Configure:104(231005790ns) {ProcessConfigured:135(56864ns) {StartInitialHandshake:157(54573ns) {DropHandshakes:470(2997ns) {DropIncomingHandshake:442(753ns) DropOutgoingHandshake:457(532ns)} GenerateSessionVirtualId:988(1005ns) PrepareNewSessionHandshake:477(2003ns) {SwitchToState:179(550ns)}}}}}
2025-11-28T14:51:23.310448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788970038075175:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-28T14:51:23.310632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788970038075176:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-28T14:51:23.256852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788971161996616:2100] [node 6] ICP01 ready to work
2025-11-28T14:51:23.261632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788969741126674:2087] [node 6] ICP01 ready to work
2025-11-28T14:51:23.316654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788970649910872:2075] [node 4] ICP01 ready to work
2025-11-28T14:51:23.273215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788966867029257:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(244346340ns) {Configure:104(244340922ns) {ProcessConfigured:135(77335ns) {StartInitialHandshake:157(74554ns) {DropHandshakes:470(3345ns) {DropIncomingHandshake:442(924ns) DropOutgoingHandshake:457(600ns)} GenerateSessionVirtualId:988(928ns) PrepareNewSessionHandshake:477(1756ns) {SwitchToState:179(688ns)}}}}}
2025-11-28T14:51:23.292104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788968869773186:2088] [node 4] ICP01 ready to work
2025-11-28T14:51:23.318937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788964574805822:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(300027845ns) {Configure:104(300022882ns) {ProcessConfigured:135(26812003ns) {StartInitialHandshake:157(26809130ns) {DropHandshakes:470(3409ns) {DropIncomingHandshake:442(836ns) DropOutgoingHandshake:457(448ns)} GenerateSessionVirtualId:988(962ns) PrepareNewSessionHandshake:477(2178ns) {SwitchToState:179(1064ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.300728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788971161996623:2096] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.390059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788965634394571:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(133017957ns) {Configure:104(119957135ns) {ProcessConfigured:135(71249ns) {StartInitialHandshake:157(69001ns) {DropHandshakes:470(3341ns) {DropIncomingHandshake:442(984ns) DropOutgoingHandshake:457(502ns)} GenerateSessionVirtualId:988(791ns) PrepareNewSessionHandshake:477(2079ns) {SwitchToState:179(638ns)}}}}}
2025-11-28T14:51:23.390643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788969929361935:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.312247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788968869773188:2089] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.395381Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788965634394572:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(152647756ns) {Configure:104(152639304ns) {ProcessConfigured:135(95804ns) {StartInitialHandshake:157(93482ns) {DropHandshakes:470(3545ns) {DropIncomingHandshake:442(1001ns) DropOutgoingHandshake:457(520ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:477(2187ns) {SwitchToState:179(796ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.418383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788971161996624:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21831
2025-11-28T14:51:23.409240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788970038075172:2089] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(57558052ns) {Configure:104(57552445ns) {ProcessConfigured:135(37513102ns) {StartInitialHandshake:157(37509637ns) {DropHandshakes:470(1729ns) {DropIncomingHandshake:442(275ns) DropOutgoingHandshake:457(153ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(3725ns) {SwitchToState:179(745ns)}}}}}
2025-11-28T14:51:23.367971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788968869773185:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54318
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.417719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788965743107857:2090] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(405061963ns) {Configure:104(405054862ns) {ProcessConfigured:135(16920189ns) {StartInitialHandshake:157(16916132ns) {DropHandshakes:470(2425ns) {DropIncomingHandshake:442(565ns) DropOutgoingHandshake:457(379ns)} GenerateSessionVirtualId:988(852ns) PrepareNewSessionHandshake:477(3602ns) {SwitchToState:179(1556ns)}}}}}
2025-11-28T14:51:23.421925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788970038075188:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-28T14:51:23.434100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788971161996625:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21831
2025-11-28T14:51:23.423332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788970649910874:2077] [node 6] ICP01 ready to work
2025-11-28T14:51:23.477452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788970649910872:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(127696700ns) {Configure:104(127684906ns) {ProcessConfigured:135(78142ns) {StartInitialHandshake:157(70858ns) {DropHandshakes:470(2198ns) {DropIncomingHandshake:442(723ns) DropOutgoingHandshake:457(277ns)} GenerateSessionVirtualId:988(524ns) PrepareNewSessionHandshake:477(1073ns) {SwitchToState:179(426ns)}}}}}
2025-11-28T14:51:23.282030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788965446159313:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(261237578ns) {Configure:104(261232039ns) {ProcessConfigured:135(63905ns) {StartInitialHandshake:157(60156ns) {DropHandshakes:470(3529ns) {DropIncomingHandshake:442(897ns) DropOutgoingHandshake:457(500ns)} GenerateSessionVirtualId:988(960ns) PrepareNewSessionHandshake:477(1914ns) {SwitchToState:179(944ns)}}}}}
2025-11-28T14:51:23.473014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788971161996623:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(171940705ns) {Configure:104(171935607ns) {ProcessConfigured:135(114260ns) {StartInitialHandshake:157(111095ns) {DropHandshakes:470(6747ns) {DropIncomingHandshake:442(4258ns) DropOutgoingHandshake:457(694ns)} GenerateSessionVirtualId:988(1397ns) PrepareNewSessionHandshake:477(2122ns) {SwitchToState:179(882ns)}}}}}
2025-11-28T14:51:23.368153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788968869773194:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54318
2025-11-28T14:51:23.318525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788969741126664:2088] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28176445ns) {Configure:104(11175018ns) {ProcessConfigured:135(92461ns) {StartInitialHandshake:157(89253ns) {DropHandshakes:470(2836ns) {DropIncomingHandshake:442(474ns) DropOutgoingHandshake:457(401ns)} GenerateSessionVirtualId:988(662ns) PrepareNewSessionHandshake:477(1640ns) {SwitchToState:179(556ns)}}}}}
2025-11-28T14:51:23.381243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788969741126681:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21066
2025-11-28T14:51:23.422675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788969741126684:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21066
2025-11-28T14:51:23.433014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788969741126655:2090] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(388215345ns) {Configure:104(388207103ns) {ProcessConfigured:135(85254ns) {StartInitialHandshake:157(81680ns) {DropHandshakes:470(3157ns) {DropIncomingHandshake:442(980ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(892ns) PrepareNewSessionHandshake:477(1929ns) {SwitchToState:179(619ns)}}}}}
2025-11-28T14:51:23.449248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788965446159314:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(433697571ns) {Configure:104(433690054ns) {ProcessConfigured:135(46474702ns) {StartInitialHandshake:157(46471527ns) {DropHandshakes:470(5821ns) {DropIncomingHandshake:442(372ns) DropOutgoingHandshake:457(4572ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:477(17621ns) {SwitchToState:179(1897ns)}}}}}
2025-11-28T14:51:23.460629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788969741126682:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21066
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.425938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788964574805867:2090] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(405223717ns) {Configure:104(389229598ns) {ProcessConfigured:135(73093ns) {StartInitialHandshake:157(71008ns) {DropHandshakes:470(3023ns) {DropIncomingHandshake:442(784ns) DropOutgoingHandshake:457(474ns)} GenerateSessionVirtualId:988(863ns) PrepareNewSessionHandshake:477(1799ns) {SwitchToState:179(636ns)}}}}}
2025-11-28T14:51:23.478421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788969929361940:2063] [node 4] ICP01 ready to work
2025-11-28T14:51:23.456630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788968869773192:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54318
2025-11-28T14:51:23.480858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788970649910873:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5726
2025-11-28T14:51:23.456896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788968869773193:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54318
2025-11-28T14:51:23.480791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788966867029259:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(410804727ns) {Configure:104(410798729ns) {ProcessConfigured:135(1529287ns) {StartInitialHandshake:157(1527008ns) {DropHandshakes:470(2734ns) {DropIncomingHandshake:442(432ns) DropOutgoingHandshake:457(464ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:477(1439ns) {SwitchToState:179(566ns)}}}}}
2025-11-28T14:51:23.482550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788965634394570:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(296185211ns) {Configure:104(296178589ns) {ProcessConfigured:135(1884110ns) {StartInitialHandshake:157(1881926ns) {DropHandshakes:470(1192ns) {DropIncomingHandshake:442(237ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(309ns) PrepareNewSessionHandshake:477(1445ns) {SwitchToState:179(642ns)}}}}}
2025-11-28T14:51:23.483176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788971161996616:2100] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(192756305ns) {Configure:104(192751292ns) {ProcessConfigured:135(384729ns) {StartInitialHandshake:157(383110ns) {DropHandshakes:470(1353ns) {DropIncomingHandshake:442(352ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(566ns) PrepareNewSessionHandshake:477(1071ns) {SwitchToState:179(467ns)}}}}}
2025-11-28T14:51:23.483520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788969741126674:2087] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(176368345ns) {Configure:104(176362944ns) {ProcessConfigured:135(606341ns) {StartInitialHandshake:157(605097ns) {DropHandshakes:470(7727ns) {DropIncomingHandshake:442(571ns) DropOutgoingHandshake:457(383ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:477(1694ns) {SwitchToState:179(781ns)}}}}}
2025-11-28T14:51:23.483477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788966354943507:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(324481243ns) {Configure:104(324475910ns) {ProcessConfigured:135(5870334ns) {StartInitialHandshake:157(5868081ns) {DropHandshakes:470(2957ns) {DropIncomingHandshake:442(665ns) DropOutgoingHandshake:457(613ns)} GenerateSessionVirtualId:988(479ns) PrepareNewSessionHandshake:477(1580ns) {SwitchToState:179(586ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.484280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788970649910876:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5726
2025-11-28T14:51:23.484413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788970649910879:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5726
2025-11-28T14:51:23.484790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788965743107814:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(299024126ns) {Configure:104(299006958ns) {ProcessConfigured:135(183260ns) {StartInitialHandshake:157(181987ns) {DropHandshakes:470(1137ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(451ns) PrepareNewSessionHandshake:477(756ns) {SwitchToState:179(362ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.487837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788971161996633:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21831
2025-11-28T14:51:23.491858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788970649910848:2091] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.497953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788970038075175:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.502740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788969929361890:2069] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.504732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788970649910846:2090] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.518976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788970038075178:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.532562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788969741126681:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.532576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773194:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.531472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788971161996618:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.545421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788970038075184:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.554508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788969929361935:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.554632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788965634394572:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:23.557693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773185:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.556872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788970649910829:2074] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.557677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788970038075184:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.582402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788969929361946:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
2025-11-28T14:51:23.568547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788970038075184:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:23.588502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788970038075172:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:23.588564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:23.589123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788970038075172:2089] [node 6] ICP22 created new session: [3:7577788970038075193:2089]
2025-11-28T14:51:23.589170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788970038075193:2089] [node 6] ICS09 handshake done sender: [3:7577788970038075184:2114] self: [3:7577788970038075182:0] peer: [6:7577788969929361945:0] socket: 83 qp: -1
2025-11-28T14:51:23.589204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788970038075193:2089] [node 6] ICS10 traffic start
2025-11-28T14:51:23.589253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:23.588934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788969929361935:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:23.592126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788965634394572:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:23.592192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:23.592239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788965634394572:2050] [node 3] ICP22 created new session: [6:7577788969929361947:2050]
2025-11-28T14:51:23.600261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788968869773182:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.592280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788969929361947:2050] [node 3] ICS09 handshake done sender: [6:7577788969929361935:2112] self: [6:7577788969929361945:0] peer: [3:7577788970038075182:0] socket: 89 qp: -1
2025-11-28T14:51:23.592305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788969929361947:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:23.592353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:23.631966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788970038075176:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.632155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788965743107813:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:23.632558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773182:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.631275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788971161996634:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21831
2025-11-28T14:51:23.634967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788968869773182:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:23.635228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788964574805823:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:23.635321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:23.635381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788964574805823:2049] [node 3] ICP22 created new session: [1:7577788968869773204:2049]
2025-11-28T14:51:23.635434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788968869773204:2049] [node 3] ICS09 handshake done sender: [1:7577788968869773182:2110] self: [1:7577788968869773180:0] peer: [3:7577788970038075196:0] socket: 64 qp: -1
2025-11-28T14:51:23.635460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788968869773204:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:23.635513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:23.656169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788968869773191:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.653346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788970649910885:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5726
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.640520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577788965743107813:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11988) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577788970038075176:2111] held: no
2025-11-28T14:51:23.677563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788970038075197:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-28T14:51:23.672500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788971161996624:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.672687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788966867029257:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:23.684464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788970038075176:2111] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:23.688561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788965743107813:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:23.688611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:23.688672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788965743107813:2048] [node 1] ICP22 created new session: [3:7577788970038075199:2048]
2025-11-28T14:51:23.688717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788970038075199:2048] [node 1] ICS09 handshake done sender: [3:7577788970038075176:2111] self: [3:7577788970038075196:0] peer: [1:7577788968869773180:0] socket: 73 qp: -1
2025-11-28T14:51:23.688744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788970038075199:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:23.688783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:23.680548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773191:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.684564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788968869773191:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:23.676867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788969741126693:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21066
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.698744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788971161996636:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21831
2025-11-28T14:51:23.709990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788971161996624:2112] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:23.710257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788966867029257:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:23.710357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:23.710406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788966867029257:2048] [node 1] ICP22 created new session: [4:7577788971161996637:2048]
2025-11-28T14:51:23.710448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788971161996637:2048] [node 1] ICS09 handshake done sender: [4:7577788971161996624:2112] self: [4:7577788971161996635:0] peer: [1:7577788968869773190:0] socket: 88 qp: -1
2025-11-28T14:51:23.710480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788971161996637:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:23.710520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:23.712503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788968869773186:2088] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:23.712563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:23.713085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788968869773186:2088] [node 4] ICP22 created new session: [1:7577788968869773208:2088]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.716541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788969741126684:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.715565Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788969929361934:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.721800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788969929361949:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
2025-11-28T14:51:23.727902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788968869773208:2088] [node 4] ICS09 handshake done sender: [1:7577788968869773191:2113] self: [1:7577788968869773190:0] peer: [4:7577788971161996635:0] socket: 84 qp: -1
2025-11-28T14:51:23.727981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788968869773208:2088] [node 4] ICS10 traffic start
2025-11-28T14:51:23.728036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:23.732920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577788968869773186:2088] [node 4] ICP35 event processing took too much time PendingConnection:279(28207846ns) {HandleHandshakeStatus:329(28204004ns) {DropIncomingHandshake:442(1362ns) DropOutgoingHandshake:457(1482ns) SwitchToState:179(1095ns) ProcessPendingSessionEvents:495(4484445ns) {ForwardSessionEventToSession:612(4416887ns) ForwardSessionEventToSession:612(10805ns) ForwardSessionEventToSession:612(2485ns) ForwardSessionEventToSession:612(2056ns) ForwardSessionEventToSession:612(6969ns) ForwardSessionEventToSession:612(2538ns) ForwardSessionEventToSession:612(2324ns) ForwardSessionEventToSession:612(16345ns) ForwardSessionEventToSession:612(2338ns) ForwardSessionEventToSession:612(7533ns)}}}
2025-11-28T14:51:23.737984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788971161996621:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.739466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773192:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.739583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577788968869773186:2088] [node 4] ICP18 handshake (actor [1:7577788968869773192:2114]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.761079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788970038075201:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-28T14:51:23.771930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788969741126677:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.788727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773193:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.812483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788969929361938:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.817225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788970038075188:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.849180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788969741126682:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.828851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788968869773212:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54318
2025-11-28T14:51:23.849336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577788965446159313:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:23.850168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788969741126696:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21066
2025-11-28T14:51:23.850803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788969741126682:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:23.850969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577788965446159313:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:23.851022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:23.851061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788965446159313:2048] [node 1] ICP22 created new session: [2:7577788969741126697:2048]
2025-11-28T14:51:23.851104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788969741126697:2048] [node 1] ICS09 handshake done sender: [2:7577788969741126682:2114] self: [2:7577788969741126695:0] peer: [1:7577788968869773187:0] socket: 86 qp: -1
2025-11-28T14:51:23.851129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788969741126697:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:23.851157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:23.848398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788968869773189:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.880890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788969741126688:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.817343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577788970038075172:2089] [node 6] ICP18 handshake (actor [3:7577788970038075188:2116]) is too old
2025-11-28T14:51:23.908455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788971161996625:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.908703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788966867029258:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:23.849602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773189:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.850342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788968869773189:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:23.868426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788964574805822:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:23.868950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:23.868998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788964574805822:2048] [node 2] ICP22 created new session: [1:7577788968869773214:2048]
2025-11-28T14:51:23.904648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788968869773214:2048] [node 2] ICS09 handshake done sender: [1:7577788968869773189:2112] self: [1:7577788968869773187:0] peer: [2:7577788969741126695:0] socket: 85 qp: -1
2025-11-28T14:51:23.920431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788968869773214:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:23.920489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:23.924912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577788964574805822:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(72087607ns) {HandleHandshakeStatus:329(72084940ns) {DropIncomingHandshake:442(22329ns) DropOutgoingHandshake:457(716ns) SwitchToState:179(723ns) ProcessPendingSessionEvents:495(128378ns) {ForwardSessionEventToSession:612(69594ns) ForwardSessionEventToSession:612(8328ns) ForwardSessionEventToSession:612(2647ns) ForwardSessionEventToSession:612(2171ns) ForwardSessionEventToSession:612(4579ns) ForwardSessionEventToSession:612(1916ns) ForwardSessionEventToSession:612(4260ns) ForwardSessionEventToSession:612(2200ns) ForwardSessionEventToSession:612(1796ns) ForwardSessionEventToSession:612(20879ns)}}}
2025-11-28T14:51:23.852314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788970038075202:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-28T14:51:23.928424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788969741126688:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:23.929029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788969741126688:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.953984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788971161996641:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21831
2025-11-28T14:51:23.956628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788969741126655:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:23.956717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:23.956762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788969741126655:2090] [node 4] ICP22 created new session: [2:7577788969741126701:2090]
2025-11-28T14:51:23.956795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788969741126701:2090] [node 4] ICS09 handshake done sender: [2:7577788969741126688:2117] self: [2:7577788969741126687:0] peer: [4:7577788971161996640:0] socket: 95 qp: -1
2025-11-28T14:51:23.956826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788969741126701:2090] [node 4] ICS10 traffic start
2025-11-28T14:51:23.956882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:23.962362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788971161996625:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:23.964808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788969929361951:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
2025-11-28T14:51:23.966238Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788966867029258:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19026) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577788971161996625:2113] held: no
2025-11-28T14:51:23.985334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788970038075181:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:23.977810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788966867029258:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.008510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788970649910873:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.008712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788966354943508:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:24.025132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788970038075181:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.052956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788974224329248:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
2025-11-28T14:51:23.977947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.056472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788970038075181:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:24.056964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788974944878183:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5726
2025-11-28T14:51:24.057646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788970649910873:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:24.057862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788966354943508:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.057949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:24.058307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788966354943508:2050] [node 3] ICP22 created new session: [5:7577788974944878184:2050]
2025-11-28T14:51:24.058347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788974944878184:2050] [node 3] ICS09 handshake done sender: [5:7577788970649910873:2113] self: [5:7577788974944878182:0] peer: [3:7577788970038075180:0] socket: 81 qp: -1
2025-11-28T14:51:24.058375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788974944878184:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:24.058408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:23.978020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788966867029258:2049] [node 2] ICP22 created new session: [4:7577788971161996642:2049]
2025-11-28T14:51:23.978087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788971161996642:2049] [node 2] ICS09 handshake done sender: [4:7577788971161996625:2113] self: [4:7577788971161996640:0] peer: [2:7577788969741126687:0] socket: 96 qp: -1
2025-11-28T14:51:23.978128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788971161996642:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:23.978174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.064891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788970038075174:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.064972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:24.065015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788970038075174:2088] [node 5] ICP22 created new session: [3:7577788974333042500:2088]
2025-11-28T14:51:24.065065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788974333042500:2088] [node 5] ICS09 handshake done sender: [3:7577788970038075181:2113] self: [3:7577788970038075180:0] peer: [5:7577788974944878182:0] socket: 77 qp: -1
2025-11-28T14:51:24.065101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788974333042500:2088] [node 5] ICS10 traffic start
2025-11-28T14:51:24.065137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:24.072965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788974036094000:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21066
2025-11-28T14:51:24.082413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788971161996627:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.092697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788970649910879:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.092938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788970649910872:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:24.093276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788971161996627:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.093808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788971161996627:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.109751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788974944878188:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5726
2025-11-28T14:51:24.094036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788971161996623:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.102114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:24.111004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788971161996623:2096] [node 5] ICP22 created new session: [4:7577788975456963942:2096]
2025-11-28T14:51:24.111449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788975456963942:2096] [node 5] ICS09 handshake done sender: [4:7577788971161996627:2114] self: [4:7577788971161996626:0] peer: [5:7577788974944878187:0] socket: 98 qp: -1
2025-11-28T14:51:24.111491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788975456963942:2096] [node 5] ICS10 traffic start
2025-11-28T14:51:24.111545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:24.112088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577788971161996623:2096] [node 5] ICP35 event processing took too much time PendingConnection:279(17834423ns) {HandleHandshakeStatus:329(17831859ns) {DropIncomingHandshake:442(426ns) DropOutgoingHandshake:457(785ns) SwitchToState:179(6587ns) ProcessPendingSessionEvents:495(115585ns) {ForwardSessionEventToSession:612(67704ns) ForwardSessionEventToSession:612(6127ns) ForwardSessionEventToSession:612(5910ns) ForwardSessionEventToSession:612(2369ns) ForwardSessionEventToSession:612(2474ns) ForwardSessionEventToSession:612(5173ns) ForwardSessionEventToSession:612(2339ns) ForwardSessionEventToSession:612(3355ns) ForwardSessionEventToSession:612(5527ns) ForwardSessionEventToSession:612(5107ns)}}}
2025-11-28T14:51:24.149168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788970649910879:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:24.149215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788969741126680:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.169161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788970649910872:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.169227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:24.169279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788970649910872:2075] [node 4] ICP22 created new session: [5:7577788974944878189:2075]
2025-11-28T14:51:24.169333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788974944878189:2075] [node 4] ICS09 handshake done sender: [5:7577788970649910879:2116] self: [5:7577788974944878187:0] peer: [4:7577788971161996626:0] socket: 99 qp: -1
2025-11-28T14:51:24.169390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788974944878189:2075] [node 4] ICS10 traffic start
2025-11-28T14:51:24.169448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:24.172150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788970649910876:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.172291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788966354943507:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:24.172750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788969741126680:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.173687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788974944878192:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5726
2025-11-28T14:51:24.173987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788969741126680:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:24.174455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788969741126664:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.174536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:24.174580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788969741126664:2088] [node 5] ICP22 created new session: [2:7577788974036094002:2088]
2025-11-28T14:51:24.174628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788974036094002:2088] [node 5] ICS09 handshake done sender: [2:7577788969741126680:2112] self: [2:7577788969741126679:0] peer: [5:7577788974944878191:0] socket: 94 qp: -1
2025-11-28T14:51:24.174660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788974036094002:2088] [node 5] ICS10 traffic start
2025-11-28T14:51:24.174693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:24.176021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788970649910876:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:24.176297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788966354943507:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.176480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:24.176527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788966354943507:2049] [node 2] ICP22 created new session: [5:7577788974944878193:2049]
2025-11-28T14:51:24.176577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788974944878193:2049] [node 2] ICS09 handshake done sender: [5:7577788970649910876:2114] self: [5:7577788974944878191:0] peer: [2:7577788969741126679:0] socket: 97 qp: -1
2025-11-28T14:51:24.176606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788974944878193:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:24.176650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:24.184840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788970649910878:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.214030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788970038075187:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.193383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788971161996633:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.193544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577788971161996623:2096] [node 5] ICP18 handshake (actor [4:7577788971161996633:2117]) is too old
2025-11-28T14:51:24.215598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788971161996634:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.215716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788966867029259:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:24.216066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788970038075187:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.220888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788970038075187:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:24.222045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788965743107857:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.220887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788974224329249:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.277128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:24.277201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788965743107857:2090] [node 4] ICP22 created new session: [3:7577788974333042503:2090]
2025-11-28T14:51:24.277251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788974333042503:2090] [node 4] ICS09 handshake done sender: [3:7577788970038075187:2115] self: [3:7577788970038075186:0] peer: [4:7577788975456963944:0] socket: 104 qp: -1
2025-11-28T14:51:24.277286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788974333042503:2090] [node 4] ICS10 traffic start
2025-11-28T14:51:24.277363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:24.280482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577788965743107857:2090] [node 4] ICP35 event processing took too much time PendingConnection:279(58269352ns) {HandleHandshakeStatus:329(58265341ns) {DropIncomingHandshake:442(921ns) DropOutgoingHandshake:457(765ns) SwitchToState:179(1007ns) ProcessPendingSessionEvents:495(2734052ns) {ForwardSessionEventToSession:612(2709263ns) ForwardSessionEventToSession:612(7576ns) ForwardSessionEventToSession:612(2567ns) ForwardSessionEventToSession:612(7335ns)}}}
2025-11-28T14:51:24.261387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788975456963945:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21831
2025-11-28T14:51:24.291804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788971161996634:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:24.291129Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788968869773196:2117] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.304376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788970649910885:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.304539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788966354943506:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:24.303374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788966867029259:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.308189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:24.308255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788966867029259:2050] [node 3] ICP22 created new session: [4:7577788975456963946:2050]
2025-11-28T14:51:24.308328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788975456963946:2050] [node 3] ICS09 handshake done sender: [4:7577788971161996634:2118] self: [4:7577788975456963944:0] peer: [3:7577788970038075186:0] socket: 105 qp: -1
2025-11-28T14:51:24.308379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788975456963946:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:24.308439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:24.316629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577788966867029259:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(24072455ns) {HandleHandshakeStatus:329(24068521ns) {DropIncomingHandshake:442(2771ns) DropOutgoingHandshake:457(8010ns) SwitchToState:179(920ns) ProcessPendingSessionEvents:495(7860976ns) {ForwardSessionEventToSession:612(7854269ns)}}}
2025-11-28T14:51:24.316494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773196:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.317536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788968869773196:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:24.318797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577788970038075190:2117] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.317751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788968869773188:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.325265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788969741126693:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.376637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577788969741126674:2087] [node 6] ICP35 event processing took too much time PendingConnection:279(19972051ns) {HandleCleanupEventQueue:832(19967277ns) {CleanupEventQueue:841(19964343ns) {DropSessionEvent:512(22457ns) DropSessionEvent:512(19908992ns) DropSessionEvent:512(11150ns) DropSessionEvent:512(4589ns) DropSessionEvent:512(8597ns)} ScheduleCleanupEventQueue:822(337ns)}}
2025-11-28T14:51:24.356621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788969929361949:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.360453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788966354943506:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11988) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788970649910885:2119] held: no
2025-11-28T14:51:24.360593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577788966354943506:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(30152855ns) {HandleHandshakeStatus:395(30149986ns) {LogHandshakeFail:472(15242ns) {UpdateErrorStateLog:545(4970ns)} DropOutgoingHandshake:457(2976ns) ProcessPendingSessionEvents:495(769ns)}}
2025-11-28T14:51:24.329198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:24.329280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788968869773188:2089] [node 5] ICP22 created new session: [1:7577788973164740513:2089]
2025-11-28T14:51:24.329329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788973164740513:2089] [node 5] ICS09 handshake done sender: [1:7577788968869773196:2117] self: [1:7577788968869773195:0] peer: [5:7577788974944878196:0] socket: 108 qp: -1
2025-11-28T14:51:24.329363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788973164740513:2089] [node 5] ICS10 traffic start
2025-11-28T14:51:24.329421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:24.341967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577788968869773200:2118] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.389199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788971161996630:2115] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.376530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788965634394570:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:24.389359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773200:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.405036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788968869773200:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:24.424615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788964574805867:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.424699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:24.424749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788964574805867:2090] [node 6] ICP22 created new session: [1:7577788973164740516:2090]
2025-11-28T14:51:24.424786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788973164740516:2090] [node 6] ICS09 handshake done sender: [1:7577788968869773200:2118] self: [1:7577788968869773199:0] peer: [6:7577788974224329250:0] socket: 112 qp: -1
2025-11-28T14:51:24.424825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788973164740516:2090] [node 6] ICS10 traffic start
2025-11-28T14:51:24.424879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:24.408905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788974224329251:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.430768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788969929361949:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:24.430964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788965634394570:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.431061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:24.431099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788965634394570:2048] [node 1] ICP22 created new session: [6:7577788974224329252:2048]
2025-11-28T14:51:24.431148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788974224329252:2048] [node 1] ICS09 handshake done sender: [6:7577788969929361949:2117] self: [6:7577788974224329250:0] peer: [1:7577788968869773199:0] socket: 113 qp: -1
2025-11-28T14:51:24.431180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788974224329252:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:24.431211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:24.430614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788974944878197:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5726
2025-11-28T14:51:24.432804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788970649910885:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:24.438593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788966354943506:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.449461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788968869773212:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.449589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577788964574805867:2090] [node 6] ICP18 handshake (actor [1:7577788968869773212:2119]) is too old
2025-11-28T14:51:24.437867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788969929361942:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.436577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788970038075201:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.436717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577788965743107857:2090] [node 4] ICP18 handshake (actor [3:7577788970038075201:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.461100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788975456963948:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21831
2025-11-28T14:51:24.461470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:24.461518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788966354943506:2048] [node 1] ICP22 created new session: [5:7577788974944878198:2048]
2025-11-28T14:51:24.461571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788974944878198:2048] [node 1] ICS09 handshake done sender: [5:7577788970649910885:2119] self: [5:7577788974944878196:0] peer: [1:7577788968869773195:0] socket: 79 qp: -1
2025-11-28T14:51:24.461606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788974944878198:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:24.461654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:24.461917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577788966354943506:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26422054ns) {HandleHandshakeStatus:329(26419501ns) {DropIncomingHandshake:442(1013ns) DropOutgoingHandshake:457(408ns) SwitchToState:179(808ns) ProcessPendingSessionEvents:495(1069ns)}}
2025-11-28T14:51:24.480345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788969741126686:2116] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.504634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788970038075202:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.516702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788965743107814:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:24.517573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577788969929361940:2063] [node 4] ICP35 event processing took too much time PendingConnection:279(36860740ns) {HandleCleanupEventQueue:832(36859325ns) {CleanupEventQueue:841(36857266ns) {DropSessionEvent:512(3562124ns) DropSessionEvent:512(27563ns) DropSessionEvent:512(11970913ns) DropSessionEvent:512(16697ns) DropSessionEvent:512(4134ns) DropSessionEvent:512(4771ns) DropSessionEvent:512(4072ns) DropSessionEvent:512(3692ns) DropSessionEvent:512(8955866ns) DropSessionEvent:512(7693ns)} ScheduleCleanupEventQueue:822(362ns)}}
2025-11-28T14:51:24.517833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788969741126686:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.518513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788974333042506:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-28T14:51:24.524739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788969741126686:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:24.536739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788965446159314:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.536847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:24.536890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788965446159314:2049] [node 3] ICP22 created new session: [2:7577788974036094006:2049]
2025-11-28T14:51:24.536928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788974036094006:2049] [node 3] ICS09 handshake done sender: [2:7577788969741126686:2116] self: [2:7577788969741126683:0] peer: [3:7577788974333042505:0] socket: 116 qp: -1
2025-11-28T14:51:24.536958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788974036094006:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:24.536992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:24.538374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577788965743107814:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19026) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577788970038075202:2120] held: no
2025-11-28T14:51:24.540476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788970038075202:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:24.544460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788965743107814:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.544527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:24.544571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788965743107814:2049] [node 2] ICP22 created new session: [3:7577788974333042507:2049]
2025-11-28T14:51:24.544610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788974333042507:2049] [node 2] ICS09 handshake done sender: [3:7577788970038075202:2120] self: [3:7577788974333042505:0] peer: [2:7577788969741126683:0] socket: 117 qp: -1
2025-11-28T14:51:24.544638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788974333042507:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:24.544684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:24.549379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577788969741126691:2118] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.562957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788969929361951:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.585980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788969741126691:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.576849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788965634394571:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.583719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577788971161996632:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.592603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788969741126691:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:24.588509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788974224329248:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.617708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788969741126674:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.617782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:24.617828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788969741126674:2087] [node 6] ICP22 created new session: [2:7577788974036094009:2087]
2025-11-28T14:51:24.617869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788974036094009:2087] [node 6] ICS09 handshake done sender: [2:7577788969741126691:2118] self: [2:7577788969741126690:0] peer: [6:7577788974224329255:0] socket: 118 qp: -1
2025-11-28T14:51:24.617899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788974036094009:2087] [node 6] ICS10 traffic start
2025-11-28T14:51:24.617940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:24.637996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788971161996632:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.609125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788969929361940:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:24.612940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788974224329256:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
2025-11-28T14:51:24.652516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:24.652756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577788969741126697:2048] [node 1] ICS01 socket: 86 reason# EndOfStream
2025-11-28T14:51:24.652789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577788965446159313:2048] [node 1] ICP30 unregister session Session# [2:7577788969741126697:2048] VirtualId# [2:7577788969741126695:0]
2025-11-28T14:51:24.652819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577788969741126697:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.652901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:24.652938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:24.641938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788971161996632:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:24.658314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788971161996616:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.658395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:24.658452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788971161996616:2100] [node 6] ICP22 created new session: [4:7577788975456963950:2100]
2025-11-28T14:51:24.658509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788975456963950:2100] [node 6] ICS09 handshake done sender: [4:7577788971161996632:2116] self: [4:7577788971161996631:0] peer: [6:7577788974224329257:0] socket: 68 qp: -1
2025-11-28T14:51:24.658549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788975456963950:2100] [node 6] ICS10 traffic start
2025-11-28T14:51:24.658616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:24.668516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:24.638038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788965634394571:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19026) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788969929361951:2118] held: no
2025-11-28T14:51:24.670732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788970038075199:2048] [node 1] ICS01 socket: 73 reason# EndOfStream
2025-11-28T14:51:24.670796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788965743107813:2048] [node 1] ICP30 unregister session Session# [3:7577788970038075199:2048] VirtualId# [3:7577788970038075196:0]
2025-11-28T14:51:24.670837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788970038075199:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.670874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:24.670964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:24.638184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788969929361951:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:24.638321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788965634394571:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.638356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:24.638416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788965634394571:2049] [node 2] ICP22 created new session: [6:7577788974224329258:2049]
2025-11-28T14:51:24.643546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788974224329258:2049] [node 2] ICS09 handshake done sender: [6:7577788969929361951:2118] self: [6:7577788974224329255:0] peer: [2:7577788969741126690:0] socket: 119 qp: -1
2025-11-28T14:51:24.650134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788974224329258:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:24.650184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:24.684945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788970649910880:2117] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.730118Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.781140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788974944878198:2048] [node 1] ICS01 socket: 79 reason# EndOfStream
2025-11-28T14:51:24.781199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788966354943506:2048] [node 1] ICP30 unregister session Session# [5:7577788974944878198:2048] VirtualId# [5:7577788974944878196:0]
2025-11-28T14:51:24.781237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788974944878198:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.781310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:24.781348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:24.780789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:24.732524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:24.782164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788974224329252:2048] [node 1] ICS01 socket: 113 reason# EndOfStream
2025-11-28T14:51:24.782268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788965634394570:2048] [node 1] ICP30 unregister session Session# [6:7577788974224329252:2048] VirtualId# [6:7577788974224329250:0]
2025-11-28T14:51:24.782348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788974224329252:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.782512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:24.782571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:24.784602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788974036094000:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.784775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577788969741126664:2088] [node 5] ICP18 handshake (actor [2:7577788974036094000:2121]) is too old
2025-11-28T14:51:24.783764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788971161996637:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-11-28T14:51:24.783815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788966867029257:2048] [node 1] ICP30 unregister session Session# [4:7577788971161996637:2048] VirtualId# [4:7577788971161996635:0]
2025-11-28T14:51:24.783860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788971161996637:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.783901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:24.783963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:24.800966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788970649910882:2118] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.787726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788974224329259:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
2025-11-28T14:51:24.813600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788970649910882:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:24.814000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:24.810184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788974224329249:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.825962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788970649910882:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:24.842880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577788970649910874:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:24.814262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788974333042507:2049] [node 2] ICS01 socket: 117 reason# EndOfStream
2025-11-28T14:51:24.810390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788974224329248:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:24.842959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:24.814308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788965743107814:2049] [node 2] ICP30 unregister session Session# [3:7577788974333042507:2049] VirtualId# [3:7577788974333042505:0]
2025-11-28T14:51:24.814342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788974333042507:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.814458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:24.814508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:24.810572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788969929361879:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:24.843017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788970649910874:2077] [node 6] ICP22 created new session: [5:7577788974944878201:2077]
2025-11-28T14:51:24.843075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788974944878201:2077] [node 6] ICS09 handshake done sender: [5:7577788970649910882:2118] self: [5:7577788970649910881:0] peer: [6:7577788974224329261:0] socket: 126 qp: -1
2025-11-28T14:51:24.811349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788969929361940:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.813517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:24.814586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788974224329263:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48681
2025-11-28T14:51:24.825778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788969929361940:2063] [node 4] ICP22 created new session: [6:7577788974224329262:2063]
2025-11-28T14:51:24.836549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788974224329262:2063] [node 4] ICS09 handshake done sender: [6:7577788974224329248:2119] self: [6:7577788974224329257:0] peer: [4:7577788971161996631:0] socket: 120 qp: -1
2025-11-28T14:51:24.836621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788974224329262:2063] [node 4] ICS10 traffic start
2025-11-28T14:51:24.836707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:24.837069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577788969929361940:2063] [node 4] ICP35 event processing took too much time PendingConnection:279(25602156ns) {HandleHandshakeStatus:329(25600395ns) {DropIncomingHandshake:442(590ns) DropOutgoingHandshake:457(4619ns) SwitchToState:179(3424ns) ProcessPendingSessionEvents:495(1835ns)}}
2025-11-28T14:51:24.856521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788974224329249:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:24.856820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788969929361879:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:24.856905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:24.856951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788969929361879:2059] [node 5] ICP22 created new session: [6:7577788974224329265:2059]
2025-11-28T14:51:24.857002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788974224329265:2059] [node 5] ICS09 handshake done sender: [6:7577788974224329249:2120] self: [6:7577788974224329261:0] peer: [5:7577788970649910881:0] socket: 100 qp: -1
2025-11-28T14:51:24.857041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788974224329265:2059] [node 5] ICS10 traffic start
2025-11-28T14:51:24.857078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:24.878818Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577788969929361879:2059] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:24.884620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [6:7577788969929361879:2059] [node 5] ICP35 event processing took too much time StateWork:294(17396131ns) {HandleHandshakeStatus:395(17390290ns) {LogHandshakeFail:472(10653414ns) {UpdateErrorStateLog:545(3069267ns)}}}
2025-11-28T14:51:24.905803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788969929361944:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:24.912091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788975456963948:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:24.912317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577788971161996616:2100] [node 6] ICP18 handshake (actor [4:7577788975456963948:2122]) is too old
2025-11-28T14:51:24.843108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788974944878201:2077] [node 6] ICS10 traffic start
2025-11-28T14:51:24.843148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:24.932554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:24.932554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:24.932802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788974944878193:2049] [node 2] ICS01 socket: 97 reason# EndOfStream
2025-11-28T14:51:24.932845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788966354943507:2049] [node 2] ICP30 unregister session Session# [5:7577788974944878193:2049] VirtualId# [5:7577788974944878191:0]
2025-11-28T14:51:24.932873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788974944878193:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.932971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:24.933019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:24.933516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788971161996642:2049] [node 2] ICS01 socket: 96 reason# EndOfStream
2025-11-28T14:51:24.933563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788966867029258:2049] [node 2] ICP30 unregister session Session# [4:7577788971161996642:2049] VirtualId# [4:7577788971161996640:0]
2025-11-28T14:51:24.933586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788971161996642:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.933646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:24.933689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:24.935189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:24.935426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788974224329258:2049] [node 2] ICS01 socket: 119 reason# EndOfStream
2025-11-28T14:51:24.935471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788965634394571:2049] [node 2] ICP30 unregister session Session# [6:7577788974224329258:2049] VirtualId# [6:7577788974224329255:0]
2025-11-28T14:51:24.935499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788974224329258:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.935601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:24.935639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:24.964596Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:24.964818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788974944878184:2050] [node 3] ICS01 socket: 81 reason# EndOfStream
2025-11-28T14:51:24.964864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788966354943508:2050] [node 3] ICP30 unregister session Session# [5:7577788974944878184:2050] VirtualId# [5:7577788974944878182:0]
2025-11-28T14:51:24.964895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788974944878184:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.964993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:24.965042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:24.965145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:24.965314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788975456963946:2050] [node 3] ICS01 socket: 105 reason# EndOfStream
2025-11-28T14:51:24.965341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788966867029259:2050] [node 3] ICP30 unregister session Session# [4:7577788975456963946:2050] VirtualId# [4:7577788975456963944:0]
2025-11-28T14:51:24.965371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788975456963946:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.965434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:24.965469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:24.967849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:24.968240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788969929361947:2050] [node 3] ICS01 socket: 89 reason# EndOfStream
2025-11-28T14:51:24.968311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788965634394572:2050] [node 3] ICP30 unregister session Session# [6:7577788969929361947:2050] VirtualId# [6:7577788969929361945:0]
2025-11-28T14:51:24.968381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788969929361947:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.968541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:24.968602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:24.992589Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:24.998305Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:24.998914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788974944878189:2075] [node 4] ICS01 socket: 99 reason# EndOfStream
2025-11-28T14:51:24.998966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788970649910872:2075] [node 4] ICP30 unregister session Session# [5:7577788974944878189:2075] VirtualId# [5:7577788974944878187:0]
2025-11-28T14:51:24.998998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788974944878189:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.999129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:24.999177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:24.999717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788974224329262:2063] [node 4] ICS01 socket: 120 reason# EndOfStream
2025-11-28T14:51:24.999765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788969929361940:2063] [node 4] ICP30 unregister session Session# [6:7577788974224329262:2063] VirtualId# [6:7577788974224329257:0]
2025-11-28T14:51:24.999793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788974224329262:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:24.999863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:24.999910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:25.020542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:25.020794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788974224329265:2059] [node 5] ICS01 socket: 100 reason# EndOfStream
2025-11-28T14:51:25.020842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788969929361879:2059] [node 5] ICP30 unregister session Session# [6:7577788974224329265:2059] VirtualId# [6:7577788974224329261:0]
2025-11-28T14:51:25.020876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788974224329265:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:25.020995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:25.021036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:25.116743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788977784458423:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:25.116782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788977784458424:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:25.136712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788977976249538:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:25.136752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788977976249539:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:25.176727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788977869542250:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:25.176761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788977869542251:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:25.256623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788980850039455:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:25.256678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788980850039456:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:25.256709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788980850039457:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:25.312630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788980259695398:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:25.312699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788980259695399:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:25.312728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788980259695400:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:25.564158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788981315099679:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:25.564240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788981315099680:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:25.564281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788981315099681:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:25.564977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788981315099701:2067] [node 4] ICP01 ready to work
2025-11-28T14:51:25.672855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788980850039477:2065] [node 5] ICP01 ready to work
2025-11-28T14:51:25.776878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788977976249601:2065] [node 6] ICP01 ready to work
2025-11-28T14:51:25.777214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788977976249602:2067] [node 5] ICP01 ready to work
2025-11-28T14:51:25.777481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788977976249603:2069] [node 4] ICP01 ready to work
2025-11-28T14:51:25.810990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788977976249538:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31529456ns) {Configure:104(31525515ns) {ProcessConfigured:135(73677ns) {StartInitialHandshake:157(70948ns) {DropHandshakes:470(1771ns) {DropIncomingHandshake:442(447ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(749ns) PrepareNewSessionHandshake:477(1798ns) {SwitchToState:179(1074ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:25.821098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788977976249610:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3872
2025-11-28T14:51:25.673232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788980850039478:2067] [node 6] ICP01 ready to work
2025-11-28T14:51:25.797831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788981315099681:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(229248556ns) {Configure:104(229231305ns) {ProcessConfigured:135(78863ns) {StartInitialHandshake:157(75714ns) {DropHandshakes:470(2801ns) {DropIncomingHandshake:442(698ns) DropOutgoingHandshake:457(415ns)} GenerateSessionVirtualId:988(1545ns) PrepareNewSessionHandshake:477(1411ns) {SwitchToState:179(518ns)}}}}}
2025-11-28T14:51:25.799654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788981315099738:2075] [node 5] ICP01 ready to work
2025-11-28T14:51:25.819324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788980259695435:2065] [node 4] ICP01 ready to work
2025-11-28T14:51:25.840488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788977784458486:2070] [node 6] ICP01 ready to work
2025-11-28T14:51:25.767070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788980850039478:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(79366925ns) {Configure:104(79361996ns) {ProcessConfigured:135(69135ns) {StartInitialHandshake:157(64523ns) {DropHandshakes:470(2126ns) {DropIncomingHandshake:442(545ns) DropOutgoingHandshake:457(326ns)} GenerateSessionVirtualId:988(1284ns) PrepareNewSessionHandshake:477(1201ns) {SwitchToState:179(378ns)}}}}}
2025-11-28T14:51:25.894028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788980850039457:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(54452070ns) {Configure:104(54445792ns) {ProcessConfigured:135(61153ns) {StartInitialHandshake:157(59524ns) {DropHandshakes:470(2540ns) {DropIncomingHandshake:442(559ns) DropOutgoingHandshake:457(391ns)} GenerateSessionVirtualId:988(863ns) PrepareNewSessionHandshake:477(1889ns) {SwitchToState:179(555ns)}}}}}
2025-11-28T14:51:25.900465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788980259695462:2066] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:25.919934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788977976249618:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3872
2025-11-28T14:51:25.922411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788977869542251:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(389700185ns) {Configure:104(378713919ns) {ProcessConfigured:135(53402ns) {StartInitialHandshake:157(49362ns) {DropHandshakes:470(1472ns) {DropIncomingHandshake:442(424ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(611ns) PrepareNewSessionHandshake:477(1170ns) {SwitchToState:179(522ns)}}}}}
2025-11-28T14:51:25.922895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788977869542319:2068] [node 4] ICP01 ready to work
2025-11-28T14:51:25.923148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788977869542320:2069] [node 6] ICP01 ready to work
2025-11-28T14:51:25.923410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788977869542321:2073] [node 5] ICP01 ready to work
2025-11-28T14:51:25.936041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788977784458424:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39367481ns) {Configure:104(31494799ns) {ProcessConfigured:135(48783ns) {StartInitialHandshake:157(47249ns) {DropHandshakes:470(1141ns) {DropIncomingHandshake:442(270ns) DropOutgoingHandshake:457(94ns)} GenerateSessionVirtualId:988(353ns) PrepareNewSessionHandshake:477(1876ns) {SwitchToState:179(1126ns)}}}}}
2025-11-28T14:51:25.936399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788977784458489:2075] [node 4] ICP01 ready to work
2025-11-28T14:51:25.936695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788977784458490:2079] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:25.944094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788981315099757:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:25.932890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788980259695462:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17095643ns) {Configure:104(17093220ns) {ProcessConfigured:135(3948313ns) {StartInitialHandshake:157(3945924ns) {DropHandshakes:470(2448ns) {DropIncomingHandshake:442(516ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(590ns) PrepareNewSessionHandshake:477(5673ns) {SwitchToState:179(1094ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.020746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788980259695400:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(56420806ns) {Configure:104(56416945ns) {ProcessConfigured:135(55222538ns) {StartInitialHandshake:157(55220256ns) {DropHandshakes:470(772ns) {DropIncomingHandshake:442(194ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:477(9922ns) {SwitchToState:179(6032ns)}}}}}
2025-11-28T14:51:26.021468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788980259695467:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50199
2025-11-28T14:51:25.999622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788977784458489:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(59819699ns) {Configure:104(59816996ns) {ProcessConfigured:135(71827ns) {StartInitialHandshake:157(69840ns) {DropHandshakes:470(2345ns) {DropIncomingHandshake:442(564ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(1076ns) PrepareNewSessionHandshake:477(1118ns) {SwitchToState:179(551ns)}}}}}
2025-11-28T14:51:26.067688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788977869542319:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(124400330ns) {Configure:104(124397587ns) {ProcessConfigured:135(10953404ns) {StartInitialHandshake:157(10950137ns) {DropHandshakes:470(2426ns) {DropIncomingHandshake:442(694ns) DropOutgoingHandshake:457(436ns)} GenerateSessionVirtualId:988(1204ns) PrepareNewSessionHandshake:477(5110ns) {SwitchToState:179(2848ns)}}}}}
2025-11-28T14:51:26.008080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788977784458495:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36340
2025-11-28T14:51:26.008257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788977784458496:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36340
2025-11-28T14:51:26.008341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788977784458497:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36340
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.149356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788982271216915:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3872
2025-11-28T14:51:26.149447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788982271216916:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3872
2025-11-28T14:51:26.156583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788977784458496:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.180504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788977869542320:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(73125356ns) {Configure:104(73120620ns) {ProcessConfigured:135(97723ns) {StartInitialHandshake:157(93784ns) {DropHandshakes:470(2828ns) {DropIncomingHandshake:442(557ns) DropOutgoingHandshake:457(513ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:477(2744ns) {SwitchToState:179(1078ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.227340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788982271216917:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3872
2025-11-28T14:51:26.225088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788977869542321:2073] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(44328245ns) {Configure:104(44320649ns) {ProcessConfigured:135(72443ns) {StartInitialHandshake:157(70161ns) {DropHandshakes:470(2024ns) {DropIncomingHandshake:442(499ns) DropOutgoingHandshake:457(360ns)} GenerateSessionVirtualId:988(739ns) PrepareNewSessionHandshake:477(1466ns) {SwitchToState:179(708ns)}}}}}
2025-11-28T14:51:26.225666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788977869542322:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64863
2025-11-28T14:51:26.225855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788977869542323:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64863
2025-11-28T14:51:26.226350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788982164509628:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64863
2025-11-28T14:51:26.240431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788977869542323:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.277038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788977869542251:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.300490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788977976249610:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.280434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788981315099757:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.375460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788985610067054:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:26.375683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788985610067055:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:26.375834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788981315099680:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.383935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788977784458495:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.384342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788982079425801:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36340
2025-11-28T14:51:26.384718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788985610067054:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.384801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788981315099679:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.382423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788977976249609:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.436450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788977784458492:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.436521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788977784458492:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.440479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788980259695467:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.440725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788980259695435:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.382475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577788977976249609:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.439643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788985145006829:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61947
2025-11-28T14:51:26.439755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788985145006830:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61947
2025-11-28T14:51:26.440298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788982271216916:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.440359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788982271216915:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.441046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788985610067055:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.441299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788981315099738:2075] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.442531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788985610067059:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:26.441973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788977784458497:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.442682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788977976249605:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.442707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577788977976249605:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.443441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788977976249618:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.443833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577788977976249538:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.444301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788980259695469:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.444323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577788980259695469:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.445350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788977784458492:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:26.445637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788977784458494:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.445653Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788977784458494:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.446139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788977784458486:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.446209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:26.446245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788977784458486:2070] [node 6] ICP22 created new session: [1:7577788982079425806:2070]
2025-11-28T14:51:26.446295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788982079425806:2070] [node 6] ICS09 handshake done sender: [1:7577788977784458492:2111] self: [1:7577788977784458491:0] peer: [6:7577788985610067057:0] socket: 88 qp: -1
2025-11-28T14:51:26.446331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788982079425806:2070] [node 6] ICS10 traffic start
2025-11-28T14:51:26.446364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:26.456515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788980259695469:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:26.468599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788977784458494:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:26.468971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788977784458423:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.469036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:26.469077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788977784458423:2048] [node 2] ICP22 created new session: [1:7577788982079425810:2048]
2025-11-28T14:51:26.469115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788982079425810:2048] [node 2] ICS09 handshake done sender: [1:7577788977784458494:2112] self: [1:7577788977784458493:0] peer: [2:7577788982271216922:0] socket: 89 qp: -1
2025-11-28T14:51:26.469142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788982079425810:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:26.469185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.456699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788985145006829:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.456634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788977976249609:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:26.477437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788980850039480:2069] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.477479Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577788980850039480:2069] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.502033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788977976249605:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:26.487436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788977869542322:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.496709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788982079425801:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.504584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577788980259695462:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.504635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:26.504689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788980259695462:2066] [node 6] ICP22 created new session: [5:7577788984554662776:2066]
2025-11-28T14:51:26.504725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788984554662776:2066] [node 6] ICS09 handshake done sender: [5:7577788980259695469:2113] self: [5:7577788980259695468:0] peer: [6:7577788985610067058:0] socket: 93 qp: -1
2025-11-28T14:51:26.504750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788984554662776:2066] [node 6] ICS10 traffic start
2025-11-28T14:51:26.504778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:26.505350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788984554662775:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50199
2025-11-28T14:51:26.506878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788985145006830:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.487520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788982164509628:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.488040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788977869542250:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.488107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788982164509633:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64863
2025-11-28T14:51:26.501772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788982164509636:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64863
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.526829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788982271216917:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.526885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788977976249601:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.526958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:26.526998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788977976249601:2065] [node 6] ICP22 created new session: [2:7577788982271216925:2065]
2025-11-28T14:51:26.527045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788982271216925:2065] [node 6] ICS09 handshake done sender: [2:7577788977976249609:2112] self: [2:7577788977976249608:0] peer: [6:7577788985610067056:0] socket: 72 qp: -1
2025-11-28T14:51:26.527075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788982271216925:2065] [node 6] ICS10 traffic start
2025-11-28T14:51:26.527113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:26.527596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788977976249539:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.527636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:26.527670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788977976249539:2049] [node 3] ICP22 created new session: [2:7577788982271216927:2049]
2025-11-28T14:51:26.527705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788982271216927:2049] [node 3] ICS09 handshake done sender: [2:7577788977976249605:2110] self: [2:7577788977976249604:0] peer: [3:7577788982164509632:0] socket: 70 qp: -1
2025-11-28T14:51:26.527738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788982271216927:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:26.527760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:26.528158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788977976249538:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577788982271216915:2117] held: no
2025-11-28T14:51:26.528689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788984554662779:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50199
2025-11-28T14:51:26.496814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788977784458488:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.534645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788984554662775:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.534821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788980259695398:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.496834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788977784458488:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.501182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788977784458488:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:26.502638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788977784458424:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.502695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:26.502737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788977784458424:2049] [node 3] ICP22 created new session: [1:7577788982079425815:2049]
2025-11-28T14:51:26.502784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788982079425815:2049] [node 3] ICS09 handshake done sender: [1:7577788977784458488:2110] self: [1:7577788977784458487:0] peer: [3:7577788982164509635:0] socket: 87 qp: -1
2025-11-28T14:51:26.502814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788982079425815:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:26.502849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:26.534448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788982079425817:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36340
2025-11-28T14:51:26.533012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788985145006832:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61947
2025-11-28T14:51:26.533143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788980850039457:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.536643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788980850039480:2069] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:26.537074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788980259695467:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:26.537328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788982079425817:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.537410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788982079425797:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.537425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788982079425797:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.538605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788982164509628:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:26.538855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788980259695435:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.538903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:26.538943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788980259695435:2065] [node 4] ICP22 created new session: [5:7577788984554662782:2065]
2025-11-28T14:51:26.538990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788984554662782:2065] [node 4] ICS09 handshake done sender: [5:7577788980259695467:2112] self: [5:7577788984554662773:0] peer: [4:7577788980850039479:0] socket: 78 qp: -1
2025-11-28T14:51:26.539035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788984554662782:2065] [node 4] ICS10 traffic start
2025-11-28T14:51:26.539063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:26.540574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788985145006832:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.540616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788980850039477:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.540685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:26.540718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788980850039477:2065] [node 5] ICP22 created new session: [4:7577788985145006835:2065]
2025-11-28T14:51:26.540752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788985145006835:2065] [node 5] ICS09 handshake done sender: [4:7577788980850039480:2069] self: [4:7577788980850039479:0] peer: [5:7577788984554662773:0] socket: 74 qp: -1
2025-11-28T14:51:26.540774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788985145006835:2065] [node 5] ICS10 traffic start
2025-11-28T14:51:26.540794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:26.541891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788977869542323:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.542326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577788977976249539:2049] [node 3] ICP18 handshake (actor [2:7577788982271216917:2119]) is too old
2025-11-28T14:51:26.542448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788982271216924:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.553033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788984554662781:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50199
2025-11-28T14:51:26.553691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788980850039455:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.560555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788982079425797:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:26.562954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788977784458499:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.563001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788977784458499:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.542123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577788977869542250:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577788982164509628:2121] held: no
2025-11-28T14:51:26.542196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788977869542250:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.542241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:26.542272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788977869542250:2048] [node 1] ICP22 created new session: [3:7577788982164509637:2048]
2025-11-28T14:51:26.542323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788982164509637:2048] [node 1] ICS09 handshake done sender: [3:7577788982164509628:2121] self: [3:7577788982164509635:0] peer: [1:7577788977784458487:0] socket: 98 qp: -1
2025-11-28T14:51:26.542348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788982164509637:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:26.542388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:26.542762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788982164509623:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.542777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577788982164509623:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.576871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788977869542251:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.576937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:26.576999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788977869542251:2049] [node 2] ICP22 created new session: [3:7577788982164509639:2049]
2025-11-28T14:51:26.577038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788982164509639:2049] [node 2] ICS09 handshake done sender: [3:7577788977869542323:2116] self: [3:7577788982164509632:0] peer: [2:7577788977976249604:0] socket: 81 qp: -1
2025-11-28T14:51:26.577068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788982164509639:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:26.577103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:26.563516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788977784458490:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.563560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.563605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788977784458490:2079] [node 5] ICP22 created new session: [1:7577788982079425820:2079]
2025-11-28T14:51:26.563642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788982079425820:2079] [node 5] ICS09 handshake done sender: [1:7577788982079425797:2117] self: [1:7577788982079425796:0] peer: [5:7577788984554662780:0] socket: 85 qp: -1
2025-11-28T14:51:26.563676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788982079425820:2079] [node 5] ICS10 traffic start
2025-11-28T14:51:26.563717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:26.584772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788985145006839:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61947
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.593689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577788977869542251:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:26.594190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788982164509641:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64863
2025-11-28T14:51:26.595076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788982164509623:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:26.564708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788985610067060:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:26.600479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788977784458499:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:26.581665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788982271216915:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:26.565084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788985610067061:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:26.595330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788977869542319:2068] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.595379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:26.595446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788977869542319:2068] [node 4] ICP22 created new session: [3:7577788982164509645:2068]
2025-11-28T14:51:26.595483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788982164509645:2068] [node 4] ICS09 handshake done sender: [3:7577788982164509623:2118] self: [3:7577788982164509622:0] peer: [4:7577788985145006834:0] socket: 101 qp: -1
2025-11-28T14:51:26.595515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788982164509645:2068] [node 4] ICS10 traffic start
2025-11-28T14:51:26.595600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:26.609117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788982164509641:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.608576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788980259695398:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788984554662775:2116] held: no
2025-11-28T14:51:26.608959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788984554662775:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:26.565380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788985610067054:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:26.565564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788981315099679:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788985610067054:2116] held: no
2025-11-28T14:51:26.565613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788981315099679:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.565678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:26.565716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788981315099679:2048] [node 1] ICP22 created new session: [6:7577788985610067062:2048]
2025-11-28T14:51:26.565749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788985610067062:2048] [node 1] ICS09 handshake done sender: [6:7577788985610067054:2116] self: [6:7577788985610067057:0] peer: [1:7577788977784458491:0] socket: 104 qp: -1
2025-11-28T14:51:26.565775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788985610067062:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:26.565799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:26.567377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788981315099680:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788981315099757:2115] held: no
2025-11-28T14:51:26.567716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788981315099757:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:26.567890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788985610067055:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:26.568145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788981315099680:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.568191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:26.568222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788981315099680:2049] [node 2] ICP22 created new session: [6:7577788985610067067:2049]
2025-11-28T14:51:26.568252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788985610067067:2049] [node 2] ICS09 handshake done sender: [6:7577788981315099757:2115] self: [6:7577788985610067056:0] peer: [2:7577788977976249608:0] socket: 82 qp: -1
2025-11-28T14:51:26.568273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788985610067067:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:26.568314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:26.588872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788981315099738:2075] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.604572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:26.612280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788981315099738:2075] [node 5] ICP22 created new session: [6:7577788985610067069:2075]
2025-11-28T14:51:26.612327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788985610067069:2075] [node 5] ICS09 handshake done sender: [6:7577788985610067055:2117] self: [6:7577788985610067058:0] peer: [5:7577788980259695468:0] socket: 105 qp: -1
2025-11-28T14:51:26.612393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788985610067069:2075] [node 5] ICS10 traffic start
2025-11-28T14:51:26.612431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:26.612845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577788981315099738:2075] [node 5] ICP35 event processing took too much time PendingConnection:279(24246996ns) {HandleHandshakeStatus:329(24244841ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(5337ns) SwitchToState:179(530ns) ProcessPendingSessionEvents:495(90331ns) {ForwardSessionEventToSession:612(54760ns) ForwardSessionEventToSession:612(2487ns) ForwardSessionEventToSession:612(4399ns) ForwardSessionEventToSession:612(2116ns) ForwardSessionEventToSession:612(4316ns) ForwardSessionEventToSession:612(2373ns) ForwardSessionEventToSession:612(1824ns) ForwardSessionEventToSession:612(4335ns) ForwardSessionEventToSession:612(4901ns) ForwardSessionEventToSession:612(1957ns)}}}
2025-11-28T14:51:26.613066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788985610067064:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:26.615624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788985610067071:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:26.620943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788985610067071:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.632290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577788977976249538:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.637429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788980259695398:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.637518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:26.637578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788980259695398:2048] [node 1] ICP22 created new session: [5:7577788984554662788:2048]
2025-11-28T14:51:26.637615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788984554662788:2048] [node 1] ICS09 handshake done sender: [5:7577788984554662775:2116] self: [5:7577788984554662780:0] peer: [1:7577788982079425796:0] socket: 120 qp: -1
2025-11-28T14:51:26.637645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788984554662788:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:26.637678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:26.638237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788984554662785:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50199
2025-11-28T14:51:26.654305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788981315099681:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.638304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788984554662786:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50199
2025-11-28T14:51:26.638475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788984554662787:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50199
2025-11-28T14:51:26.656633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788984554662786:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.661038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788980259695400:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.660408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788977784458489:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.660483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:26.660529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788977784458489:2075] [node 4] ICP22 created new session: [1:7577788982079425823:2075]
2025-11-28T14:51:26.660568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788982079425823:2075] [node 4] ICS09 handshake done sender: [1:7577788977784458499:2116] self: [1:7577788977784458498:0] peer: [4:7577788985145006838:0] socket: 115 qp: -1
2025-11-28T14:51:26.660599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788982079425823:2075] [node 4] ICS10 traffic start
2025-11-28T14:51:26.660631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:26.652461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:26.656864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788977976249538:2048] [node 1] ICP22 created new session: [2:7577788982271216931:2048]
2025-11-28T14:51:26.656919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788982271216931:2048] [node 1] ICS09 handshake done sender: [2:7577788982271216915:2117] self: [2:7577788982271216922:0] peer: [1:7577788977784458493:0] socket: 95 qp: -1
2025-11-28T14:51:26.656959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788982271216931:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:26.657000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:26.657234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577788977976249538:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24904173ns) {HandleHandshakeStatus:329(24900827ns) {DropIncomingHandshake:442(799ns) DropOutgoingHandshake:457(427ns) SwitchToState:179(3290ns) ProcessPendingSessionEvents:495(1048ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.661924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788985145006840:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61947
2025-11-28T14:51:26.660332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788985610067064:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.660678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788981315099701:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.664670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788982164509625:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.664704Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577788982164509625:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.665221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788982164509627:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.665233Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577788982164509627:2120] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.666439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788984554662785:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.666720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788980259695399:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.666790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788984554662791:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50199
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.672727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788985145006841:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61947
2025-11-28T14:51:26.673349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788980850039525:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.673372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577788980850039525:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.676571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788982164509625:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:26.676608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788985610067074:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:26.676788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788977976249612:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.680780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788985610067071:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:26.676888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577788977976249612:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.688655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788982164509627:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:26.688505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788985145006832:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:26.692580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788977869542320:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.692673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:26.692719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788977869542320:2069] [node 6] ICP22 created new session: [3:7577788982164509650:2069]
2025-11-28T14:51:26.692766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788982164509650:2069] [node 6] ICS09 handshake done sender: [3:7577788982164509625:2119] self: [3:7577788982164509624:0] peer: [6:7577788985610067072:0] socket: 125 qp: -1
2025-11-28T14:51:26.692796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788982164509650:2069] [node 6] ICS10 traffic start
2025-11-28T14:51:26.692827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:26.693201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788977869542321:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.693251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:26.693287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788977869542321:2073] [node 5] ICP22 created new session: [3:7577788982164509652:2073]
2025-11-28T14:51:26.693328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788982164509652:2073] [node 5] ICS09 handshake done sender: [3:7577788982164509627:2120] self: [3:7577788982164509626:0] peer: [5:7577788984554662790:0] socket: 126 qp: -1
2025-11-28T14:51:26.693346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788982164509652:2073] [node 5] ICS10 traffic start
2025-11-28T14:51:26.693368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:26.689445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788984554662786:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:26.708502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788981315099681:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.708577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:26.708621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788981315099681:2050] [node 3] ICP22 created new session: [6:7577788985610067075:2050]
2025-11-28T14:51:26.708657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788985610067075:2050] [node 3] ICS09 handshake done sender: [6:7577788985610067071:2122] self: [6:7577788985610067072:0] peer: [3:7577788982164509624:0] socket: 132 qp: -1
2025-11-28T14:51:26.708697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788985610067075:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:26.708733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:26.696493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788977976249612:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:26.704792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788977976249602:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.704873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:26.704920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788977976249602:2067] [node 5] ICP22 created new session: [2:7577788982271216934:2067]
2025-11-28T14:51:26.704961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788982271216934:2067] [node 5] ICS09 handshake done sender: [2:7577788977976249612:2114] self: [2:7577788977976249611:0] peer: [5:7577788984554662792:0] socket: 69 qp: -1
2025-11-28T14:51:26.704994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788982271216934:2067] [node 5] ICS10 traffic start
2025-11-28T14:51:26.705026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:26.714009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788980259695400:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.714121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:26.714172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788980259695400:2050] [node 3] ICP22 created new session: [5:7577788984554662794:2050]
2025-11-28T14:51:26.714217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788984554662794:2050] [node 3] ICS09 handshake done sender: [5:7577788984554662786:2120] self: [5:7577788984554662790:0] peer: [3:7577788982164509626:0] socket: 130 qp: -1
2025-11-28T14:51:26.714251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788984554662794:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:26.714289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:26.714551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577788980259695400:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.717270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788984554662793:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50199
2025-11-28T14:51:26.717798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788980259695399:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788984554662785:2119] held: no
2025-11-28T14:51:26.717999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788984554662785:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:26.717108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788985145006842:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61947
2025-11-28T14:51:26.717297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788980850039455:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577788985145006832:2117] held: no
2025-11-28T14:51:26.717382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788980850039455:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.717429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:26.717464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788980850039455:2048] [node 1] ICP22 created new session: [4:7577788985145006844:2048]
2025-11-28T14:51:26.717509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788985145006844:2048] [node 1] ICS09 handshake done sender: [4:7577788985145006832:2117] self: [4:7577788985145006838:0] peer: [1:7577788977784458498:0] socket: 122 qp: -1
2025-11-28T14:51:26.717539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788985145006844:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:26.717566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:26.717969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788985145006830:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:26.718480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788980259695399:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.718726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577788981315099681:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:26.718542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:26.718576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788980259695399:2049] [node 2] ICP22 created new session: [5:7577788984554662796:2049]
2025-11-28T14:51:26.718617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788984554662796:2049] [node 2] ICS09 handshake done sender: [5:7577788984554662785:2119] self: [5:7577788984554662792:0] peer: [2:7577788977976249611:0] socket: 129 qp: -1
2025-11-28T14:51:26.718638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788984554662796:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:26.718660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:26.726476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788985145006839:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.727044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788985610067077:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16142
2025-11-28T14:51:26.736764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788985145006842:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.736827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788980850039457:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.736877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:26.736920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788980850039457:2050] [node 3] ICP22 created new session: [4:7577788985145006846:2050]
2025-11-28T14:51:26.736965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788985145006846:2050] [node 3] ICS09 handshake done sender: [4:7577788985145006830:2116] self: [4:7577788985145006834:0] peer: [3:7577788982164509622:0] socket: 107 qp: -1
2025-11-28T14:51:26.736998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788985145006846:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:26.737030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:26.738032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788980850039525:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:26.738576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788980850039478:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.738611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:26.738650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788980850039478:2067] [node 6] ICP22 created new session: [4:7577788985145006849:2067]
2025-11-28T14:51:26.738686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788985145006849:2067] [node 6] ICS09 handshake done sender: [4:7577788980850039525:2112] self: [4:7577788980850039524:0] peer: [6:7577788985610067073:0] socket: 97 qp: -1
2025-11-28T14:51:26.738715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788985145006849:2067] [node 6] ICS10 traffic start
2025-11-28T14:51:26.738741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:26.738938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577788980850039478:2067] [node 6] ICP18 handshake (actor [4:7577788985145006842:2121]) is too old
2025-11-28T14:51:26.739236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788980850039456:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:26.742809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788981315099701:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64497) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788985610067064:2121] held: no
2025-11-28T14:51:26.742958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788985610067064:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:26.740504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788977976249614:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:26.740540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577788977976249614:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:26.742558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788977976249614:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:26.743232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788977976249603:2069] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:26.743291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:26.743329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788977976249603:2069] [node 4] ICP22 created new session: [2:7577788982271216937:2069]
2025-11-28T14:51:26.743363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788982271216937:2069] [node 4] ICS09 handshake done sender: [2:7577788977976249614:2115] self: [2:7577788977976249613:0] peer: [4:7577788985145006851:0] socket: 102 qp: -1
2025-11-28T14:51:26.743391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788982271216937:2069] [node 4] ICS10 traffic start
2025-11-28T14:51:26.743418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:26.744113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788981315099701:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.744164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:26.744198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788981315099701:2067] [node 4] ICP22 created new session: [6:7577788985610067078:2067]
2025-11-28T14:51:26.744233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788985610067078:2067] [node 4] ICS09 handshake done sender: [6:7577788985610067064:2121] self: [6:7577788985610067073:0] peer: [4:7577788980850039524:0] socket: 66 qp: -1
2025-11-28T14:51:26.744266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788985610067078:2067] [node 4] ICS10 traffic start
2025-11-28T14:51:26.744290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:26.745153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788985145006852:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61947
2025-11-28T14:51:26.746347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788980850039456:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577788985145006839:2118] held: no
2025-11-28T14:51:26.746440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788985145006839:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:26.756546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788980850039456:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:26.756636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:26.756697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788980850039456:2049] [node 2] ICP22 created new session: [4:7577788985145006853:2049]
2025-11-28T14:51:26.756738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788985145006853:2049] [node 2] ICS09 handshake done sender: [4:7577788985145006839:2118] self: [4:7577788985145006851:0] peer: [2:7577788977976249613:0] socket: 124 qp: -1
2025-11-28T14:51:26.756768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788985145006853:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:26.756810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:26.793828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:26.794059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577788982271216931:2048] [node 1] ICS01 socket: 95 reason# EndOfStream
2025-11-28T14:51:26.794091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577788977976249538:2048] [node 1] ICP30 unregister session Session# [2:7577788982271216931:2048] VirtualId# [2:7577788982271216922:0]
2025-11-28T14:51:26.794120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577788982271216931:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.794195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:26.794227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:26.795636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:26.795857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788982164509637:2048] [node 1] ICS01 socket: 98 reason# EndOfStream
2025-11-28T14:51:26.795895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788977869542250:2048] [node 1] ICP30 unregister session Session# [3:7577788982164509637:2048] VirtualId# [3:7577788982164509635:0]
2025-11-28T14:51:26.795933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788982164509637:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.795996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:26.796026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:26.808502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:26.808742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788984554662788:2048] [node 1] ICS01 socket: 120 reason# EndOfStream
2025-11-28T14:51:26.808776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788980259695398:2048] [node 1] ICP30 unregister session Session# [5:7577788984554662788:2048] VirtualId# [5:7577788984554662780:0]
2025-11-28T14:51:26.808802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788984554662788:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.808882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:26.808924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:26.809002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:26.809176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788985145006844:2048] [node 1] ICS01 socket: 122 reason# EndOfStream
2025-11-28T14:51:26.809199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788980850039455:2048] [node 1] ICP30 unregister session Session# [4:7577788985145006844:2048] VirtualId# [4:7577788985145006838:0]
2025-11-28T14:51:26.809217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788985145006844:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.809251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:26.809285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:26.812431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:26.812628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788985610067062:2048] [node 1] ICS01 socket: 104 reason# EndOfStream
2025-11-28T14:51:26.812657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788981315099679:2048] [node 1] ICP30 unregister session Session# [6:7577788985610067062:2048] VirtualId# [6:7577788985610067057:0]
2025-11-28T14:51:26.812686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788985610067062:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.812756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:26.812790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:26.841764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:26.841997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788982164509639:2049] [node 2] ICS01 socket: 81 reason# EndOfStream
2025-11-28T14:51:26.842036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788977869542251:2049] [node 2] ICP30 unregister session Session# [3:7577788982164509639:2049] VirtualId# [3:7577788982164509632:0]
2025-11-28T14:51:26.842064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788982164509639:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.842140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:26.842175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:26.852533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:26.852752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788984554662796:2049] [node 2] ICS01 socket: 129 reason# EndOfStream
2025-11-28T14:51:26.852786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788980259695399:2049] [node 2] ICP30 unregister session Session# [5:7577788984554662796:2049] VirtualId# [5:7577788984554662792:0]
2025-11-28T14:51:26.852812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788984554662796:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.852893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:26.852931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:26.853028Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:26.853220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788985145006853:2049] [node 2] ICS01 socket: 124 reason# EndOfStream
2025-11-28T14:51:26.853251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788980850039456:2049] [node 2] ICP30 unregister session Session# [4:7577788985145006853:2049] VirtualId# [4:7577788985145006851:0]
2025-11-28T14:51:26.853272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788985145006853:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.853309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:26.853339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:26.860533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:26.860897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788985610067067:2049] [node 2] ICS01 socket: 82 reason# EndOfStream
2025-11-28T14:51:26.860957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788981315099680:2049] [node 2] ICP30 unregister session Session# [6:7577788985610067067:2049] VirtualId# [6:7577788985610067056:0]
2025-11-28T14:51:26.860984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788985610067067:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.861072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:26.861108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:26.892590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:26.892820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788984554662794:2050] [node 3] ICS01 socket: 130 reason# EndOfStream
2025-11-28T14:51:26.892861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788980259695400:2050] [node 3] ICP30 unregister session Session# [5:7577788984554662794:2050] VirtualId# [5:7577788984554662790:0]
2025-11-28T14:51:26.892891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788984554662794:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.892970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:26.893004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:26.893086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:26.893272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577788985145006846:2050] [node 3] ICS07 socket disconnect 107 reason# EndOfStream
2025-11-28T14:51:26.893293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788985145006846:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.893333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:26.893364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577788985145006846:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:26.899821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788980850039457:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24573) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:26.899868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788985145006846:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:26.899893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788980850039457:2050] [node 3] ICP30 unregister session Session# [4:7577788985145006846:2050] VirtualId# [4:7577788985145006834:0]
2025-11-28T14:51:26.899922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:26.899947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577788980850039457:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24573) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:26.899965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24573) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:26.898156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:26.899322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788985610067075:2050] [node 3] ICS01 socket: 132 reason# EndOfStream
2025-11-28T14:51:26.899359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788981315099681:2050] [node 3] ICP30 unregister session Session# [6:7577788985610067075:2050] VirtualId# [6:7577788985610067072:0]
2025-11-28T14:51:26.899384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788985610067075:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.899449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:26.899494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:26.938376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:26.938605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788984554662782:2065] [node 4] ICS01 socket: 78 reason# EndOfStream
2025-11-28T14:51:26.938638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788980259695435:2065] [node 4] ICP30 unregister session Session# [5:7577788984554662782:2065] VirtualId# [5:7577788984554662773:0]
2025-11-28T14:51:26.938664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788984554662782:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.938722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:26.938767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:26.940534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:26.940753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788985610067078:2067] [node 4] ICS01 socket: 66 reason# EndOfStream
2025-11-28T14:51:26.940780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788981315099701:2067] [node 4] ICP30 unregister session Session# [6:7577788985610067078:2067] VirtualId# [6:7577788985610067073:0]
2025-11-28T14:51:26.940800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788985610067078:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.940852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:26.940884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:26.988522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:26.988752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788985610067069:2075] [node 5] ICS01 socket: 105 reason# EndOfStream
2025-11-28T14:51:26.988790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788981315099738:2075] [node 5] ICP30 unregister session Session# [6:7577788985610067069:2075] VirtualId# [6:7577788985610067058:0]
2025-11-28T14:51:26.988822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788985610067069:2075] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:26.988934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:26.988978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:27.140087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788987954409535:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:27.140131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788987954409536:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:27.220948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788986948939039:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:27.221004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788986948939040:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:27.262205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788986263078008:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:27.262241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788986263078009:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:27.308512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788987297587246:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:27.308546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788987297587247:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:27.308568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788987297587248:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:27.360688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788986682022972:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:27.360738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788986682022973:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:27.360762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788986682022974:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:27.545017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788986948939057:2063] [node 6] ICP01 ready to work
2025-11-28T14:51:27.649034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788987954409568:2061] [node 5] ICP01 ready to work
2025-11-28T14:51:27.649348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788987954409569:2064] [node 6] ICP01 ready to work
2025-11-28T14:51:27.700485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788986948939069:2062] [node 5] ICP01 ready to work
2025-11-28T14:51:27.700563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788986263078040:2068] [node 5] ICP01 ready to work
2025-11-28T14:51:27.728621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788987954409574:2060] [node 4] ICP01 ready to work
2025-11-28T14:51:27.717123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788986263078044:2070] [node 4] ICP01 ready to work
2025-11-28T14:51:27.754072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788987954409568:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25190627ns) {Configure:104(25185663ns) {ProcessConfigured:135(85407ns) {StartInitialHandshake:157(82201ns) {DropHandshakes:470(4983ns) {DropIncomingHandshake:442(606ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(1580ns) PrepareNewSessionHandshake:477(2698ns) {SwitchToState:179(1087ns)}}}}}
2025-11-28T14:51:27.805630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788987297587265:2060] [node 6] ICP01 ready to work
2025-11-28T14:51:27.801154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788986263078053:2069] [node 6] ICP01 ready to work
2025-11-28T14:51:27.947709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788986948939039:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(402398601ns) {Configure:104(402394170ns) {ProcessConfigured:135(69935ns) {StartInitialHandshake:157(67310ns) {DropHandshakes:470(2195ns) {DropIncomingHandshake:442(564ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(674ns) PrepareNewSessionHandshake:477(4597ns) {SwitchToState:179(791ns)}}}}}
2025-11-28T14:51:27.949251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788986948939108:2068] [node 4] ICP01 ready to work
2025-11-28T14:51:27.977499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788987954409574:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(221017028ns) {Configure:104(221014503ns) {ProcessConfigured:135(79975ns) {StartInitialHandshake:157(76724ns) {DropHandshakes:470(8023ns) {DropIncomingHandshake:442(954ns) DropOutgoingHandshake:457(392ns)} GenerateSessionVirtualId:988(849ns) PrepareNewSessionHandshake:477(1554ns) {SwitchToState:179(706ns)}}}}}
2025-11-28T14:51:27.969343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788986948939069:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19801862ns) {Configure:104(19796053ns) {ProcessConfigured:135(71862ns) {StartInitialHandshake:157(69877ns) {DropHandshakes:470(2881ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(360ns)} GenerateSessionVirtualId:988(846ns) PrepareNewSessionHandshake:477(1335ns) {SwitchToState:179(557ns)}}}}}
2025-11-28T14:51:28.089739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788986263078040:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(249067341ns) {Configure:104(249060549ns) {ProcessConfigured:135(89268ns) {StartInitialHandshake:157(87314ns) {DropHandshakes:470(2194ns) {DropIncomingHandshake:442(431ns) DropOutgoingHandshake:457(433ns)} GenerateSessionVirtualId:988(455ns) PrepareNewSessionHandshake:477(3990ns) {SwitchToState:179(723ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.096952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788992249376908:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56916
2025-11-28T14:51:28.097029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788992249376909:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56916
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:27.576624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788988887303166:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:27.576672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788988887303167:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:27.576697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788988887303168:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:27.962953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788987297587248:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(401125273ns) {Configure:104(401120707ns) {ProcessConfigured:135(74647ns) {StartInitialHandshake:157(72176ns) {DropHandshakes:470(2572ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(627ns)} GenerateSessionVirtualId:988(843ns) PrepareNewSessionHandshake:477(1847ns) {SwitchToState:179(1006ns)}}}}}
2025-11-28T14:51:28.133170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788987954409535:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(483530928ns) {Configure:104(483527385ns) {ProcessConfigured:135(69010ns) {StartInitialHandshake:157(65152ns) {DropHandshakes:470(1644ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(818ns) PrepareNewSessionHandshake:477(1870ns) {SwitchToState:179(1046ns)}}}}}
2025-11-28T14:51:28.129148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788986948939040:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(428375950ns) {Configure:104(428370473ns) {ProcessConfigured:135(62161ns) {StartInitialHandshake:157(60271ns) {DropHandshakes:470(4699ns) {DropIncomingHandshake:442(689ns) DropOutgoingHandshake:457(515ns)} GenerateSessionVirtualId:988(759ns) PrepareNewSessionHandshake:477(1181ns) {SwitchToState:179(372ns)}}}}}
2025-11-28T14:51:28.146365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788992249376909:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.106429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788991592554616:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9004
2025-11-28T14:51:28.128499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788991592554618:2069] [node 5] ICP01 ready to work
2025-11-28T14:51:28.169348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788993182270526:2061] [node 4] ICP01 ready to work
2025-11-28T14:51:28.153135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788986263078008:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(435745480ns) {Configure:104(435740434ns) {ProcessConfigured:135(77322ns) {StartInitialHandshake:157(76011ns) {DropHandshakes:470(1791ns) {DropIncomingHandshake:442(356ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(706ns) PrepareNewSessionHandshake:477(1505ns) {SwitchToState:179(763ns)}}}}}
2025-11-28T14:51:28.196554Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788988887303168:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(593666540ns) {Configure:104(593663025ns) {ProcessConfigured:135(29685664ns) {StartInitialHandshake:157(29683500ns) {DropHandshakes:470(2011ns) {DropIncomingHandshake:442(407ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(4040ns) {SwitchToState:179(2120ns)}}}}}
2025-11-28T14:51:28.197072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788993182270528:2069] [node 5] ICP01 ready to work
2025-11-28T14:51:28.184628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788990976990331:2060] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.212498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788992249376908:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.185091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788990976990332:2065] [node 4] ICP01 ready to work
2025-11-28T14:51:28.205107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788990976990332:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17038134ns) {Configure:104(17035662ns) {ProcessConfigured:135(48454ns) {StartInitialHandshake:157(47095ns) {DropHandshakes:470(1174ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(162ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:477(687ns) {SwitchToState:179(271ns)}}}}}
2025-11-28T14:51:28.206436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788990976990333:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-28T14:51:28.206619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788990976990334:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-28T14:51:28.206712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788990976990335:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-28T14:51:28.217078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788990558045383:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47393
2025-11-28T14:51:28.240935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788986682022973:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(691518890ns) {Configure:104(691513448ns) {ProcessConfigured:135(51720ns) {StartInitialHandshake:157(49838ns) {DropHandshakes:470(1737ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(295ns)} GenerateSessionVirtualId:988(470ns) PrepareNewSessionHandshake:477(1718ns) {SwitchToState:179(854ns)}}}}}
2025-11-28T14:51:28.241086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788990976990334:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.241151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788990976990335:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.241220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788990976990333:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.225327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788993182270530:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-11-28T14:51:28.276489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788987954409587:2093] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.276529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788987954409587:2093] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.276715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788987954409583:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.276729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788987954409583:2091] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.244424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788991592554616:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.272995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788991592554621:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9004
2025-11-28T14:51:28.273184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788987297587248:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.244210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788986682022973:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.217189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788990558045384:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47393
2025-11-28T14:51:28.217268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788990558045385:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47393
2025-11-28T14:51:28.228763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788990558045383:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.236526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788986263078008:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.236858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788990558045384:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.225481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788993182270531:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-11-28T14:51:28.225569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788993182270532:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-11-28T14:51:28.251306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788993182270529:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-11-28T14:51:28.283987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788993182270531:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.244265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788986682022974:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.292396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788986948939110:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.280785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788987954409583:2091] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:28.320496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788993182270532:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.244318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788986682022972:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.332733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788990558045371:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.337034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788993182270530:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.337151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788993182270542:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-11-28T14:51:28.337558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788988887303167:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.337605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788988887303168:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.337880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788993182270526:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.392406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788987297587267:2066] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.392464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577788987297587267:2066] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.395268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788990976990339:2114] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.292445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577788986948939110:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.364448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577788990558045371:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.367038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577788986263078044:2070] [node 4] ICP35 event processing took too much time PendingConnection:279(28850884ns) {IncomingHandshake:261(28849956ns) {DropIncomingHandshake:442(1231ns) PrepareNewSessionHandshake:477(2955ns) {SwitchToState:179(2315ns)} GenerateSessionVirtualId:988(665ns) IssueIncomingHandshakeReply:193(7004ns)}}
2025-11-28T14:51:28.398149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788990558045375:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.398170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577788990558045375:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.398697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788990558045385:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.398995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788990558045388:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47393
2025-11-28T14:51:28.400108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788990558045373:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.400138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577788990558045373:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.408600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788990558045371:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:28.420920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788990558045375:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:28.432490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788987297587267:2066] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:28.432883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788987297587265:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.432944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:28.432991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788987297587265:2060] [node 6] ICP22 created new session: [4:7577788991592554624:2060]
2025-11-28T14:51:28.433030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788991592554624:2060] [node 6] ICS09 handshake done sender: [4:7577788987297587267:2066] self: [4:7577788987297587266:0] peer: [6:7577788993182270547:0] socket: 75 qp: -1
2025-11-28T14:51:28.433071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788991592554624:2060] [node 6] ICS10 traffic start
2025-11-28T14:51:28.433108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:28.401014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788993182270548:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.465267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788991592554627:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9004
2025-11-28T14:51:28.465437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788991592554628:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9004
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.364460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788986948939110:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:28.436520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788987954409568:2061] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.416917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788993182270549:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-11-28T14:51:28.388525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788986948939107:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.388563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577788986948939107:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.396277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788986948939069:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.396338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:28.396399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788986948939069:2062] [node 5] ICP22 created new session: [2:7577788991243906415:2062]
2025-11-28T14:51:28.396441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788991243906415:2062] [node 5] ICS09 handshake done sender: [2:7577788986948939110:2112] self: [2:7577788986948939109:0] peer: [5:7577788990976990349:0] socket: 79 qp: -1
2025-11-28T14:51:28.396473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788991243906415:2062] [node 5] ICS10 traffic start
2025-11-28T14:51:28.396522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:28.408645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788986948939107:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:28.547345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788986948939057:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.547388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:28.547431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788986948939057:2063] [node 6] ICP22 created new session: [2:7577788991243906420:2063]
2025-11-28T14:51:28.547478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788991243906420:2063] [node 6] ICS09 handshake done sender: [2:7577788986948939107:2111] self: [2:7577788986948939106:0] peer: [6:7577788993182270545:0] socket: 78 qp: -1
2025-11-28T14:51:28.547508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788991243906420:2063] [node 6] ICS10 traffic start
2025-11-28T14:51:28.547544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:28.548411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788991243906419:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13891
2025-11-28T14:51:28.436600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:28.436644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788987954409568:2061] [node 5] ICP22 created new session: [1:7577788992249376916:2061]
2025-11-28T14:51:28.436698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788992249376916:2061] [node 5] ICS09 handshake done sender: [1:7577788987954409583:2091] self: [1:7577788987954409582:0] peer: [5:7577788990976990351:0] socket: 72 qp: -1
2025-11-28T14:51:28.436744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788992249376916:2061] [node 5] ICS10 traffic start
2025-11-28T14:51:28.436771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:28.468638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788987954409587:2093] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:28.494168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788992249376919:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56916
2025-11-28T14:51:28.536433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788991592554628:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.417045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788993182270529:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.569332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788987954409536:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.572417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788991592554627:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.588583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788987297587246:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.588688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788987297587247:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.608886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788991592554631:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9004
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.584929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788991243906419:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.624755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788987954409608:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.624790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788987954409608:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.628547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788991592554621:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.592803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788986263078040:2068] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.584780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788993182270531:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:28.618582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788988887303166:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.618660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788993182270550:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-11-28T14:51:28.646036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788991592554616:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:28.646229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788987297587248:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.646293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:28.646333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788987297587248:2050] [node 3] ICP22 created new session: [4:7577788991592554633:2050]
2025-11-28T14:51:28.646369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788991592554633:2050] [node 3] ICS09 handshake done sender: [4:7577788991592554616:2114] self: [4:7577788991592554622:0] peer: [3:7577788990558045372:0] socket: 86 qp: -1
2025-11-28T14:51:28.646401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788991592554633:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:28.646430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:28.617321Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788990976990341:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:28.618329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788990976990352:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.654597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788991592554636:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9004
2025-11-28T14:51:28.655029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788991592554627:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:28.655195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788987297587247:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.655261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:28.655314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788987297587247:2049] [node 2] ICP22 created new session: [4:7577788991592554637:2049]
2025-11-28T14:51:28.655352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788991592554637:2049] [node 2] ICS09 handshake done sender: [4:7577788991592554627:2117] self: [4:7577788991592554630:0] peer: [2:7577788991243906407:0] socket: 122 qp: -1
2025-11-28T14:51:28.655379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788991592554637:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:28.655435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:28.592916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:28.624931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788991243906408:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.636409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.624957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577788991243906408:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.592988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788986263078040:2068] [node 5] ICP22 created new session: [3:7577788990558045395:2068]
2025-11-28T14:51:28.593038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788990558045395:2068] [node 5] ICS09 handshake done sender: [3:7577788990558045371:2110] self: [3:7577788990558045370:0] peer: [5:7577788990976990350:0] socket: 82 qp: -1
2025-11-28T14:51:28.593079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788990558045395:2068] [node 5] ICS10 traffic start
2025-11-28T14:51:28.593121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:28.593532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788986263078053:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.593570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:28.593600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788986263078053:2069] [node 6] ICP22 created new session: [3:7577788990558045397:2069]
2025-11-28T14:51:28.593639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788990558045397:2069] [node 6] ICS09 handshake done sender: [3:7577788990558045375:2112] self: [3:7577788990558045374:0] peer: [6:7577788993182270546:0] socket: 84 qp: -1
2025-11-28T14:51:28.593661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788990558045397:2069] [node 6] ICS10 traffic start
2025-11-28T14:51:28.593694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:28.612809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788990558045393:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47393
2025-11-28T14:51:28.641905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788990558045394:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47393
2025-11-28T14:51:28.642154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788990558045373:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:28.680942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577788986263078044:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.681018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788990558045394:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.681086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788990558045383:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:28.660413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788987954409536:2049] [node 3] ICP22 created new session: [1:7577788992249376920:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.713594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577788991592554639:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9004
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.716560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788990976990353:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-28T14:51:28.648527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788991243906408:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:28.681265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788986948939108:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.681324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:28.681365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788986948939108:2068] [node 4] ICP22 created new session: [2:7577788991243906425:2068]
2025-11-28T14:51:28.681402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788991243906425:2068] [node 4] ICS09 handshake done sender: [2:7577788991243906408:2113] self: [2:7577788991243906407:0] peer: [4:7577788991592554630:0] socket: 114 qp: -1
2025-11-28T14:51:28.681435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788991243906425:2068] [node 4] ICS10 traffic start
2025-11-28T14:51:28.681468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:28.682588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788991243906427:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13891
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.744562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788991243906428:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13891
2025-11-28T14:51:28.717627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:28.748483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788986263078044:2070] [node 4] ICP22 created new session: [3:7577788990558045401:2070]
2025-11-28T14:51:28.748551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788990558045401:2070] [node 4] ICS09 handshake done sender: [3:7577788990558045373:2111] self: [3:7577788990558045372:0] peer: [4:7577788991592554622:0] socket: 83 qp: -1
2025-11-28T14:51:28.748590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788990558045401:2070] [node 4] ICS10 traffic start
2025-11-28T14:51:28.748642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:28.749089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577788986263078044:2070] [node 4] ICP35 event processing took too much time PendingConnection:279(67965356ns) {HandleHandshakeStatus:329(67962988ns) {DropIncomingHandshake:442(15241ns) DropOutgoingHandshake:457(412ns) SwitchToState:179(491ns) ProcessPendingSessionEvents:495(87553ns) {ForwardSessionEventToSession:612(46373ns) ForwardSessionEventToSession:612(3642ns) ForwardSessionEventToSession:612(7104ns) ForwardSessionEventToSession:612(2119ns) ForwardSessionEventToSession:612(2226ns) ForwardSessionEventToSession:612(4622ns) ForwardSessionEventToSession:612(4462ns) ForwardSessionEventToSession:612(4157ns) ForwardSessionEventToSession:612(2859ns) ForwardSessionEventToSession:612(2305ns)}}}
2025-11-28T14:51:28.749158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788986263078008:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.749193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:28.749228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788986263078008:2048] [node 1] ICP22 created new session: [3:7577788990558045403:2048]
2025-11-28T14:51:28.749268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788990558045403:2048] [node 1] ICS09 handshake done sender: [3:7577788990558045383:2115] self: [3:7577788990558045386:0] peer: [1:7577788987954409586:0] socket: 89 qp: -1
2025-11-28T14:51:28.749292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788990558045403:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:28.749332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:28.684883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788992249376920:2049] [node 3] ICS09 handshake done sender: [1:7577788987954409587:2093] self: [1:7577788987954409586:0] peer: [3:7577788990558045386:0] socket: 74 qp: -1
2025-11-28T14:51:28.717051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577788987297587247:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(61628346ns) {HandleHandshakeStatus:329(61625860ns) {DropIncomingHandshake:442(867ns) DropOutgoingHandshake:457(4331ns) SwitchToState:179(410ns) ProcessPendingSessionEvents:495(12304932ns) {ForwardSessionEventToSession:612(12257434ns) ForwardSessionEventToSession:612(4837ns) ForwardSessionEventToSession:612(2852ns) ForwardSessionEventToSession:612(5440ns) ForwardSessionEventToSession:612(2023ns) ForwardSessionEventToSession:612(7364ns) ForwardSessionEventToSession:612(2555ns) ForwardSessionEventToSession:612(1880ns) ForwardSessionEventToSession:612(7329ns) ForwardSessionEventToSession:612(1875ns)}}}
2025-11-28T14:51:28.757362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788986263078009:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.692553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788992249376920:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.708297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:28.713248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577788992249376922:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56916
2025-11-28T14:51:28.717286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788987954409608:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:28.744740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577788987954409536:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(187998410ns) {HandleHandshakeStatus:329(187996014ns) {DropIncomingHandshake:442(1740ns) DropOutgoingHandshake:457(443ns) SwitchToState:179(691ns) ProcessPendingSessionEvents:495(7327542ns) {ForwardSessionEventToSession:612(7272667ns) ForwardSessionEventToSession:612(6184ns) ForwardSessionEventToSession:612(2803ns) ForwardSessionEventToSession:612(8369ns) ForwardSessionEventToSession:612(2297ns) ForwardSessionEventToSession:612(4689ns) ForwardSessionEventToSession:612(1527ns) ForwardSessionEventToSession:612(2665ns) ForwardSessionEventToSession:612(6301ns) ForwardSessionEventToSession:612(2160ns)}}}
2025-11-28T14:51:28.744877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788987954409574:2060] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.744937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:28.744979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788987954409574:2060] [node 4] ICP22 created new session: [1:7577788992249376925:2060]
2025-11-28T14:51:28.745017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788992249376925:2060] [node 4] ICS09 handshake done sender: [1:7577788987954409608:2113] self: [1:7577788987954409607:0] peer: [4:7577788991592554629:0] socket: 130 qp: -1
2025-11-28T14:51:28.745046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788992249376925:2060] [node 4] ICS10 traffic start
2025-11-28T14:51:28.745104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.781626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788990976990354:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-28T14:51:28.781799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788990976990355:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-28T14:51:28.782642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788986682022974:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8633) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788990976990335:2112] held: no
2025-11-28T14:51:28.782843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788990976990333:2110] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:28.782979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788990976990334:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:28.783105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788990976990335:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:28.783309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788986682022972:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.783366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:28.783402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788986682022972:2048] [node 1] ICP22 created new session: [5:7577788990976990357:2048]
2025-11-28T14:51:28.783441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788990976990357:2048] [node 1] ICS09 handshake done sender: [5:7577788990976990333:2110] self: [5:7577788990976990351:0] peer: [1:7577788987954409582:0] socket: 93 qp: -1
2025-11-28T14:51:28.783466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788990976990357:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:28.783493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:28.784085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788986682022973:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.784143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:28.784188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788986682022973:2049] [node 2] ICP22 created new session: [5:7577788990976990359:2049]
2025-11-28T14:51:28.784216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788990976990359:2049] [node 2] ICS09 handshake done sender: [5:7577788990976990334:2111] self: [5:7577788990976990349:0] peer: [2:7577788986948939109:0] socket: 94 qp: -1
2025-11-28T14:51:28.784254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788990976990359:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:28.784275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:28.789608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788986682022974:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.789658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:28.789691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788986682022974:2050] [node 3] ICP22 created new session: [5:7577788990976990361:2050]
2025-11-28T14:51:28.789730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788990976990361:2050] [node 3] ICS09 handshake done sender: [5:7577788990976990335:2112] self: [5:7577788990976990350:0] peer: [3:7577788990558045370:0] socket: 95 qp: -1
2025-11-28T14:51:28.789752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788990976990361:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:28.789790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:28.792550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788991243906427:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.790164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788990976990356:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-28T14:51:28.740451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788991592554628:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:28.765268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788987297587246:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.765332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:28.765378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788987297587246:2048] [node 1] ICP22 created new session: [4:7577788991592554640:2048]
2025-11-28T14:51:28.765427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788991592554640:2048] [node 1] ICS09 handshake done sender: [4:7577788991592554628:2118] self: [4:7577788991592554629:0] peer: [1:7577788987954409607:0] socket: 131 qp: -1
2025-11-28T14:51:28.765461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788991592554640:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:28.765498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.793275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788991243906413:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.793312Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577788991243906413:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.799417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577788990558045406:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47393
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.808582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788991243906413:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:28.798214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577788991592554620:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.816927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577788990558045394:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:28.797054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788990976990356:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.831351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788986263078009:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.852558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:28.860479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788986263078009:2049] [node 2] ICP22 created new session: [3:7577788990558045407:2049]
2025-11-28T14:51:28.827418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788986948939040:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.798238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577788991592554620:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.797160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788990976990332:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.861437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577788991243906428:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.800098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577788991592554620:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:28.808966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577788990976990365:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-28T14:51:28.821607Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788990976990356:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:28.821765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788990976990332:2065] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11299) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577788990976990356:2122] held: no
2025-11-28T14:51:28.821862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788990976990332:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.821917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:28.821961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788990976990332:2065] [node 4] ICP22 created new session: [5:7577788990976990366:2065]
2025-11-28T14:51:28.822002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788990976990366:2065] [node 4] ICS09 handshake done sender: [5:7577788990976990356:2122] self: [5:7577788990976990364:0] peer: [4:7577788991592554619:0] socket: 81 qp: -1
2025-11-28T14:51:28.822033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788990976990366:2065] [node 4] ICS10 traffic start
2025-11-28T14:51:28.822065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:28.826606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577788990976990337:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:28.861738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:28.861800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788986948939040:2049] [node 3] ICP22 created new session: [2:7577788991243906431:2049]
2025-11-28T14:51:28.861841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788991243906431:2049] [node 3] ICS09 handshake done sender: [2:7577788991243906413:2114] self: [2:7577788991243906412:0] peer: [3:7577788990558045405:0] socket: 133 qp: -1
2025-11-28T14:51:28.861884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788991243906431:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:28.861921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:28.921018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577788986948939039:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:28.864462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577788990558045407:2049] [node 2] ICS09 handshake done sender: [3:7577788990558045394:2120] self: [3:7577788990558045405:0] peer: [2:7577788991243906412:0] socket: 135 qp: -1
2025-11-28T14:51:28.804808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577788991592554618:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.829598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:28.851653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788991592554618:2069] [node 5] ICP22 created new session: [4:7577788991592554643:2069]
2025-11-28T14:51:28.888412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577788991592554643:2069] [node 5] ICS09 handshake done sender: [4:7577788991592554620:2115] self: [4:7577788991592554619:0] peer: [5:7577788990976990364:0] socket: 87 qp: -1
2025-11-28T14:51:28.888467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577788991592554643:2069] [node 5] ICS10 traffic start
2025-11-28T14:51:28.912463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:28.924789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788992249376911:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:28.924824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788992249376911:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:28.928750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577788991592554618:2069] [node 5] ICP35 event processing took too much time PendingConnection:279(123862691ns) {HandleHandshakeStatus:329(123859917ns) {DropIncomingHandshake:442(13632ns) DropOutgoingHandshake:457(805ns) SwitchToState:179(333ns) ProcessPendingSessionEvents:495(7062938ns) {ForwardSessionEventToSession:612(7011697ns) ForwardSessionEventToSession:612(4496ns) ForwardSessionEventToSession:612(3174ns) ForwardSessionEventToSession:612(5629ns) ForwardSessionEventToSession:612(5923ns) ForwardSessionEventToSession:612(11824ns) ForwardSessionEventToSession:612(2655ns) ForwardSessionEventToSession:612(2342ns) ForwardSessionEventToSession:612(4259ns) ForwardSessionEventToSession:612(1957ns)}}}
2025-11-28T14:51:28.864505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577788990558045407:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:28.864543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:28.940623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788992249376911:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:28.864905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577788986263078009:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(33391437ns) {HandleHandshakeStatus:329(33388650ns) {DropIncomingHandshake:442(630ns) DropOutgoingHandshake:457(6046ns) SwitchToState:179(1232ns) ProcessPendingSessionEvents:495(94442ns) {ForwardSessionEventToSession:612(56585ns) ForwardSessionEventToSession:612(4916ns) ForwardSessionEventToSession:612(2267ns) ForwardSessionEventToSession:612(2191ns) ForwardSessionEventToSession:612(4292ns) ForwardSessionEventToSession:612(1975ns) ForwardSessionEventToSession:612(4898ns) ForwardSessionEventToSession:612(2592ns) ForwardSessionEventToSession:612(3698ns)}}}
2025-11-28T14:51:28.960712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577788991243906434:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13891
2025-11-28T14:51:28.961055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577788986948939040:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(112560423ns) {HandleHandshakeStatus:329(101940693ns) {DropIncomingHandshake:442(22186ns) DropOutgoingHandshake:457(806ns) SwitchToState:179(811ns) ProcessPendingSessionEvents:495(67283167ns) {ForwardSessionEventToSession:612(29503380ns) ForwardSessionEventToSession:612(17851ns) ForwardSessionEventToSession:612(20039426ns) ForwardSessionEventToSession:612(4689ns) ForwardSessionEventToSession:612(3220ns) ForwardSessionEventToSession:612(8774730ns) ForwardSessionEventToSession:612(1139888ns) ForwardSessionEventToSession:612(7772588ns)}}}
2025-11-28T14:51:28.968865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788987954409535:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:28.971338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577788991243906428:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:28.982561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788993182270534:2115] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:28.968947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:28.971517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577788986948939039:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.983289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788988887303167:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:28.983341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:28.983388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788988887303167:2049] [node 2] ICP22 created new session: [6:7577788993182270552:2049]
2025-11-28T14:51:28.983433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788993182270552:2049] [node 2] ICS09 handshake done sender: [6:7577788993182270531:2113] self: [6:7577788993182270545:0] peer: [2:7577788986948939106:0] socket: 98 qp: -1
2025-11-28T14:51:28.983462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788993182270552:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:28.983491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:28.983975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788993182270532:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:28.971570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:28.971608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788986948939039:2048] [node 1] ICP22 created new session: [2:7577788991243906435:2048]
2025-11-28T14:51:28.971641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577788991243906435:2048] [node 1] ICS09 handshake done sender: [2:7577788991243906428:2117] self: [2:7577788991243906433:0] peer: [1:7577788992249376910:0] socket: 144 qp: -1
2025-11-28T14:51:28.971671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577788991243906435:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:28.971699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:28.968995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788987954409535:2048] [node 2] ICP22 created new session: [1:7577788992249376928:2048]
2025-11-28T14:51:28.969035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788992249376928:2048] [node 2] ICS09 handshake done sender: [1:7577788992249376911:2116] self: [1:7577788992249376910:0] peer: [2:7577788991243906433:0] socket: 142 qp: -1
2025-11-28T14:51:28.969069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788992249376928:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:28.969106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:29.012439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788992249376919:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:29.068619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577788987954409585:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:29.068652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577788987954409585:2092] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:29.070035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577788987954409585:2092] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:29.070261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788987954409569:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:29.070312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:29.070352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788987954409569:2064] [node 6] ICP22 created new session: [1:7577788996544344228:2064]
2025-11-28T14:51:29.070388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577788996544344228:2064] [node 6] ICS09 handshake done sender: [1:7577788987954409585:2092] self: [1:7577788987954409584:0] peer: [6:7577788993182270551:0] socket: 73 qp: -1
2025-11-28T14:51:29.070415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577788996544344228:2064] [node 6] ICS10 traffic start
2025-11-28T14:51:29.070455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:29.094194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577788990558045388:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:29.094312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577788986263078053:2069] [node 6] ICP18 handshake (actor [3:7577788990558045388:2118]) is too old
2025-11-28T14:51:29.092908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788993182270527:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:29.093788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577788993182270542:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:29.094087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788988887303168:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:29.094137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:29.094176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788988887303168:2050] [node 3] ICP22 created new session: [6:7577788997477237850:2050]
2025-11-28T14:51:29.094219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788997477237850:2050] [node 3] ICS09 handshake done sender: [6:7577788993182270532:2114] self: [6:7577788993182270546:0] peer: [3:7577788990558045374:0] socket: 99 qp: -1
2025-11-28T14:51:29.094255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788997477237850:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:29.094286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:29.094881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788993182270530:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:29.189180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788993182270526:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:29.189234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:29.189277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788993182270526:2061] [node 4] ICP22 created new session: [6:7577788997477237853:2061]
2025-11-28T14:51:29.189319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788997477237853:2061] [node 4] ICS09 handshake done sender: [6:7577788993182270530:2112] self: [6:7577788993182270547:0] peer: [4:7577788987297587266:0] socket: 97 qp: -1
2025-11-28T14:51:29.189351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788997477237853:2061] [node 4] ICS10 traffic start
2025-11-28T14:51:29.189384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:29.189924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788988887303166:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21726) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577788993182270529:2111] held: no
2025-11-28T14:51:29.190411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788993182270528:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:29.190553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788997477237852:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-11-28T14:51:29.191308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788993182270529:2111] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:29.191480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788990976990337:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:29.193144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577788990976990337:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:29.196778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788988887303166:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:29.200354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577788990976990331:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:29.200411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:29.200453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788990976990331:2060] [node 6] ICP22 created new session: [5:7577788995271957665:2060]
2025-11-28T14:51:29.200491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577788995271957665:2060] [node 6] ICS09 handshake done sender: [5:7577788990976990337:2113] self: [5:7577788990976990336:0] peer: [6:7577788997477237855:0] socket: 100 qp: -1
2025-11-28T14:51:29.200516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577788995271957665:2060] [node 6] ICS10 traffic start
2025-11-28T14:51:29.200548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:29.196826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:29.196868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788988887303166:2048] [node 1] ICP22 created new session: [6:7577788997477237858:2048]
2025-11-28T14:51:29.196905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788997477237858:2048] [node 1] ICS09 handshake done sender: [6:7577788993182270529:2111] self: [6:7577788993182270551:0] peer: [1:7577788987954409584:0] socket: 96 qp: -1
2025-11-28T14:51:29.196931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788997477237858:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:29.196964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:29.197390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577788997477237857:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-11-28T14:51:29.212626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577788993182270542:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:29.233944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788993182270528:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:29.234002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:29.234057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788993182270528:2069] [node 5] ICP22 created new session: [6:7577788997477237860:2069]
2025-11-28T14:51:29.234100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577788997477237860:2069] [node 5] ICS09 handshake done sender: [6:7577788993182270542:2119] self: [6:7577788997477237855:0] peer: [5:7577788990976990336:0] socket: 106 qp: -1
2025-11-28T14:51:29.234132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577788997477237860:2069] [node 5] ICS10 traffic start
2025-11-28T14:51:29.234163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:29.242219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577788990976990354:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:29.242336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577788990976990331:2060] [node 6] ICP18 handshake (actor [5:7577788990976990354:2120]) is too old
2025-11-28T14:51:29.241397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577788993182270536:2116] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:29.264186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:29.264513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577788991243906435:2048] [node 1] ICS01 socket: 144 reason# EndOfStream
2025-11-28T14:51:29.264567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577788986948939039:2048] [node 1] ICP30 unregister session Session# [2:7577788991243906435:2048] VirtualId# [2:7577788991243906433:0]
2025-11-28T14:51:29.264599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577788991243906435:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.264685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:29.264746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:29.272522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:29.272734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788990558045403:2048] [node 1] ICS01 socket: 89 reason# EndOfStream
2025-11-28T14:51:29.272772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788986263078008:2048] [node 1] ICP30 unregister session Session# [3:7577788990558045403:2048] VirtualId# [3:7577788990558045386:0]
2025-11-28T14:51:29.272822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788990558045403:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.272898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:29.272937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:29.288501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:29.288733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788990976990357:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-11-28T14:51:29.288783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788986682022972:2048] [node 1] ICP30 unregister session Session# [5:7577788990976990357:2048] VirtualId# [5:7577788990976990351:0]
2025-11-28T14:51:29.288825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788990976990357:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.288976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:29.289067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:29.289568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:29.289875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788997477237858:2048] [node 1] ICS01 socket: 96 reason# EndOfStream
2025-11-28T14:51:29.289934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788988887303166:2048] [node 1] ICP30 unregister session Session# [6:7577788997477237858:2048] VirtualId# [6:7577788993182270551:0]
2025-11-28T14:51:29.289984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788997477237858:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.290060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:29.290128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:29.292185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:29.292382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788991592554640:2048] [node 1] ICS01 socket: 131 reason# EndOfStream
2025-11-28T14:51:29.292425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788987297587246:2048] [node 1] ICP30 unregister session Session# [4:7577788991592554640:2048] VirtualId# [4:7577788991592554629:0]
2025-11-28T14:51:29.292447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788991592554640:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.292521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:29.292558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:29.320509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:29.320762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577788990558045407:2049] [node 2] ICS01 socket: 135 reason# EndOfStream
2025-11-28T14:51:29.320863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788986263078009:2049] [node 2] ICP30 unregister session Session# [3:7577788990558045407:2049] VirtualId# [3:7577788990558045405:0]
2025-11-28T14:51:29.320904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577788990558045407:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.320968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:29.321006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:29.338439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:29.338666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577788993182270552:2049] [node 2] ICS07 socket disconnect 98 reason# EndOfStream
2025-11-28T14:51:29.338696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788993182270552:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.338768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:29.338802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577788993182270552:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:29.339447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:29.339636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788990976990359:2049] [node 2] ICS01 socket: 94 reason# EndOfStream
2025-11-28T14:51:29.339679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788986682022973:2049] [node 2] ICP30 unregister session Session# [5:7577788990976990359:2049] VirtualId# [5:7577788990976990349:0]
2025-11-28T14:51:29.339705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788990976990359:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.340095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:29.340335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788988887303167:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17206) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:29.340393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788993182270552:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:29.340420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788988887303167:2049] [node 2] ICP30 unregister session Session# [6:7577788993182270552:2049] VirtualId# [6:7577788993182270545:0]
2025-11-28T14:51:29.340456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:29.340698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577788988887303167:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17206) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:29.340714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17206) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:29.342209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788991592554637:2049] [node 2] ICS01 socket: 122 reason# EndOfStream
2025-11-28T14:51:29.342261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788987297587247:2049] [node 2] ICP30 unregister session Session# [4:7577788991592554637:2049] VirtualId# [4:7577788991592554630:0]
2025-11-28T14:51:29.342289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788991592554637:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.342360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:29.342394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:29.339760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:29.339794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:29.353510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577788988887303167:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:29.412520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:29.412507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:29.412761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788990976990361:2050] [node 3] ICS01 socket: 95 reason# EndOfStream
2025-11-28T14:51:29.412799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788986682022974:2050] [node 3] ICP30 unregister session Session# [5:7577788990976990361:2050] VirtualId# [5:7577788990976990350:0]
2025-11-28T14:51:29.412595Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:29.412827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788990976990361:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.412807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577788991592554633:2050] [node 3] ICS01 socket: 86 reason# EndOfStream
2025-11-28T14:51:29.412905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:29.412843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788987297587248:2050] [node 3] ICP30 unregister session Session# [4:7577788991592554633:2050] VirtualId# [4:7577788991592554622:0]
2025-11-28T14:51:29.412942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:29.412869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577788991592554633:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.412936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:29.412971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:29.413411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788997477237850:2050] [node 3] ICS01 socket: 99 reason# EndOfStream
2025-11-28T14:51:29.413446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788988887303168:2050] [node 3] ICP30 unregister session Session# [6:7577788997477237850:2050] VirtualId# [6:7577788993182270546:0]
2025-11-28T14:51:29.413464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788997477237850:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.413505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:29.413535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:29.472504Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:29.472740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577788990976990366:2065] [node 4] ICS01 socket: 81 reason# EndOfStream
2025-11-28T14:51:29.472782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788990976990332:2065] [node 4] ICP30 unregister session Session# [5:7577788990976990366:2065] VirtualId# [5:7577788990976990364:0]
2025-11-28T14:51:29.472809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577788990976990366:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.472880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:29.472937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:29.473050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:29.473280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788997477237853:2061] [node 4] ICS01 socket: 97 reason# EndOfStream
2025-11-28T14:51:29.473323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788993182270526:2061] [node 4] ICP30 unregister session Session# [6:7577788997477237853:2061] VirtualId# [6:7577788993182270547:0]
2025-11-28T14:51:29.473363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788997477237853:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.473438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:29.473497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:29.500505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:29.500751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577788997477237860:2069] [node 5] ICS01 socket: 106 reason# EndOfStream
2025-11-28T14:51:29.500794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788993182270528:2069] [node 5] ICP30 unregister session Session# [6:7577788997477237860:2069] VirtualId# [6:7577788997477237855:0]
2025-11-28T14:51:29.500823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577788997477237860:2069] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:29.500893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:29.500928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:29.593784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788997815553618:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:29.593821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577788997815553619:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:29.648824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788996091845950:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:29.648871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577788996091845951:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:29.684607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788996749666568:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:29.684649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577788996749666569:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:29.755302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788996813951188:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:29.755336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788996813951189:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:29.755357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577788996813951190:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:29.836651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788995862190235:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:29.836708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788995862190236:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:29.836731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577788995862190237:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:29.859672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788996743719921:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:29.859705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788996743719922:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:29.859734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577788996743719923:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:30.263345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788997815553618:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(178657675ns) {Configure:104(167361176ns) {ProcessConfigured:135(72640ns) {StartInitialHandshake:157(66482ns) {DropHandshakes:470(2010ns) {DropIncomingHandshake:442(376ns) DropOutgoingHandshake:457(512ns)} GenerateSessionVirtualId:988(671ns) PrepareNewSessionHandshake:477(795ns) {SwitchToState:179(250ns)}}}}}
2025-11-28T14:51:30.271495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789002110520979:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:30.271550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789001108918533:2095] [node 6] ICP01 ready to work
2025-11-28T14:51:30.271510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789002110520980:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:30.317489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789000157157579:2088] [node 6] ICP01 ready to work
2025-11-28T14:51:30.317899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577788997815553619:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(38483693ns) {Configure:104(38474741ns) {ProcessConfigured:135(7016539ns) {StartInitialHandshake:157(7012500ns) {DropHandshakes:470(2852ns) {DropIncomingHandshake:442(814ns) DropOutgoingHandshake:457(352ns)} GenerateSessionVirtualId:988(1106ns) PrepareNewSessionHandshake:477(623716ns) {SwitchToState:179(3812ns)}}}}}
2025-11-28T14:51:30.317981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789000386813309:2103] [node 5] ICP01 ready to work
2025-11-28T14:51:30.319563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789000157157581:2089] [node 4] ICP01 ready to work
2025-11-28T14:51:30.319984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789001108918544:2096] [node 5] ICP01 ready to work
2025-11-28T14:51:30.321558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789001044633928:2098] [node 4] ICP01 ready to work
2025-11-28T14:51:30.323552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789001038687239:2067] [node 5] ICP01 ready to work
2025-11-28T14:51:30.324715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789001044633927:2099] [node 5] ICP01 ready to work
2025-11-28T14:51:30.324788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789000386813310:2102] [node 6] ICP01 ready to work
2025-11-28T14:51:30.332899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789001108918533:2095] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(61047985ns) {Configure:104(61043993ns) {ProcessConfigured:135(6863592ns) {StartInitialHandshake:157(6858642ns) {DropHandshakes:470(2403ns) {DropIncomingHandshake:442(579ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:477(6127ns) {SwitchToState:179(2866ns)}}}}}
2025-11-28T14:51:30.322026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789002110520979:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(50114054ns) {Configure:104(50111014ns) {ProcessConfigured:135(2036897ns) {StartInitialHandshake:157(2033925ns) {DropHandshakes:470(2122ns) {DropIncomingHandshake:442(651ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(621ns) PrepareNewSessionHandshake:477(3104ns) {SwitchToState:179(1470ns)}}}}}
2025-11-28T14:51:30.322998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789002110520989:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:30.334745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789000386813311:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:30.331727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789002110520980:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(48876300ns) {Configure:104(48873354ns) {ProcessConfigured:135(1518450ns) {StartInitialHandshake:157(1515757ns) {DropHandshakes:470(3505ns) {DropIncomingHandshake:442(843ns) DropOutgoingHandshake:457(623ns)} GenerateSessionVirtualId:988(798ns) PrepareNewSessionHandshake:477(2937ns) {SwitchToState:179(1542ns)}}}}}
2025-11-28T14:51:30.342468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788996749666569:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51517771ns) {Configure:104(51513552ns) {ProcessConfigured:135(66205ns) {StartInitialHandshake:157(63417ns) {DropHandshakes:470(2479ns) {DropIncomingHandshake:442(505ns) DropOutgoingHandshake:457(654ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:477(2007ns) {SwitchToState:179(524ns)}}}}}
2025-11-28T14:51:30.339013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788996743719921:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52031759ns) {Configure:104(52028781ns) {ProcessConfigured:135(3740306ns) {StartInitialHandshake:157(3738348ns) {DropHandshakes:470(1077ns) {DropIncomingHandshake:442(296ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(406ns) PrepareNewSessionHandshake:477(1643ns) {SwitchToState:179(269ns)}}}}}
2025-11-28T14:51:30.343087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789001044633929:2104] [node 6] ICP01 ready to work
2025-11-28T14:51:30.365437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788996091845950:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33426709ns) {Configure:104(33420877ns) {ProcessConfigured:135(58239ns) {StartInitialHandshake:157(55464ns) {DropHandshakes:470(3097ns) {DropIncomingHandshake:442(718ns) DropOutgoingHandshake:457(407ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:477(716ns) {SwitchToState:179(162ns)}}}}}
2025-11-28T14:51:30.364316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789001038687267:2069] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.469193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789001044633928:2098] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(124191614ns) {Configure:104(124186893ns) {ProcessConfigured:135(74621ns) {StartInitialHandshake:157(73057ns) {DropHandshakes:470(869ns) {DropIncomingHandshake:442(132ns) DropOutgoingHandshake:457(99ns)} GenerateSessionVirtualId:988(404ns) PrepareNewSessionHandshake:477(2067ns) {SwitchToState:179(847ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.470824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788995862190236:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(196204843ns) {Configure:104(196201332ns) {ProcessConfigured:135(4060310ns) {StartInitialHandshake:157(4057411ns) {DropHandshakes:470(1086ns) {DropIncomingHandshake:442(284ns) DropOutgoingHandshake:457(315ns)} GenerateSessionVirtualId:988(2036ns) PrepareNewSessionHandshake:477(2811ns) {SwitchToState:179(1209ns)}}}}}
2025-11-28T14:51:30.458674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577788996091845951:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(139032158ns) {Configure:104(139023442ns) {ProcessConfigured:135(29959612ns) {StartInitialHandshake:157(29955106ns) {DropHandshakes:470(3983ns) {DropIncomingHandshake:442(1015ns) DropOutgoingHandshake:457(965ns)} GenerateSessionVirtualId:988(4530ns) PrepareNewSessionHandshake:477(8176ns) {SwitchToState:179(2314ns)}}}}}
2025-11-28T14:51:30.473385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789001044633937:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25178
2025-11-28T14:51:30.461982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789000386813314:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64372
2025-11-28T14:51:30.474458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788996813951189:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(91647653ns) {Configure:104(91644634ns) {ProcessConfigured:135(6660132ns) {StartInitialHandshake:157(6656280ns) {DropHandshakes:470(3503ns) {DropIncomingHandshake:442(890ns) DropOutgoingHandshake:457(702ns)} GenerateSessionVirtualId:988(5191ns) PrepareNewSessionHandshake:477(3916ns) {SwitchToState:179(2130ns)}}}}}
2025-11-28T14:51:30.474932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789001044633938:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25178
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.469907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789000386813314:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.476422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788995862190237:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(150694870ns) {Configure:104(150688528ns) {ProcessConfigured:135(5387641ns) {StartInitialHandshake:157(5384218ns) {DropHandshakes:470(2018ns) {DropIncomingHandshake:442(508ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(562ns) PrepareNewSessionHandshake:477(3386ns) {SwitchToState:179(1314ns)}}}}}
2025-11-28T14:51:30.475334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788996743719923:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(73649835ns) {Configure:104(73645804ns) {ProcessConfigured:135(1310149ns) {StartInitialHandshake:157(1306822ns) {DropHandshakes:470(3474ns) {DropIncomingHandshake:442(812ns) DropOutgoingHandshake:457(597ns)} GenerateSessionVirtualId:988(1073ns) PrepareNewSessionHandshake:477(2228ns) {SwitchToState:179(942ns)}}}}}
2025-11-28T14:51:30.474508Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788996813951188:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(130178375ns) {Configure:104(130172317ns) {ProcessConfigured:135(87752474ns) {StartInitialHandshake:157(87748493ns) {DropHandshakes:470(3677ns) {DropIncomingHandshake:442(1066ns) DropOutgoingHandshake:457(517ns)} GenerateSessionVirtualId:988(1936ns) PrepareNewSessionHandshake:477(5951ns) {SwitchToState:179(2493ns)}}}}}
2025-11-28T14:51:30.481442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789001044633937:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.485350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789001108918557:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20069
2025-11-28T14:51:30.498659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789000157157579:2088] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(158689490ns) {Configure:104(158685586ns) {ProcessConfigured:135(2377708ns) {StartInitialHandshake:157(2374352ns) {DropHandshakes:470(2653ns) {DropIncomingHandshake:442(768ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:477(2847ns) {SwitchToState:179(1130ns)}}}}}
2025-11-28T14:51:30.485417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789001108918558:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20069
2025-11-28T14:51:30.490521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577788996091845950:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.499130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789002110520989:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(148442802ns) {Configure:104(146780215ns) {ProcessConfigured:135(8669920ns) {StartInitialHandshake:157(8667474ns) {DropHandshakes:470(1969ns) {DropIncomingHandshake:442(480ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(786ns) PrepareNewSessionHandshake:477(3560ns) {SwitchToState:179(2089ns)}}}}}
2025-11-28T14:51:30.486008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577788996749666568:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(153862186ns) {Configure:104(153858075ns) {ProcessConfigured:135(9295910ns) {StartInitialHandshake:157(9293108ns) {DropHandshakes:470(3979459ns) {DropIncomingHandshake:442(451ns) DropOutgoingHandshake:457(1941ns)} GenerateSessionVirtualId:988(1688ns) PrepareNewSessionHandshake:477(2105ns) {SwitchToState:179(628ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.487602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577788996813951190:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(129760561ns) {Configure:104(129755207ns) {ProcessConfigured:135(52521ns) {StartInitialHandshake:157(51345ns) {DropHandshakes:470(2001ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(283ns)} GenerateSessionVirtualId:988(597ns) PrepareNewSessionHandshake:477(1099ns) {SwitchToState:179(285ns)}}}}}
2025-11-28T14:51:30.507824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577788996743719922:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(136877679ns) {Configure:104(136868177ns) {ProcessConfigured:135(55745ns) {StartInitialHandshake:157(53891ns) {DropHandshakes:470(3004ns) {DropIncomingHandshake:442(1024ns) DropOutgoingHandshake:457(540ns)} GenerateSessionVirtualId:988(536ns) PrepareNewSessionHandshake:477(964ns) {SwitchToState:179(410ns)}}}}}
2025-11-28T14:51:30.495362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788996749666568:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.491463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789000386813309:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(147489974ns) {Configure:104(147484910ns) {ProcessConfigured:135(2537748ns) {StartInitialHandshake:157(2536250ns) {DropHandshakes:470(1155ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(2068ns) {SwitchToState:179(889ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.509053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789002110520973:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.509098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789002110520973:2105] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.513667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789001044633927:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(155095449ns) {Configure:104(155089956ns) {ProcessConfigured:135(4247692ns) {StartInitialHandshake:157(4244714ns) {DropHandshakes:470(1757ns) {DropIncomingHandshake:442(839ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(679ns) PrepareNewSessionHandshake:477(1896ns) {SwitchToState:179(719ns)}}}}}
2025-11-28T14:51:30.511374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687290:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.518850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789001108918558:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.511427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687291:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.514076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789002110520983:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.514130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789002110520983:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.520664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788996813951190:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.511472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687292:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.523676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789001044633933:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.523454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789000157157581:2089] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(51628570ns) {Configure:104(51624719ns) {ProcessConfigured:135(535947ns) {StartInitialHandshake:157(534278ns) {DropHandshakes:470(4326ns) {DropIncomingHandshake:442(278ns) DropOutgoingHandshake:457(339ns)} GenerateSessionVirtualId:988(562ns) PrepareNewSessionHandshake:477(1194ns) {SwitchToState:179(410ns)}}}}}
2025-11-28T14:51:30.523342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789001108918557:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.520552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789002110520983:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:30.524815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789001044633933:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.520917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789002110520973:2105] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:30.526462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789000386813326:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64372
2025-11-28T14:51:30.521308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789002110520994:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1885
2025-11-28T14:51:30.527143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789001044633938:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.515959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789001038687291:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.524984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788997815553619:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.524530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788996813951189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.526591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577788995862190235:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48461704ns) {Configure:104(48456969ns) {ProcessConfigured:135(2131734ns) {StartInitialHandshake:157(2129899ns) {DropHandshakes:470(5304ns) {DropIncomingHandshake:442(450ns) DropOutgoingHandshake:457(3928ns)} GenerateSessionVirtualId:988(1497ns) PrepareNewSessionHandshake:477(2075ns) {SwitchToState:179(973ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.525086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:30.518529Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789001038687239:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(162723457ns) {Configure:104(161302753ns) {ProcessConfigured:135(7918952ns) {StartInitialHandshake:157(7913785ns) {DropHandshakes:470(4404ns) {DropIncomingHandshake:442(1001ns) DropOutgoingHandshake:457(598ns)} GenerateSessionVirtualId:988(1107ns) PrepareNewSessionHandshake:477(3176ns) {SwitchToState:179(1775ns)}}}}}
2025-11-28T14:51:30.528858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789000386813320:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.532312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577788996749666569:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.528229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789000157157602:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16920
2025-11-28T14:51:30.535113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789001044633945:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25178
2025-11-28T14:51:30.535107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789000157157603:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16920
2025-11-28T14:51:30.528899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789000386813320:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.535691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789000157157602:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.532348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789000386813328:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64372
2025-11-28T14:51:30.526219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788997815553619:2049] [node 3] ICP22 created new session: [1:7577789002110520997:2049]
2025-11-28T14:51:30.534641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789000386813320:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:30.527343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789001038687290:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.528684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789001038687292:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.535041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789000386813311:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.526719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789002110520997:2049] [node 3] ICS09 handshake done sender: [1:7577789002110520983:2111] self: [1:7577789002110520982:0] peer: [3:7577789001044633943:0] socket: 81 qp: -1
2025-11-28T14:51:30.535154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788996743719922:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.535928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789001044633933:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:30.526781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789002110520997:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:30.535851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789001038687267:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(54967106ns) {Configure:104(54961846ns) {ProcessConfigured:135(4876283ns) {StartInitialHandshake:157(4874081ns) {DropHandshakes:470(1667ns) {DropIncomingHandshake:442(488ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(572ns) PrepareNewSessionHandshake:477(1631ns) {SwitchToState:179(644ns)}}}}}
2025-11-28T14:51:30.536800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789000157157605:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16920
2025-11-28T14:51:30.535981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789001044633947:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25178
2025-11-28T14:51:30.535202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789000386813326:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.535574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:30.526673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577788997815553618:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.536849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788995862190235:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.535993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788996743719923:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.535866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789000386813311:2108] [node 4] ICP22 created new session: [2:7577789000386813331:2108]
2025-11-28T14:51:30.527071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:30.547695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789001038687267:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.550407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789001044633945:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.527090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:30.535913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789000386813331:2108] [node 4] ICS09 handshake done sender: [2:7577789000386813320:2114] self: [2:7577789000386813319:0] peer: [4:7577789001108918565:0] socket: 85 qp: -1
2025-11-28T14:51:30.552840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789001044633928:2098] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.527315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577788997815553618:2048] [node 2] ICP22 created new session: [1:7577789002110520998:2048]
2025-11-28T14:51:30.535949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789000386813331:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:30.554078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789001044633937:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:30.531916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789002110520998:2048] [node 2] ICS09 handshake done sender: [1:7577789002110520973:2105] self: [1:7577789002110520972:0] peer: [2:7577789000386813327:0] socket: 78 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.554086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:30.558204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789001108918547:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.536158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:30.532304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789002110520998:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:30.558739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687301:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.554270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789001044633928:2098] [node 4] ICP22 created new session: [3:7577789001044633951:2098]
2025-11-28T14:51:30.536820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789000386813316:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.535296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:30.554329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789001044633936:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.558520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789001108918547:2107] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.536845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789000386813316:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.550438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577788997815553619:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(19978958ns) {HandleHandshakeStatus:329(19972052ns) {DropIncomingHandshake:442(514ns) DropOutgoingHandshake:457(656ns) SwitchToState:179(471ns) ProcessPendingSessionEvents:495(12519973ns) {ForwardSessionEventToSession:612(3070380ns) ForwardSessionEventToSession:612(8082ns) ForwardSessionEventToSession:612(699815ns) ForwardSessionEventToSession:612(6564ns) ForwardSessionEventToSession:612(2686ns) ForwardSessionEventToSession:612(624403ns) ForwardSessionEventToSession:612(3291ns) ForwardSessionEventToSession:612(2551ns) ForwardSessionEventToSession:612(8048257ns) ForwardSessionEventToSession:612(21614ns)}}}
2025-11-28T14:51:30.554355Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789001044633936:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.559363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687302:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.554851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789001044633951:2098] [node 4] ICS09 handshake done sender: [3:7577789001044633933:2111] self: [3:7577789001044633932:0] peer: [4:7577789001108918564:0] socket: 89 qp: -1
2025-11-28T14:51:30.537118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789000386813328:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.552811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789002110520986:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.555917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789001044633951:2098] [node 4] ICS10 traffic start
2025-11-28T14:51:30.549162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789000386813329:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64372
2025-11-28T14:51:30.559996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789001108918566:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20069
2025-11-28T14:51:30.552839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789002110520994:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.557404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:30.552848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789002110520986:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.550332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789000386813318:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.561785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789001038687290:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:30.550373Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789000386813318:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.561481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789001108918567:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20069
2025-11-28T14:51:30.557923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789001044633950:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25178
2025-11-28T14:51:30.554136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577788997815553618:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(23390394ns) {HandleHandshakeStatus:329(23179286ns) {DropIncomingHandshake:442(683ns) DropOutgoingHandshake:457(793ns) SwitchToState:179(450ns) ProcessPendingSessionEvents:495(12552792ns) {ForwardSessionEventToSession:612(11018105ns) ForwardSessionEventToSession:612(6170ns) ForwardSessionEventToSession:612(3611ns) ForwardSessionEventToSession:612(164962ns) ForwardSessionEventToSession:612(2838ns) ForwardSessionEventToSession:612(2438ns) ForwardSessionEventToSession:612(403815ns) ForwardSessionEventToSession:612(501406ns) ForwardSessionEventToSession:612(424698ns) ForwardSessionEventToSession:612(2441ns)}}}
2025-11-28T14:51:30.554845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789000386813314:2111] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:30.558150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788996749666568:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.562259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789001108918568:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20069
2025-11-28T14:51:30.560837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789002110521003:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1885
2025-11-28T14:51:30.558094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789000386813316:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:30.563199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788996743719922:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.558408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:30.561304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789002110521004:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1885
2025-11-28T14:51:30.563254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:30.558456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788996749666568:2048] [node 1] ICP22 created new session: [3:7577789001044633953:2048]
2025-11-28T14:51:30.561896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789002110520986:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:30.559003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789000386813318:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:30.564170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789002110520980:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.563468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788996743719922:2049] [node 2] ICP22 created new session: [6:7577789001038687304:2049]
2025-11-28T14:51:30.559065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789001044633953:2048] [node 1] ICS09 handshake done sender: [3:7577789001044633937:2113] self: [3:7577789001044633943:0] peer: [1:7577789002110520982:0] socket: 86 qp: -1
2025-11-28T14:51:30.559029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789000386813311:2108] [node 4] ICP18 handshake (actor [2:7577789000386813328:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.559193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789001044633953:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:30.564884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:30.559280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577788996091845950:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.559479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577788996091845951:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.559515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:30.564938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789002110520980:2109] [node 5] ICP22 created new session: [1:7577789002110521007:2109]
2025-11-28T14:51:30.564125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789001038687304:2049] [node 2] ICS09 handshake done sender: [6:7577789001038687290:2114] self: [6:7577789001038687298:0] peer: [2:7577789000386813317:0] socket: 102 qp: -1
2025-11-28T14:51:30.565657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789001108918569:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20069
2025-11-28T14:51:30.565648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789001044633936:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:30.564260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789001038687304:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:30.566173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789001108918558:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:30.559902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:30.565105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789002110521007:2109] [node 5] ICS09 handshake done sender: [1:7577789002110520986:2113] self: [1:7577789002110520984:0] peer: [5:7577789000157157612:0] socket: 79 qp: -1
2025-11-28T14:51:30.566018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789001044633954:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25178
2025-11-28T14:51:30.564914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:30.560009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788996091845951:2049] [node 3] ICP22 created new session: [2:7577789000386813337:2049]
2025-11-28T14:51:30.565237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789002110521007:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:30.566210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687303:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.565607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:30.560740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:30.567157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789001044633929:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.567568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577788997815553618:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:30.567383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:30.560782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577788996091845950:2048] [node 1] ICP22 created new session: [2:7577789000386813338:2048]
2025-11-28T14:51:30.567439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789001044633929:2104] [node 6] ICP22 created new session: [3:7577789001044633960:2104]
2025-11-28T14:51:30.567772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789001108918557:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:30.560825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789000386813338:2048] [node 1] ICS09 handshake done sender: [2:7577789000386813314:2111] self: [2:7577789000386813327:0] peer: [1:7577789002110520972:0] socket: 82 qp: -1
2025-11-28T14:51:30.567745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789001044633960:2104] [node 6] ICS09 handshake done sender: [3:7577789001044633936:2112] self: [3:7577789001044633935:0] peer: [6:7577789001038687299:0] socket: 94 qp: -1
2025-11-28T14:51:30.567781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789001044633960:2104] [node 6] ICS10 traffic start
2025-11-28T14:51:30.568170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789001038687292:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:30.567901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:30.560874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789000386813337:2049] [node 3] ICS09 handshake done sender: [2:7577789000386813316:2112] self: [2:7577789000386813315:0] peer: [3:7577789001044633949:0] socket: 83 qp: -1
2025-11-28T14:51:30.568155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789001044633938:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:30.567871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788996813951189:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29947) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789001108918557:2114] held: no
2025-11-28T14:51:30.560917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789000386813337:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:30.568163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788996813951190:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.561305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:30.568177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788996813951189:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.561309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789000386813338:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:30.568320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:30.561606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:30.568384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788996813951190:2050] [node 3] ICP22 created new session: [4:7577789001108918571:2050]
2025-11-28T14:51:30.570238Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577788996749666569:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29947) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789001044633938:2114] held: no
2025-11-28T14:51:30.570157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788996743719923:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.569148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789001108918571:2050] [node 3] ICS09 handshake done sender: [4:7577789001108918558:2115] self: [4:7577789001108918564:0] peer: [3:7577789001044633932:0] socket: 92 qp: -1
2025-11-28T14:51:30.570335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577788996749666569:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.565676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789000386813310:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.570227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:30.569168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:30.565713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:30.570327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788996743719923:2050] [node 3] ICP22 created new session: [6:7577789001038687308:2050]
2025-11-28T14:51:30.570541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:30.569180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789001108918571:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:30.565828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789000386813310:2102] [node 6] ICP22 created new session: [2:7577789000386813341:2102]
2025-11-28T14:51:30.570640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789001038687308:2050] [node 3] ICS09 handshake done sender: [6:7577789001038687292:2116] self: [6:7577789001038687299:0] peer: [3:7577789001044633935:0] socket: 104 qp: -1
2025-11-28T14:51:30.570569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577788996749666569:2049] [node 2] ICP22 created new session: [3:7577789001044633962:2049]
2025-11-28T14:51:30.566196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789000386813341:2102] [node 6] ICS09 handshake done sender: [2:7577789000386813318:2113] self: [2:7577789000386813317:0] peer: [6:7577789001038687298:0] socket: 84 qp: -1
2025-11-28T14:51:30.569379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788996813951189:2049] [node 2] ICP22 created new session: [4:7577789001108918572:2049]
2025-11-28T14:51:30.570670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789001038687308:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:30.570657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789001044633962:2049] [node 2] ICS09 handshake done sender: [3:7577789001044633938:2114] self: [3:7577789001044633949:0] peer: [2:7577789000386813315:0] socket: 87 qp: -1
2025-11-28T14:51:30.566236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789000386813341:2102] [node 6] ICS10 traffic start
2025-11-28T14:51:30.570680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789001044633962:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:30.569599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789001108918572:2049] [node 2] ICS09 handshake done sender: [4:7577789001108918557:2114] self: [4:7577789001108918565:0] peer: [2:7577789000386813319:0] socket: 91 qp: -1
2025-11-28T14:51:30.570708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:30.570701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:30.566575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:30.569677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:30.569816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789001108918572:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:30.570053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:30.571612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789001108918569:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.571796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789001108918547:2107] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.572897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687307:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.573462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789001038687291:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:30.573255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789001108918533:2095] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.573639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:30.573698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789001108918533:2095] [node 6] ICP22 created new session: [4:7577789001108918576:2095]
2025-11-28T14:51:30.573917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789001108918576:2095] [node 6] ICS09 handshake done sender: [4:7577789001108918547:2107] self: [4:7577789001108918546:0] peer: [6:7577789001038687300:0] socket: 93 qp: -1
2025-11-28T14:51:30.574086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789001038687267:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.573942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789001108918576:2095] [node 6] ICS10 traffic start
2025-11-28T14:51:30.574139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:30.574042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:30.574227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789001038687267:2069] [node 4] ICP22 created new session: [6:7577789001038687310:2069]
2025-11-28T14:51:30.574286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789001038687310:2069] [node 4] ICS09 handshake done sender: [6:7577789001038687291:2115] self: [6:7577789001038687300:0] peer: [4:7577789001108918546:0] socket: 103 qp: -1
2025-11-28T14:51:30.574348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789001038687310:2069] [node 4] ICS10 traffic start
2025-11-28T14:51:30.574348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577788996813951188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.574386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:30.574444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789001108918575:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20069
2025-11-28T14:51:30.574696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789002110520987:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.574919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789002110520987:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.575278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789002110521009:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1885
2025-11-28T14:51:30.576228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789002110520987:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:30.576407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789002110520979:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.576601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:30.576833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789002110520979:2107] [node 4] ICP22 created new session: [1:7577789002110521011:2107]
2025-11-28T14:51:30.576888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789002110521011:2107] [node 4] ICS09 handshake done sender: [1:7577789002110520987:2114] self: [1:7577789002110520985:0] peer: [4:7577789001108918579:0] socket: 128 qp: -1
2025-11-28T14:51:30.576894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789002110521009:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.576916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789002110521011:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:30.576949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:30.577024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789001108918580:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20069
2025-11-28T14:51:30.577398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789002110520979:2107] [node 4] ICP18 handshake (actor [1:7577789002110521009:2119]) is too old
2025-11-28T14:51:30.577437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789001108918569:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:30.577948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577788996813951188:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.578027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:30.578131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577788996813951188:2048] [node 1] ICP22 created new session: [4:7577789001108918581:2048]
2025-11-28T14:51:30.578199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789001108918581:2048] [node 1] ICS09 handshake done sender: [4:7577789001108918569:2120] self: [4:7577789001108918579:0] peer: [1:7577789002110520985:0] socket: 136 qp: -1
2025-11-28T14:51:30.578836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789001108918581:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:30.579412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:30.579443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789001108918549:2108] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:30.586360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789001108918533:2095] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:30.591758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789001038687287:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:30.604464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789000157157598:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:30.618291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789000157157608:2117] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.626854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789002110521003:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.624775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789000157157603:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.624769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789001044633950:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.624970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789000157157613:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16920
2025-11-28T14:51:30.624980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789000157157605:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.633038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789002110520980:2109] [node 5] ICP18 handshake (actor [1:7577789002110521003:2117]) is too old
2025-11-28T14:51:30.638936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789000157157614:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16920
2025-11-28T14:51:30.639170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789000157157615:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16920
2025-11-28T14:51:30.639307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789000157157602:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:30.639745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789000157157581:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.644676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788995862190235:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.648320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:30.648390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788995862190235:2048] [node 1] ICP22 created new session: [5:7577789000157157618:2048]
2025-11-28T14:51:30.648444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789000157157618:2048] [node 1] ICS09 handshake done sender: [5:7577789000157157602:2113] self: [5:7577789000157157612:0] peer: [1:7577789002110520984:0] socket: 100 qp: -1
2025-11-28T14:51:30.648485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789000157157618:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:30.648544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:30.655770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789000386813325:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.654850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789000157157615:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.655810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789000386813325:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.655886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789001044633946:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.655918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789001044633946:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:30.657495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789001108918549:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.654997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789000157157614:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.655089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577788995862190235:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:30.655242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788995862190236:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.659700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789000386813344:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64372
2025-11-28T14:51:30.661659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789000386813325:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:30.655297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577788995862190237:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.659562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789001044633946:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:30.659185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789000157157623:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16920
2025-11-28T14:51:30.659326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789001108918549:2108] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:30.662726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789001108918544:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.667269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789000386813309:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.666736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789001044633927:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.667384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:30.667445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789001044633927:2099] [node 5] ICP22 created new session: [3:7577789001044633966:2099]
2025-11-28T14:51:30.670934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:30.670851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789001044633966:2099] [node 5] ICS09 handshake done sender: [3:7577789001044633946:2117] self: [3:7577789001044633944:0] peer: [5:7577789000157157622:0] socket: 141 qp: -1
2025-11-28T14:51:30.671005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789001044633966:2099] [node 5] ICS10 traffic start
2025-11-28T14:51:30.671288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:30.671182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789000386813309:2103] [node 5] ICP22 created new session: [2:7577789000386813346:2103]
2025-11-28T14:51:30.671789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:30.671229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789000386813346:2103] [node 5] ICS09 handshake done sender: [2:7577789000386813325:2115] self: [2:7577789000386813324:0] peer: [5:7577789000157157621:0] socket: 98 qp: -1
2025-11-28T14:51:30.671513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789000386813346:2103] [node 5] ICS10 traffic start
2025-11-28T14:51:30.672012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:30.673823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789001108918544:2096] [node 5] ICP22 created new session: [4:7577789001108918584:2096]
2025-11-28T14:51:30.674246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789000157157603:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:30.674154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789001108918584:2096] [node 5] ICS09 handshake done sender: [4:7577789001108918549:2108] self: [4:7577789001108918548:0] peer: [5:7577789000157157616:0] socket: 96 qp: -1
2025-11-28T14:51:30.674216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789001108918584:2096] [node 5] ICS10 traffic start
2025-11-28T14:51:30.674299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:30.675453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789000157157581:2089] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.675545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:30.675590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789000157157581:2089] [node 4] ICP22 created new session: [5:7577789000157157626:2089]
2025-11-28T14:51:30.675649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789000157157626:2089] [node 4] ICS09 handshake done sender: [5:7577789000157157603:2114] self: [5:7577789000157157616:0] peer: [4:7577789001108918548:0] socket: 110 qp: -1
2025-11-28T14:51:30.675675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789000157157626:2089] [node 4] ICS10 traffic start
2025-11-28T14:51:30.675713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.676284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788995862190237:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23138) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789000157157614:2119] held: no
2025-11-28T14:51:30.676483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789000157157624:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16920
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.677065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789000157157625:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16920
2025-11-28T14:51:30.677847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789000157157614:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:30.678839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789000157157615:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.679211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788995862190237:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.679279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:30.679666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788995862190237:2050] [node 3] ICP22 created new session: [5:7577789000157157629:2050]
2025-11-28T14:51:30.679710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789000157157629:2050] [node 3] ICS09 handshake done sender: [5:7577789000157157614:2119] self: [5:7577789000157157622:0] peer: [3:7577789001044633944:0] socket: 88 qp: -1
2025-11-28T14:51:30.679739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789000157157629:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:30.679841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:30.679805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687312:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.680198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577788995862190236:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.680280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:30.680317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577788995862190236:2049] [node 2] ICP22 created new session: [5:7577789000157157631:2049]
2025-11-28T14:51:30.680349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789000157157631:2049] [node 2] ICS09 handshake done sender: [5:7577789000157157615:2120] self: [5:7577789000157157621:0] peer: [2:7577789000386813324:0] socket: 106 qp: -1
2025-11-28T14:51:30.680392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789000157157631:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:30.680427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:30.682232Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789000157157606:2116] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:30.684744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789001108918567:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.684870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789001108918544:2096] [node 5] ICP18 handshake (actor [4:7577789001108918567:2118]) is too old
2025-11-28T14:51:30.716772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789002110520993:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:30.737957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789001038687301:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.738153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577788996743719921:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.741520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789002110520993:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.742108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789002110520993:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.742439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789002110520989:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.742699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687314:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.744902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:30.745006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789002110520989:2110] [node 6] ICP22 created new session: [1:7577789002110521014:2110]
2025-11-28T14:51:30.745482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789002110521014:2110] [node 6] ICS09 handshake done sender: [1:7577789002110520993:2115] self: [1:7577789002110520992:0] peer: [6:7577789001038687313:0] socket: 129 qp: -1
2025-11-28T14:51:30.746063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789001038687301:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:30.745533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789002110521014:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:30.745574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:30.747807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788996743719921:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789001038687301:2118] held: no
2025-11-28T14:51:30.748237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577788996743719921:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.748286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:30.748333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577788996743719921:2048] [node 1] ICP22 created new session: [6:7577789001038687315:2048]
2025-11-28T14:51:30.748401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789001038687315:2048] [node 1] ICS09 handshake done sender: [6:7577789001038687301:2118] self: [6:7577789001038687313:0] peer: [1:7577789002110520992:0] socket: 130 qp: -1
2025-11-28T14:51:30.748448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789001038687315:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:30.749084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:30.752750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789000157157599:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:30.762808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789000386813344:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.775535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789000386813309:2103] [node 5] ICP18 handshake (actor [2:7577789000386813344:2119]) is too old
2025-11-28T14:51:30.787649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789000157157601:2112] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:30.788698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789001038687312:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.788867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789001038687239:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:30.789130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789000157157601:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:30.789736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789000157157601:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:30.789969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789000157157579:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:30.790075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:30.790122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789000157157579:2088] [node 6] ICP22 created new session: [5:7577789000157157634:2088]
2025-11-28T14:51:30.790187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789000157157634:2088] [node 6] ICS09 handshake done sender: [5:7577789000157157601:2112] self: [5:7577789000157157600:0] peer: [6:7577789001038687317:0] socket: 119 qp: -1
2025-11-28T14:51:30.790223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789000157157634:2088] [node 6] ICS10 traffic start
2025-11-28T14:51:30.790287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:30.789934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789001038687318:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-11-28T14:51:30.790248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789001038687312:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:30.790540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789001038687239:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:30.790584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:30.790628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789001038687239:2067] [node 5] ICP22 created new session: [6:7577789001038687319:2067]
2025-11-28T14:51:30.790882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789001038687319:2067] [node 5] ICS09 handshake done sender: [6:7577789001038687312:2122] self: [6:7577789001038687317:0] peer: [5:7577789000157157600:0] socket: 123 qp: -1
2025-11-28T14:51:30.790917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789001038687319:2067] [node 5] ICS10 traffic start
2025-11-28T14:51:30.790947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:30.791923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789001038687239:2067] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:30.809254Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:30.809502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789000386813338:2048] [node 1] ICS07 socket disconnect 82 reason# EndOfStream
2025-11-28T14:51:30.809542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789000386813338:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.809623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:30.809684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789000386813338:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:30.810458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:30.810694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789001044633953:2048] [node 1] ICS07 socket disconnect 86 reason# EndOfStream
2025-11-28T14:51:30.810726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789001044633953:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.810845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:30.810991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789001044633953:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:30.813549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788996091845950:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:30.815098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789000386813338:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:30.816746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577788996749666568:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:30.816790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789001044633953:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:30.817888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577788996091845950:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:30.817950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789000386813338:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:30.817992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577788996091845950:2048] [node 1] ICP30 unregister session Session# [2:7577789000386813338:2048] VirtualId# [2:7577789000386813327:0]
2025-11-28T14:51:30.818046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:30.818023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577788996749666568:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:30.818142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789001044633953:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:30.818078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577788996091845950:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.818092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.818205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788996749666568:2048] [node 1] ICP30 unregister session Session# [3:7577789001044633953:2048] VirtualId# [3:7577789001044633943:0]
2025-11-28T14:51:30.818255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:30.818306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577788996749666568:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.818317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.821620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:30.821938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789000157157618:2048] [node 1] ICS07 socket disconnect 100 reason# EndOfStream
2025-11-28T14:51:30.821972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789000157157618:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.822042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:30.822081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789000157157618:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:30.823339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:30.823624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788995862190235:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:30.823863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789001038687315:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-28T14:51:30.820573Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:30.824092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789000157157618:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:30.820831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789001108918581:2048] [node 1] ICS07 socket disconnect 136 reason# EndOfStream
2025-11-28T14:51:30.823962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788996743719921:2048] [node 1] ICP30 unregister session Session# [6:7577789001038687315:2048] VirtualId# [6:7577789001038687313:0]
2025-11-28T14:51:30.820860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789001108918581:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.824272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788995862190235:2048] [node 1] ICP30 unregister session Session# [5:7577789000157157618:2048] VirtualId# [5:7577789000157157612:0]
2025-11-28T14:51:30.824074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789001038687315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.820976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:30.824348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:30.824472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577788995862190235:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.824487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.821028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789001108918581:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:30.832709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577788996749666568:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:30.824127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:30.824161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:30.823010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577788996813951188:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:30.823071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789001108918581:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:30.823105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788996813951188:2048] [node 1] ICP30 unregister session Session# [4:7577789001108918581:2048] VirtualId# [4:7577789001108918579:0]
2025-11-28T14:51:30.823159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:30.823185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577788996813951188:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.823219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.829022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577788996091845950:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:30.833040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577788996813951188:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:30.834138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577788995862190235:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:30.843837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:30.844069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789001044633962:2049] [node 2] ICS01 socket: 87 reason# EndOfStream
2025-11-28T14:51:30.844109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577788996749666569:2049] [node 2] ICP30 unregister session Session# [3:7577789001044633962:2049] VirtualId# [3:7577789001044633949:0]
2025-11-28T14:51:30.844139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789001044633962:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.844255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:30.844302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:30.849774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:30.849983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789001038687304:2049] [node 2] ICS01 socket: 102 reason# EndOfStream
2025-11-28T14:51:30.850026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788996743719922:2049] [node 2] ICP30 unregister session Session# [6:7577789001038687304:2049] VirtualId# [6:7577789001038687298:0]
2025-11-28T14:51:30.850061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789001038687304:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.850170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:30.850222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:30.850640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:30.850882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789000157157631:2049] [node 2] ICS07 socket disconnect 106 reason# EndOfStream
2025-11-28T14:51:30.850920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789000157157631:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.851059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:30.851125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789000157157631:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:30.851475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:30.851680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789001108918572:2049] [node 2] ICS01 socket: 91 reason# EndOfStream
2025-11-28T14:51:30.851717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788996813951189:2049] [node 2] ICP30 unregister session Session# [4:7577789001108918572:2049] VirtualId# [4:7577789001108918565:0]
2025-11-28T14:51:30.851750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789001108918572:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.851823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:30.851877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:30.851946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577788995862190236:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29947) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:30.852062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789000157157631:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:30.852092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788995862190236:2049] [node 2] ICP30 unregister session Session# [5:7577789000157157631:2049] VirtualId# [5:7577789000157157621:0]
2025-11-28T14:51:30.852340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:30.852408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577788995862190236:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29947) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.852422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29947) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.866303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577788995862190236:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:30.872736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:30.872957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789001108918571:2050] [node 3] ICS01 socket: 92 reason# EndOfStream
2025-11-28T14:51:30.873002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577788996813951190:2050] [node 3] ICP30 unregister session Session# [4:7577789001108918571:2050] VirtualId# [4:7577789001108918564:0]
2025-11-28T14:51:30.873034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789001108918571:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.873115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:30.873154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:30.873843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:30.874061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789000157157629:2050] [node 3] ICS01 socket: 88 reason# EndOfStream
2025-11-28T14:51:30.874108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577788995862190237:2050] [node 3] ICP30 unregister session Session# [5:7577789000157157629:2050] VirtualId# [5:7577789000157157622:0]
2025-11-28T14:51:30.874141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789000157157629:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.874253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:30.874298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:30.875144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:30.875324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789001038687308:2050] [node 3] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-28T14:51:30.875352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789001038687308:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.875425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:30.875471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789001038687308:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:30.876191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577788996743719923:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23138) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:30.876278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789001038687308:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:30.876329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577788996743719923:2050] [node 3] ICP30 unregister session Session# [6:7577789001038687308:2050] VirtualId# [6:7577789001038687299:0]
2025-11-28T14:51:30.876383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:30.876409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577788996743719923:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23138) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.876421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23138) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.886295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577788996743719923:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:30.893884Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:30.894046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789001038687310:2069] [node 4] ICS07 socket disconnect 103 reason# EndOfStream
2025-11-28T14:51:30.894068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789001038687310:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.894141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:30.894181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789001038687310:2069] [node 4] ICS15 start handshake
2025-11-28T14:51:30.894880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789001038687267:2069] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25934) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:30.894922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789001038687310:2069] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:30.895086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:30.895286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789000157157626:2089] [node 4] ICS01 socket: 110 reason# EndOfStream
2025-11-28T14:51:30.895321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789000157157581:2089] [node 4] ICP30 unregister session Session# [5:7577789000157157626:2089] VirtualId# [5:7577789000157157616:0]
2025-11-28T14:51:30.895351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789000157157626:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.895569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789001038687267:2069] [node 4] ICP30 unregister session Session# [6:7577789001038687310:2069] VirtualId# [6:7577789001038687300:0]
2025-11-28T14:51:30.895424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:30.895463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:30.895637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:30.895671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789001038687267:2069] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25934) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.895682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25934) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.905828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789001038687267:2069] [node 4] ICP33 wake up from error state
2025-11-28T14:51:30.907408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:30.907603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789001038687319:2067] [node 5] ICS07 socket disconnect 123 reason# EndOfStream
2025-11-28T14:51:30.907634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789001038687319:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:30.907724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:30.907770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789001038687319:2067] [node 5] ICS15 start handshake
2025-11-28T14:51:30.908519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789001038687239:2067] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6210) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:30.908572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789001038687319:2067] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:30.908707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789001038687239:2067] [node 5] ICP30 unregister session Session# [6:7577789001038687319:2067] VirtualId# [6:7577789001038687317:0]
2025-11-28T14:51:30.908757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:30.908791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789001038687239:2067] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6210) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:30.908802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6210) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:31.005929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789006986309550:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:31.006050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789006986309551:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:31.029976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789005379029523:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:31.030163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789005379029522:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:31.051694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789004557864906:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:31.051727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789004557864907:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:31.076043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789006994902678:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:31.076085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789006994902677:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:31.076095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789006994902676:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:31.095087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789006923783645:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:31.095131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789006923783646:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:31.095154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789006923783647:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:31.119087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789007029951354:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:31.119127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789007029951355:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:31.119145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789007029951356:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:31.204959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789005379029583:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:31.245062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789007029951416:2103] [node 5] ICP01 ready to work
2025-11-28T14:51:31.271878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789006994902740:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:31.268991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789004557864979:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:31.274132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789004557864980:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:31.274344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789005379029586:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:31.274081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789006986309613:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:31.275166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789006986309614:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:31.274733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789007029951417:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:31.274789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789006994902741:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:31.275136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789006923783708:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:31.275412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789004557864981:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:31.276011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789005379029587:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:31.276078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789006986309615:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:31.276600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789006923783709:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:31.278021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789006986309550:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(86139140ns) {Configure:104(86133866ns) {ProcessConfigured:135(287879ns) {StartInitialHandshake:157(283107ns) {DropHandshakes:470(5395ns) {DropIncomingHandshake:442(3068ns) DropOutgoingHandshake:457(395ns)} GenerateSessionVirtualId:988(823ns) PrepareNewSessionHandshake:477(1545ns) {SwitchToState:179(587ns)}}}}}
2025-11-28T14:51:31.278611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789006986309551:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(85451757ns) {Configure:104(85449365ns) {ProcessConfigured:135(422885ns) {StartInitialHandshake:157(420769ns) {DropHandshakes:470(2102ns) {DropIncomingHandshake:442(374ns) DropOutgoingHandshake:457(573ns)} GenerateSessionVirtualId:988(672ns) PrepareNewSessionHandshake:477(1385ns) {SwitchToState:179(396ns)}}}}}
2025-11-28T14:51:31.278844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789005379029522:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(78509972ns) {Configure:104(78504438ns) {ProcessConfigured:135(186314ns) {StartInitialHandshake:157(185011ns) {DropHandshakes:470(709ns) {DropIncomingHandshake:442(192ns) DropOutgoingHandshake:457(148ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:477(782ns) {SwitchToState:179(304ns)}}}}}
2025-11-28T14:51:31.278988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789005379029523:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(78348600ns) {Configure:104(78344580ns) {ProcessConfigured:135(498683ns) {StartInitialHandshake:157(496854ns) {DropHandshakes:470(1419ns) {DropIncomingHandshake:442(357ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(496ns) PrepareNewSessionHandshake:477(839ns) {SwitchToState:179(216ns)}}}}}
2025-11-28T14:51:31.279064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789005379029583:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(69734851ns) {Configure:104(69726207ns) {ProcessConfigured:135(155300ns) {StartInitialHandshake:157(152728ns) {DropHandshakes:470(1343ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(529ns) PrepareNewSessionHandshake:477(2268ns) {SwitchToState:179(1014ns)}}}}}
2025-11-28T14:51:31.281259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789007029951354:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35753254ns) {Configure:104(35749869ns) {ProcessConfigured:135(893650ns) {StartInitialHandshake:157(891731ns) {DropHandshakes:470(72255ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(533ns) PrepareNewSessionHandshake:477(1500ns) {SwitchToState:179(584ns)}}}}}
2025-11-28T14:51:31.282619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789006994902677:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(53572782ns) {Configure:104(53567166ns) {ProcessConfigured:135(505648ns) {StartInitialHandshake:157(503871ns) {DropHandshakes:470(3679ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(451ns) PrepareNewSessionHandshake:477(1722ns) {SwitchToState:179(1010ns)}}}}}
2025-11-28T14:51:31.282704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789006994902676:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53694609ns) {Configure:104(53687820ns) {ProcessConfigured:135(582756ns) {StartInitialHandshake:157(581344ns) {DropHandshakes:470(1108ns) {DropIncomingHandshake:442(277ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(597ns) PrepareNewSessionHandshake:477(984ns) {SwitchToState:179(357ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.287748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789004557864988:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22035
2025-11-28T14:51:31.292577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789004557864989:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22035
2025-11-28T14:51:31.293229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789005379029598:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32875
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.299960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789005379029602:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32875
2025-11-28T14:51:31.293462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789006986309627:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9318
2025-11-28T14:51:31.293719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789006923783645:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16542781ns) {Configure:104(16537917ns) {ProcessConfigured:135(10046972ns) {StartInitialHandshake:157(10045036ns) {DropHandshakes:470(1023ns) {DropIncomingHandshake:442(205ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(3779ns) {SwitchToState:179(1975ns)}}}}}
2025-11-28T14:51:31.320796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789004557864906:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24081143ns) {Configure:104(24074111ns) {ProcessConfigured:135(9973240ns) {StartInitialHandshake:157(9971497ns) {DropHandshakes:470(1202773ns) {DropIncomingHandshake:442(1809ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(1366ns) PrepareNewSessionHandshake:477(2551ns) {SwitchToState:179(994ns)}}}}}
2025-11-28T14:51:31.315946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789006923783708:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16291002ns) {Configure:104(16287709ns) {ProcessConfigured:135(1038193ns) {StartInitialHandshake:157(1036272ns) {DropHandshakes:470(2112ns) {DropIncomingHandshake:442(697ns) DropOutgoingHandshake:457(244ns)} GenerateSessionVirtualId:988(513ns) PrepareNewSessionHandshake:477(994ns) {SwitchToState:179(319ns)}}}}}
2025-11-28T14:51:31.317295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789006923783647:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25885926ns) {Configure:104(25875925ns) {ProcessConfigured:135(5164137ns) {StartInitialHandshake:157(5161938ns) {DropHandshakes:470(1807ns) {DropIncomingHandshake:442(447ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:477(2079ns) {SwitchToState:179(587ns)}}}}}
2025-11-28T14:51:31.325689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789004557864980:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26912865ns) {Configure:104(26907792ns) {ProcessConfigured:135(18918925ns) {StartInitialHandshake:157(18912548ns) {DropHandshakes:470(1134ns) {DropIncomingHandshake:442(354ns) DropOutgoingHandshake:457(145ns)} GenerateSessionVirtualId:988(707ns) PrepareNewSessionHandshake:477(4386ns) {SwitchToState:179(2170ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.331031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309627:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.330862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789005379029598:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.332349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789006986309630:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9318
2025-11-28T14:51:31.332516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789006986309631:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9318
2025-11-28T14:51:31.332895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309617:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.332512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789006994902751:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2575
2025-11-28T14:51:31.332564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789005379029522:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.332914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789006986309617:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.332515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789006994902750:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2575
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.333872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789004557864988:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.336808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789006986309617:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:31.335183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789007029951417:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(49597177ns) {Configure:104(49591117ns) {ProcessConfigured:135(1473055ns) {StartInitialHandshake:157(1470315ns) {DropHandshakes:470(168658ns) {DropIncomingHandshake:442(982ns) DropOutgoingHandshake:457(529ns)} GenerateSessionVirtualId:988(1012ns) PrepareNewSessionHandshake:477(2823ns) {SwitchToState:179(1334ns)}}}}}
2025-11-28T14:51:31.335133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789005379029603:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32875
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.335306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789006994902741:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(49198605ns) {Configure:104(49194264ns) {ProcessConfigured:135(1131480ns) {StartInitialHandshake:157(1128850ns) {DropHandshakes:470(1219ns) {DropIncomingHandshake:442(418ns) DropOutgoingHandshake:457(258ns)} GenerateSessionVirtualId:988(1027ns) PrepareNewSessionHandshake:477(1414ns) {SwitchToState:179(714ns)}}}}}
2025-11-28T14:51:31.334706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789004557864989:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.336689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789004557864994:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22035
2025-11-28T14:51:31.338154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309631:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.339750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789005379029605:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32875
2025-11-28T14:51:31.335585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789006994902750:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.339259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789006923783721:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47382
2025-11-28T14:51:31.338608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789004557864995:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22035
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.336747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789006994902751:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.340285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789005379029602:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.350824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789006986309550:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.350933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789006923783709:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(42807537ns) {Configure:104(42799836ns) {ProcessConfigured:135(540939ns) {StartInitialHandshake:157(538805ns) {DropHandshakes:470(2538ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(802ns)} GenerateSessionVirtualId:988(639ns) PrepareNewSessionHandshake:477(1691ns) {SwitchToState:179(637ns)}}}}}
2025-11-28T14:51:31.339593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789004557864981:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(48712730ns) {Configure:104(48706928ns) {ProcessConfigured:135(2069588ns) {StartInitialHandshake:157(2067167ns) {DropHandshakes:470(2669ns) {DropIncomingHandshake:442(667ns) DropOutgoingHandshake:457(334ns)} GenerateSessionVirtualId:988(350ns) PrepareNewSessionHandshake:477(1906ns) {SwitchToState:179(924ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.350555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789005379029593:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.350911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:31.350594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789005379029593:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:31.339234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789006994902676:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.350472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789004557864906:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.353974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789006923783722:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47382
2025-11-28T14:51:31.350965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789006986309550:2048] [node 2] ICP22 created new session: [1:7577789006986309637:2048]
2025-11-28T14:51:31.339501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789006994902756:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2575
2025-11-28T14:51:31.350788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789004557864907:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.353243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789005379029606:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32875
2025-11-28T14:51:31.352270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789004557864995:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.351665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789006986309637:2048] [node 2] ICS09 handshake done sender: [1:7577789006986309617:2110] self: [1:7577789006986309616:0] peer: [2:7577789005379029604:0] socket: 84 qp: -1
2025-11-28T14:51:31.339934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789006994902677:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.351702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789006986309637:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:31.353998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789005379029593:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.354015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789005379029591:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.351730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:31.352238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309623:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.354041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789005379029591:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:31.352265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789006986309623:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:31.358530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789006923783725:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47382
2025-11-28T14:51:31.353518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789006986309633:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9318
2025-11-28T14:51:31.355846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789005379029522:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789005379029598:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.354423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789006986309635:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9318
2025-11-28T14:51:31.355534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309619:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.355574Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789006986309619:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:31.356883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789005379029598:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:31.355905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789006986309623:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:31.357765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789005379029583:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.357797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789006986309613:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.359080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789006994902760:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2575
2025-11-28T14:51:31.358136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:31.357848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:31.358190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789005379029583:2107] [node 4] ICP22 created new session: [2:7577789005379029610:2107]
2025-11-28T14:51:31.357897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789006986309613:2107] [node 4] ICP22 created new session: [1:7577789006986309642:2107]
2025-11-28T14:51:31.358449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789005379029610:2107] [node 4] ICS09 handshake done sender: [2:7577789005379029593:2112] self: [2:7577789005379029592:0] peer: [4:7577789006994902759:0] socket: 92 qp: -1
2025-11-28T14:51:31.358445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789006986309619:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:31.358544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789006986309642:2107] [node 4] ICS09 handshake done sender: [1:7577789006986309623:2112] self: [1:7577789006986309621:0] peer: [4:7577789006994902758:0] socket: 89 qp: -1
2025-11-28T14:51:31.358583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789005379029610:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:31.359412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789006923783725:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.358631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:31.358579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789006986309642:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:31.359738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789006923783645:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:51:31.358694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789005379029522:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:51:31.358611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.358721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:31.358827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789006986309551:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.360951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789006994902761:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2575
2025-11-28T14:51:31.359135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:31.359251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789006986309551:2049] [node 3] ICP22 created new session: [1:7577789006986309645:2049]
2025-11-28T14:51:31.358755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789005379029522:2048] [node 1] ICP22 created new session: [2:7577789005379029612:2048]
2025-11-28T14:51:31.359428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789006986309645:2049] [node 3] ICS09 handshake done sender: [1:7577789006986309619:2111] self: [1:7577789006986309618:0] peer: [3:7577789004557864999:0] socket: 85 qp: -1
2025-11-28T14:51:31.361151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789004557865002:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22035
2025-11-28T14:51:31.359459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789006986309645:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.359508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:31.359097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789005379029612:2048] [node 1] ICS09 handshake done sender: [2:7577789005379029598:2114] self: [2:7577789005379029604:0] peer: [1:7577789006986309616:0] socket: 86 qp: -1
2025-11-28T14:51:31.360499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309625:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.360520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789006986309625:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:31.359122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789005379029612:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:31.361555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789006923783727:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47382
2025-11-28T14:51:31.359248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789005379029591:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.362598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789006986309625:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:31.359385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:31.362635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789004557865003:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22035
2025-11-28T14:51:31.359900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789005379029523:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.362618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789006994902751:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:51:31.362770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951432:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.360581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:31.362856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789006986309614:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.360937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789005379029523:2049] [node 3] ICP22 created new session: [2:7577789005379029616:2049]
2025-11-28T14:51:31.363153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789006923783728:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47382
2025-11-28T14:51:31.363145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951433:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.363247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:31.361203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789005379029616:2049] [node 3] ICS09 handshake done sender: [2:7577789005379029591:2111] self: [2:7577789005379029588:0] peer: [3:7577789004557865000:0] socket: 87 qp: -1
2025-11-28T14:51:31.363204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789006923783727:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.361289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789005379029616:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:31.363512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789006923783646:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.363495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789006994902750:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:31.363482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789006986309614:2108] [node 5] ICP22 created new session: [1:7577789006986309648:2108]
2025-11-28T14:51:31.363533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789007029951432:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.361335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:31.363883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789007029951433:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.363581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789006986309648:2108] [node 5] ICS09 handshake done sender: [1:7577789006986309625:2113] self: [1:7577789006986309624:0] peer: [5:7577789006923783726:0] socket: 90 qp: -1
2025-11-28T14:51:31.363892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789004557864988:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:31.363906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789005379029597:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.363726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789006986309648:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:31.363925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789005379029597:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:31.364039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789006994902677:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.364085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789006923783725:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:31.363776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:31.365101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789007029951354:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.365002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:31.365164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789004557864906:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.365136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789006923783645:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789006923783725:2117] held: no
2025-11-28T14:51:31.365279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789007029951355:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.365671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:31.365135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789006994902677:2049] [node 2] ICP22 created new session: [4:7577789006994902762:2049]
2025-11-28T14:51:31.365304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789006923783645:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.365701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789004557864906:2048] [node 1] ICP22 created new session: [3:7577789004557865004:2048]
2025-11-28T14:51:31.365575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:31.365509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789006994902762:2049] [node 2] ICS09 handshake done sender: [4:7577789006994902751:2116] self: [4:7577789006994902759:0] peer: [2:7577789005379029592:0] socket: 95 qp: -1
2025-11-28T14:51:31.365732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789004557865004:2048] [node 1] ICS09 handshake done sender: [3:7577789004557864988:2122] self: [3:7577789004557864999:0] peer: [1:7577789006986309618:0] socket: 96 qp: -1
2025-11-28T14:51:31.365695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789006923783645:2048] [node 1] ICP22 created new session: [5:7577789006923783730:2048]
2025-11-28T14:51:31.365909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789005379029597:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:31.365722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309629:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.365754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789004557865004:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:31.365732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789006923783730:2048] [node 1] ICS09 handshake done sender: [5:7577789006923783725:2117] self: [5:7577789006923783726:0] peer: [1:7577789006986309624:0] socket: 131 qp: -1
2025-11-28T14:51:31.366114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789005379029601:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.365747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789006986309629:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:31.365756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789006923783730:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:31.366133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789005379029601:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:31.365777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:31.365593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789006994902762:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:31.366027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:31.365628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:31.366987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789005379029586:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.367490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:31.367581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789006986309629:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:31.368521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789005379029586:2108] [node 5] ICP22 created new session: [2:7577789005379029619:2108]
2025-11-28T14:51:31.368799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789005379029619:2108] [node 5] ICS09 handshake done sender: [2:7577789005379029597:2113] self: [2:7577789005379029594:0] peer: [5:7577789006923783729:0] socket: 138 qp: -1
2025-11-28T14:51:31.368896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789006923783732:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47382
2025-11-28T14:51:31.368949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789005379029619:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:31.368954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789006986309615:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.368975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789005379029601:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:31.369002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:31.368996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:31.369045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789006986309615:2109] [node 6] ICP22 created new session: [1:7577789006986309651:2109]
2025-11-28T14:51:31.369092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789006986309651:2109] [node 6] ICS09 handshake done sender: [1:7577789006986309629:2115] self: [1:7577789006986309628:0] peer: [6:7577789007029951434:0] socket: 135 qp: -1
2025-11-28T14:51:31.368971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789006923783646:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789006923783727:2118] held: no
2025-11-28T14:51:31.369131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789006986309651:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:31.369119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789006994902676:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.373344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789005379029587:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.380816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:31.373457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:31.369180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:31.376452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789006923783727:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:31.386598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789005379029587:2109] [node 6] ICP22 created new session: [2:7577789005379029622:2109]
2025-11-28T14:51:31.384945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789006986309615:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16076008ns) {HandleHandshakeStatus:329(15892424ns) {DropIncomingHandshake:442(490ns) DropOutgoingHandshake:457(470ns) SwitchToState:179(449ns) ProcessPendingSessionEvents:495(150699ns) {ForwardSessionEventToSession:612(84796ns) ForwardSessionEventToSession:612(3106ns) ForwardSessionEventToSession:612(2327ns) ForwardSessionEventToSession:612(24183ns) ForwardSessionEventToSession:612(5233ns) ForwardSessionEventToSession:612(5197ns) ForwardSessionEventToSession:612(2306ns) ForwardSessionEventToSession:612(2267ns) ForwardSessionEventToSession:612(5082ns) ForwardSessionEventToSession:612(2615ns)}}}
2025-11-28T14:51:31.391777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789006923783646:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.369764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789006994902676:2048] [node 1] ICP22 created new session: [4:7577789006994902764:2048]
2025-11-28T14:51:31.391209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789005379029622:2109] [node 6] ICS09 handshake done sender: [2:7577789005379029601:2115] self: [2:7577789005379029600:0] peer: [6:7577789007029951435:0] socket: 139 qp: -1
2025-11-28T14:51:31.376903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789006994902764:2048] [node 1] ICS09 handshake done sender: [4:7577789006994902750:2115] self: [4:7577789006994902758:0] peer: [1:7577789006986309621:0] socket: 94 qp: -1
2025-11-28T14:51:31.380890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789006994902764:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:31.390965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.391970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789005379029586:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24366443ns) {HandleHandshakeStatus:329(24364238ns) {DropIncomingHandshake:442(28624ns) DropOutgoingHandshake:457(291ns) SwitchToState:179(279ns) ProcessPendingSessionEvents:495(17866398ns) {ForwardSessionEventToSession:612(9607561ns) ForwardSessionEventToSession:612(5721ns) ForwardSessionEventToSession:612(3974717ns) ForwardSessionEventToSession:612(4292ns) ForwardSessionEventToSession:612(2930ns) ForwardSessionEventToSession:612(681510ns) ForwardSessionEventToSession:612(5467ns) ForwardSessionEventToSession:612(167034ns) ForwardSessionEventToSession:612(2408ns) ForwardSessionEventToSession:612(3011ns)}}}
2025-11-28T14:51:31.392109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951436:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.391877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:31.391943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789006923783646:2049] [node 2] ICP22 created new session: [5:7577789006923783734:2049]
2025-11-28T14:51:31.391995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789006923783734:2049] [node 2] ICS09 handshake done sender: [5:7577789006923783727:2118] self: [5:7577789006923783729:0] peer: [2:7577789005379029594:0] socket: 140 qp: -1
2025-11-28T14:51:31.392030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789006923783734:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:31.392074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:31.390972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789004557864989:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:31.393155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789005379029622:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:31.393413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:31.394460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789006994902676:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25156417ns) {HandleHandshakeStatus:329(25154672ns) {DropIncomingHandshake:442(664ns) DropOutgoingHandshake:457(6580ns) SwitchToState:179(372ns) ProcessPendingSessionEvents:495(1008720ns) {ForwardSessionEventToSession:612(956883ns) ForwardSessionEventToSession:612(11283ns) ForwardSessionEventToSession:612(5061ns) ForwardSessionEventToSession:612(2446ns) ForwardSessionEventToSession:612(5460ns) ForwardSessionEventToSession:612(2525ns) ForwardSessionEventToSession:612(2794ns) ForwardSessionEventToSession:612(5167ns) ForwardSessionEventToSession:612(1872ns) ForwardSessionEventToSession:612(4904ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.395004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951437:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.395004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789004557864907:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.395097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:31.395150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789004557864907:2049] [node 2] ICP22 created new session: [3:7577789004557865006:2049]
2025-11-28T14:51:31.395615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789004557865006:2049] [node 2] ICS09 handshake done sender: [3:7577789004557864989:2123] self: [3:7577789004557865000:0] peer: [2:7577789005379029588:0] socket: 97 qp: -1
2025-11-28T14:51:31.395649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789004557865006:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:31.395687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:31.400136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789007029951432:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:31.400309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789005379029587:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(29709317ns) {HandleHandshakeStatus:329(29706515ns) {DropIncomingHandshake:442(1801ns) DropOutgoingHandshake:457(1201ns) SwitchToState:179(701ns) ProcessPendingSessionEvents:495(1284911ns) {ForwardSessionEventToSession:612(439622ns) ForwardSessionEventToSession:612(4558ns) ForwardSessionEventToSession:612(796511ns) ForwardSessionEventToSession:612(4734ns) ForwardSessionEventToSession:612(7090ns) ForwardSessionEventToSession:612(2233ns) ForwardSessionEventToSession:612(4799ns) ForwardSessionEventToSession:612(5148ns) ForwardSessionEventToSession:612(1965ns) ForwardSessionEventToSession:612(2720ns)}}}
2025-11-28T14:51:31.401292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789007029951424:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.403692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789007029951354:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.403754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:31.404077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789007029951354:2048] [node 1] ICP22 created new session: [6:7577789007029951439:2048]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:31.408978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789007029951439:2048] [node 1] ICS09 handshake done sender: [6:7577789007029951432:2115] self: [6:7577789007029951434:0] peer: [1:7577789006986309628:0] socket: 136 qp: -1
2025-11-28T14:51:31.409037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789007029951439:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:31.409090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:31.409177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789007029951433:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:31.409290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789006994902766:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2575
2025-11-28T14:51:31.409270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789006923783721:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.420273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789006994902766:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.420166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951438:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.420167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789007029951354:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16123357ns) {HandleHandshakeStatus:329(16120897ns) {DropIncomingHandshake:442(1119ns) DropOutgoingHandshake:457(5780ns) SwitchToState:179(759ns) ProcessPendingSessionEvents:495(10537207ns) {ForwardSessionEventToSession:612(8381158ns) ForwardSessionEventToSession:612(1231664ns) ForwardSessionEventToSession:612(4483ns) ForwardSessionEventToSession:612(2915ns) ForwardSessionEventToSession:612(867899ns) ForwardSessionEventToSession:612(2589ns) ForwardSessionEventToSession:612(13328ns) ForwardSessionEventToSession:612(2745ns) ForwardSessionEventToSession:612(2700ns) ForwardSessionEventToSession:612(5481ns)}}}
2025-11-28T14:51:31.420262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789007029951355:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.420304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:31.420344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789007029951355:2049] [node 2] ICP22 created new session: [6:7577789007029951441:2049]
2025-11-28T14:51:31.420407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789007029951441:2049] [node 2] ICS09 handshake done sender: [6:7577789007029951433:2116] self: [6:7577789007029951435:0] peer: [2:7577789005379029600:0] socket: 142 qp: -1
2025-11-28T14:51:31.420436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789007029951441:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:31.420466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.421944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789006994902768:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2575
2025-11-28T14:51:31.424394Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789007029951422:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.425147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309630:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.425370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789006986309615:2109] [node 6] ICP18 handshake (actor [1:7577789006986309630:2116]) is too old
2025-11-28T14:51:31.454955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789007029951425:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.459330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789004557864994:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.466796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789004557864983:2120] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.483324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789006994902756:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.483519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789006994902678:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.483841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789004557864983:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.484327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789004557864983:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:31.484841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789004557864979:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.491338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789006994902770:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2575
2025-11-28T14:51:31.488669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:31.492157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789004557864979:2107] [node 4] ICP22 created new session: [3:7577789004557865010:2107]
2025-11-28T14:51:31.492528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789004557865010:2107] [node 4] ICS09 handshake done sender: [3:7577789004557864983:2120] self: [3:7577789004557864982:0] peer: [4:7577789006994902769:0] socket: 108 qp: -1
2025-11-28T14:51:31.492628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789004557865010:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:31.492722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.493013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951444:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.496348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789006994902748:2113] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.512965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309633:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.513038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789006994902756:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:31.513187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789006986309613:2107] [node 4] ICP18 handshake (actor [1:7577789006986309633:2118]) is too old
2025-11-28T14:51:31.513599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789004557864979:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(28603169ns) {HandleHandshakeStatus:329(28600962ns) {DropIncomingHandshake:442(827ns) DropOutgoingHandshake:457(737ns) SwitchToState:179(987ns) ProcessPendingSessionEvents:495(20349457ns) {ForwardSessionEventToSession:612(2383286ns) ForwardSessionEventToSession:612(1513218ns) ForwardSessionEventToSession:612(5968ns) ForwardSessionEventToSession:612(2716ns) ForwardSessionEventToSession:612(5930082ns) ForwardSessionEventToSession:612(5410ns) ForwardSessionEventToSession:612(2718ns) ForwardSessionEventToSession:612(2204324ns) ForwardSessionEventToSession:612(3036ns) ForwardSessionEventToSession:612(8600ns)}}}
2025-11-28T14:51:31.515921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789006994902678:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.515998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:31.516051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789006994902678:2050] [node 3] ICP22 created new session: [4:7577789006994902771:2050]
2025-11-28T14:51:31.516091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789006994902771:2050] [node 3] ICS09 handshake done sender: [4:7577789006994902756:2117] self: [4:7577789006994902769:0] peer: [3:7577789004557864982:0] socket: 115 qp: -1
2025-11-28T14:51:31.518125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789006994902771:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:31.518177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.524964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951445:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.538719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789004557864990:2124] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.579547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789006986309635:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.579752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789006923783738:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47382
2025-11-28T14:51:31.583537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789006986309551:2049] [node 3] ICP18 handshake (actor [1:7577789006986309635:2119]) is too old
2025-11-28T14:51:31.586849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789006994902749:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.616051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789006923783722:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.616275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789006923783708:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.616727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789006994902749:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.617458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789006994902749:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.618031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789006994902740:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.627480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789004557864987:2121] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.622312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789006923783740:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47382
2025-11-28T14:51:31.627881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789006923783722:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:31.618095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:31.622508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789006994902740:2108] [node 5] ICP22 created new session: [4:7577789006994902776:2108]
2025-11-28T14:51:31.641405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789006923783708:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3850) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789006923783722:2115] held: no
2025-11-28T14:51:31.623846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789006994902776:2108] [node 5] ICS09 handshake done sender: [4:7577789006994902749:2114] self: [4:7577789006994902746:0] peer: [5:7577789006923783739:0] socket: 117 qp: -1
2025-11-28T14:51:31.644554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789005379029603:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.645899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789006923783708:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.623885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789006994902776:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:31.646561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789005379029523:2049] [node 3] ICP18 handshake (actor [2:7577789005379029603:2117]) is too old
2025-11-28T14:51:31.623937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:31.646203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:31.646584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789006923783708:2108] [node 4] ICP22 created new session: [5:7577789006923783741:2108]
2025-11-28T14:51:31.648912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789006923783741:2108] [node 4] ICS09 handshake done sender: [5:7577789006923783722:2115] self: [5:7577789006923783739:0] peer: [4:7577789006994902746:0] socket: 121 qp: -1
2025-11-28T14:51:31.649138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789006923783741:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:31.652908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:31.655152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789006994902747:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.673653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789005379029605:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.673889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789005379029583:2107] [node 4] ICP18 handshake (actor [2:7577789005379029605:2118]) is too old
2025-11-28T14:51:31.675618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789006923783724:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.692236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789007029951438:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.692477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789007029951416:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.692731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789006923783724:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.693170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789006923783724:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.693494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789006923783709:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.693485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951447:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.693623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:31.693671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789006923783709:2109] [node 6] ICP22 created new session: [5:7577789006923783744:2109]
2025-11-28T14:51:31.693713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789006923783744:2109] [node 6] ICS09 handshake done sender: [5:7577789006923783724:2116] self: [5:7577789006923783723:0] peer: [6:7577789007029951446:0] socket: 102 qp: -1
2025-11-28T14:51:31.693788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789007029951438:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:31.693755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789006923783744:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:31.693818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:31.694351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789007029951416:2103] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5817) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789007029951438:2119] held: no
2025-11-28T14:51:31.694476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789007029951416:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.694641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:31.694695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789007029951416:2103] [node 5] ICP22 created new session: [6:7577789007029951448:2103]
2025-11-28T14:51:31.694746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789007029951448:2103] [node 5] ICS09 handshake done sender: [6:7577789007029951438:2119] self: [6:7577789007029951446:0] peer: [5:7577789006923783723:0] socket: 148 qp: -1
2025-11-28T14:51:31.694776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789007029951448:2103] [node 5] ICS10 traffic start
2025-11-28T14:51:31.694819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:31.696410Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789007029951431:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.697253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789006994902768:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.728406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789004557864993:2125] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.741977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789004557864993:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.741309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789007029951444:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.742546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789004557864993:2125] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:31.741558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789007029951356:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.744325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789007029951356:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4950) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789007029951444:2120] held: no
2025-11-28T14:51:31.743370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789004557864980:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.747194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789006994902757:2118] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.744919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951451:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.743846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:31.747803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789007029951444:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:31.743920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789004557864980:2110] [node 6] ICP22 created new session: [3:7577789004557865015:2110]
2025-11-28T14:51:31.743975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789004557865015:2110] [node 6] ICS09 handshake done sender: [3:7577789004557864993:2125] self: [3:7577789004557864986:0] peer: [6:7577789007029951450:0] socket: 152 qp: -1
2025-11-28T14:51:31.744396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789004557865015:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:31.744894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:31.748097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789007029951356:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.748145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:31.753390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789007029951356:2050] [node 3] ICP22 created new session: [6:7577789007029951452:2050]
2025-11-28T14:51:31.753393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789007029951445:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.757236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789007029951417:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.757316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789007029951452:2050] [node 3] ICS09 handshake done sender: [6:7577789007029951444:2120] self: [6:7577789007029951450:0] peer: [3:7577789004557864986:0] socket: 153 qp: -1
2025-11-28T14:51:31.757591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789007029951452:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:31.757660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:31.757637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789006994902757:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.758180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789006994902757:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.758588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789007029951455:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9550
2025-11-28T14:51:31.758406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789006994902741:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.758904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:31.759191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789006994902741:2109] [node 6] ICP22 created new session: [4:7577789006994902780:2109]
2025-11-28T14:51:31.759378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789007029951417:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3850) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789007029951445:2121] held: no
2025-11-28T14:51:31.759589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789007029951445:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:31.759340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789006994902780:2109] [node 6] ICS09 handshake done sender: [4:7577789006994902757:2118] self: [4:7577789006994902755:0] peer: [6:7577789007029951453:0] socket: 116 qp: -1
2025-11-28T14:51:31.759546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789006994902780:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:31.760114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789007029951417:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.759710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:31.760328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:31.760386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789007029951417:2108] [node 4] ICP22 created new session: [6:7577789007029951456:2108]
2025-11-28T14:51:31.760478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789007029951456:2108] [node 4] ICS09 handshake done sender: [6:7577789007029951445:2121] self: [6:7577789007029951453:0] peer: [4:7577789006994902755:0] socket: 119 qp: -1
2025-11-28T14:51:31.760518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789007029951456:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:31.760555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:31.763025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789004557864998:2128] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:31.763853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789006923783738:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.764057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789006923783647:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:31.764314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789004557864998:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:31.764925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789004557864998:2128] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:31.765127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789004557864981:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:31.765224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:31.765277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789004557864981:2108] [node 5] ICP22 created new session: [3:7577789004557865018:2108]
2025-11-28T14:51:31.765315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789004557865018:2108] [node 5] ICS09 handshake done sender: [3:7577789004557864998:2128] self: [3:7577789004557864997:0] peer: [5:7577789006923783746:0] socket: 154 qp: -1
2025-11-28T14:51:31.765846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789004557865018:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:31.765914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:31.766043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789006923783647:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4950) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789006923783738:2121] held: no
2025-11-28T14:51:31.766424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789006923783747:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47382
2025-11-28T14:51:31.767145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789006923783738:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:31.767429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789006923783647:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:31.767498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:31.767544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789006923783647:2050] [node 3] ICP22 created new session: [5:7577789006923783748:2050]
2025-11-28T14:51:31.767595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789006923783748:2050] [node 3] ICS09 handshake done sender: [5:7577789006923783738:2121] self: [5:7577789006923783746:0] peer: [3:7577789004557864997:0] socket: 155 qp: -1
2025-11-28T14:51:31.767632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789006923783748:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:31.767681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:31.787351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:31.787629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789005379029612:2048] [node 1] ICS01 socket: 86 reason# EndOfStream
2025-11-28T14:51:31.787677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789005379029522:2048] [node 1] ICP30 unregister session Session# [2:7577789005379029612:2048] VirtualId# [2:7577789005379029604:0]
2025-11-28T14:51:31.787719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789005379029612:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.787874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:31.787940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:31.788701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:31.788909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789004557865004:2048] [node 1] ICS07 socket disconnect 96 reason# EndOfStream
2025-11-28T14:51:31.788959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789004557865004:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.789051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:31.789095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789004557865004:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:31.789973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789004557864906:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:31.790013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789004557865004:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:31.790033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789004557864906:2048] [node 1] ICP30 unregister session Session# [3:7577789004557865004:2048] VirtualId# [3:7577789004557864999:0]
2025-11-28T14:51:31.790065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:31.790082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789004557864906:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.790089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.791182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:31.791322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789006994902764:2048] [node 1] ICS07 socket disconnect 94 reason# EndOfStream
2025-11-28T14:51:31.791345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789006994902764:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.791398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:31.791447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789006994902764:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:31.792057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789006994902676:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:31.792150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789006994902764:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:31.792182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789006994902676:2048] [node 1] ICP30 unregister session Session# [4:7577789006994902764:2048] VirtualId# [4:7577789006994902758:0]
2025-11-28T14:51:31.792219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:31.792244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789006994902676:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.792255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.792724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:31.792898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789006923783730:2048] [node 1] ICS01 socket: 131 reason# EndOfStream
2025-11-28T14:51:31.792964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789006923783645:2048] [node 1] ICP30 unregister session Session# [5:7577789006923783730:2048] VirtualId# [5:7577789006923783726:0]
2025-11-28T14:51:31.792996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789006923783730:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.793068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:31.793116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:31.796564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:31.796967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789007029951439:2048] [node 1] ICS07 socket disconnect 136 reason# EndOfStream
2025-11-28T14:51:31.797036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789007029951439:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.797201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:31.797300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789007029951439:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:31.799237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789007029951354:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:31.799327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789007029951439:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:31.799380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789007029951354:2048] [node 1] ICP30 unregister session Session# [6:7577789007029951439:2048] VirtualId# [6:7577789007029951434:0]
2025-11-28T14:51:31.799451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:31.799496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789007029951354:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.799513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26148) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.800892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789004557864906:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:31.801748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789006994902676:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:31.809308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789007029951354:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:31.839345Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:31.839519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789004557865006:2049] [node 2] ICS07 socket disconnect 97 reason# EndOfStream
2025-11-28T14:51:31.839544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789004557865006:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.839599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:31.839629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789004557865006:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:31.842102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789004557864907:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:31.842152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789004557865006:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:31.842972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789004557864907:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:31.843067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789004557865006:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:31.843107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789004557864907:2049] [node 2] ICP30 unregister session Session# [3:7577789004557865006:2049] VirtualId# [3:7577789004557865000:0]
2025-11-28T14:51:31.843201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:31.843247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789004557864907:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.843274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.844005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:31.844187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789006994902762:2049] [node 2] ICS07 socket disconnect 95 reason# EndOfStream
2025-11-28T14:51:31.844226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789006994902762:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.844304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:31.844379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789006994902762:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:31.844848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:31.844959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789006994902677:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:31.845053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789006994902762:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:31.845122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789006923783734:2049] [node 2] ICS01 socket: 140 reason# EndOfStream
2025-11-28T14:51:31.845102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789006994902677:2049] [node 2] ICP30 unregister session Session# [4:7577789006994902762:2049] VirtualId# [4:7577789006994902759:0]
2025-11-28T14:51:31.845167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789006923783646:2049] [node 2] ICP30 unregister session Session# [5:7577789006923783734:2049] VirtualId# [5:7577789006923783729:0]
2025-11-28T14:51:31.845191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789006923783734:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.845154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:31.845181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789006994902677:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.845193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.845269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:31.845324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:31.845704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:31.845834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789007029951441:2049] [node 2] ICS07 socket disconnect 142 reason# EndOfStream
2025-11-28T14:51:31.845861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789007029951441:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.845929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:31.845961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789007029951441:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:31.846394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789007029951355:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:31.846488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789007029951441:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:31.846531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789007029951355:2049] [node 2] ICP30 unregister session Session# [6:7577789007029951441:2049] VirtualId# [6:7577789007029951435:0]
2025-11-28T14:51:31.846563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:31.846607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789007029951355:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.846623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.855052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789006994902677:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:31.856098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789007029951355:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:31.856707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:31.856979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789006923783748:2050] [node 3] ICS01 socket: 155 reason# EndOfStream
2025-11-28T14:51:31.857028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789006923783647:2050] [node 3] ICP30 unregister session Session# [5:7577789006923783748:2050] VirtualId# [5:7577789006923783746:0]
2025-11-28T14:51:31.857059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789006923783748:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.857131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:31.857213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:31.857538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:31.857676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789007029951452:2050] [node 3] ICS01 socket: 153 reason# EndOfStream
2025-11-28T14:51:31.857711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789007029951356:2050] [node 3] ICP30 unregister session Session# [6:7577789007029951452:2050] VirtualId# [6:7577789007029951450:0]
2025-11-28T14:51:31.857728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789007029951452:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.857785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:31.857817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:31.858752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:31.858954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789006994902771:2050] [node 3] ICS07 socket disconnect 115 reason# EndOfStream
2025-11-28T14:51:31.858984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789006994902771:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.859071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:31.859126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789006994902771:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:31.859995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789006994902678:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4950) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:31.860041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789006994902771:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:31.860129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789006994902678:2050] [node 3] ICP30 unregister session Session# [4:7577789006994902771:2050] VirtualId# [4:7577789006994902769:0]
2025-11-28T14:51:31.860179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:31.860219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789006994902678:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4950) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.860243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4950) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:31.867374Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:31.867556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789006923783741:2108] [node 4] ICS01 socket: 121 reason# EndOfStream
2025-11-28T14:51:31.867592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789006923783708:2108] [node 4] ICP30 unregister session Session# [5:7577789006923783741:2108] VirtualId# [5:7577789006923783739:0]
2025-11-28T14:51:31.867635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789006923783741:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.867723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:31.867766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:31.867983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:31.868183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789007029951456:2108] [node 4] ICS01 socket: 119 reason# EndOfStream
2025-11-28T14:51:31.868234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789007029951417:2108] [node 4] ICP30 unregister session Session# [6:7577789007029951456:2108] VirtualId# [6:7577789007029951453:0]
2025-11-28T14:51:31.868259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789007029951456:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.868324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:31.868389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:31.885614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:31.885778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789007029951448:2103] [node 5] ICS01 socket: 148 reason# EndOfStream
2025-11-28T14:51:31.885811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789007029951416:2103] [node 5] ICP30 unregister session Session# [6:7577789007029951448:2103] VirtualId# [6:7577789007029951446:0]
2025-11-28T14:51:31.885831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789007029951448:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:31.885913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:31.885986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:31.926290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789006358649187:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:31.926298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789006358649186:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:31.947122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789004692799157:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:31.947157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789004692799158:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:31.967941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789004786363071:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:31.968375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789004786363072:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:31.988496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789003808820007:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:31.988555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789003808820008:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:31.988664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789003808820006:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:32.007398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008474134143:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:32.007428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008474134145:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:32.007717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008474134144:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:32.025685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789008027681871:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:32.025699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789008027681872:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:32.028154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789008027681873:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:32.138323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789008027681934:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:32.138643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789010653616546:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:32.138732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789010653616547:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:32.138853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789008987766516:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:32.139057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789008027681935:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:32.139784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789008103787365:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:32.139931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789010653616548:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:32.139934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789009081330432:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:32.140158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789009081330434:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:32.140490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789008103787366:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:32.140288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008474134206:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:32.140919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789008987766517:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:32.140798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789009081330433:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:32.141552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789008987766518:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:32.141253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008474134207:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:32.144796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789006358649187:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(60616174ns) {Configure:104(60610589ns) {ProcessConfigured:135(104432ns) {StartInitialHandshake:157(102322ns) {DropHandshakes:470(2660ns) {DropIncomingHandshake:442(621ns) DropOutgoingHandshake:457(395ns)} GenerateSessionVirtualId:988(973ns) PrepareNewSessionHandshake:477(1587ns) {SwitchToState:179(684ns)}}}}}
2025-11-28T14:51:32.145932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789006358649186:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(60925535ns) {Configure:104(60919336ns) {ProcessConfigured:135(71559ns) {StartInitialHandshake:157(69755ns) {DropHandshakes:470(1196ns) {DropIncomingHandshake:442(401ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(430ns) PrepareNewSessionHandshake:477(767ns) {SwitchToState:179(307ns)}}}}}
2025-11-28T14:51:32.146738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789008027681872:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45236912ns) {Configure:104(45232608ns) {ProcessConfigured:135(719874ns) {StartInitialHandshake:157(718140ns) {DropHandshakes:470(221063ns) {DropIncomingHandshake:442(294ns) DropOutgoingHandshake:457(451ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:477(800ns) {SwitchToState:179(368ns)}}}}}
2025-11-28T14:51:32.147676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789003808820006:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(57614683ns) {Configure:104(57611029ns) {ProcessConfigured:135(1533433ns) {StartInitialHandshake:157(1531874ns) {DropHandshakes:470(11345ns) {DropIncomingHandshake:442(293ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(1678ns) {SwitchToState:179(78ns)}}}}}
2025-11-28T14:51:32.150289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789008474134144:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22335960ns) {Configure:104(22330471ns) {ProcessConfigured:135(78814ns) {StartInitialHandshake:157(77168ns) {DropHandshakes:470(1349ns) {DropIncomingHandshake:442(359ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(848ns) {SwitchToState:179(364ns)}}}}}
2025-11-28T14:51:32.147673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789003808820007:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56742815ns) {Configure:104(56735308ns) {ProcessConfigured:135(1144522ns) {StartInitialHandshake:157(1142694ns) {DropHandshakes:470(1811ns) {DropIncomingHandshake:442(383ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(492ns) PrepareNewSessionHandshake:477(1972ns) {SwitchToState:179(1493ns)}}}}}
2025-11-28T14:51:32.150460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789003808820008:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20711339ns) {Configure:104(13478927ns) {ProcessConfigured:135(884982ns) {StartInitialHandshake:157(882158ns) {DropHandshakes:470(1316ns) {DropIncomingHandshake:442(494ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(1063ns) {SwitchToState:179(140ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.152772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789004786363071:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(61173825ns) {Configure:104(61168669ns) {ProcessConfigured:135(508172ns) {StartInitialHandshake:157(506498ns) {DropHandshakes:470(1146ns) {DropIncomingHandshake:442(336ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(474ns) PrepareNewSessionHandshake:477(1497ns) {SwitchToState:179(148ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.154246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789008103787378:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28912
2025-11-28T14:51:32.153034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789004786363072:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51665420ns) {Configure:104(44274203ns) {ProcessConfigured:135(1466576ns) {StartInitialHandshake:157(1465421ns) {DropHandshakes:470(1299ns) {DropIncomingHandshake:442(333ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(1077ns) {SwitchToState:179(191ns)}}}}}
2025-11-28T14:51:32.163466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008474134221:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46367
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.161501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681949:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.163931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789004692799158:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17879603ns) {Configure:104(17871388ns) {ProcessConfigured:135(517065ns) {StartInitialHandshake:157(514748ns) {DropHandshakes:470(1324ns) {DropIncomingHandshake:442(373ns) DropOutgoingHandshake:457(185ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(1510ns) {SwitchToState:179(536ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.169074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008474134222:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46367
2025-11-28T14:51:32.169614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789008027681949:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.169891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789009081330450:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23834
No RDMA context found for GID: 2025-11-28T14:51:32.170395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789008103787378:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.169623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008474134221:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:51:32.171610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789009081330445:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23834
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.174221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789008027681871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.174644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789009081330450:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.174263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789008987766527:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42271
2025-11-28T14:51:32.174687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008474134222:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.174640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789010653616553:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.176077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789008103787381:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28912
2025-11-28T14:51:32.176686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681951:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-28T14:51:32.175079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789010653616553:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.175011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789009081330445:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.178130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789008987766533:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42271
2025-11-28T14:51:32.179308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789008103787383:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28912
2025-11-28T14:51:32.174914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008474134223:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46367
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.178127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681950:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-28T14:51:32.176075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789010653616563:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22645
2025-11-28T14:51:32.178121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789009081330449:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23834
2025-11-28T14:51:32.176192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789008474134207:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.179609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789008987766527:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.176586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789010653616562:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22645
2025-11-28T14:51:32.179396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789010653616563:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.179569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789008103787372:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.178533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681952:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-28T14:51:32.186900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789004692799157:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.185199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789010653616553:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:32.179601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789008103787372:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.179391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789008027681950:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.188124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789010653616546:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.187042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789008103787374:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.185144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789008027681935:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.188257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789008987766521:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.188171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.187066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789008103787374:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.188271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789010653616546:2109] [node 6] ICP22 created new session: [1:7577789010653616566:2109]
2025-11-28T14:51:32.188435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789008987766521:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.187144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789008027681952:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.187165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789008103787382:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28912
2025-11-28T14:51:32.188268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789010653616555:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.191416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789008987766521:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.187770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789008027681872:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.188079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789008103787372:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:32.188297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789010653616555:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.192386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008474134226:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46367
2025-11-28T14:51:32.191353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789008103787387:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28912
2025-11-28T14:51:32.194487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681956:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-28T14:51:32.196203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789008987766517:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.190114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789010653616566:2109] [node 6] ICS09 handshake done sender: [1:7577789010653616553:2112] self: [1:7577789010653616551:0] peer: [6:7577789008027681953:0] socket: 91 qp: -1
2025-11-28T14:51:32.191543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789008103787365:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.196941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.191813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789008103787374:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:32.197429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789008987766517:2109] [node 6] ICP22 created new session: [2:7577789008987766537:2109]
2025-11-28T14:51:32.190508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789010653616566:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.199323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789008987766537:2109] [node 6] ICS09 handshake done sender: [2:7577789008987766521:2110] self: [2:7577789008987766519:0] peer: [6:7577789008027681955:0] socket: 123 qp: -1
2025-11-28T14:51:32.191925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.199706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789008987766537:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:32.190547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:32.203615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008474134222:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:32.192093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789008103787365:2108] [node 5] ICP22 created new session: [4:7577789008103787389:2108]
2025-11-28T14:51:32.191546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789010653616555:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.199095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789008027681935:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789008027681950:2116] held: no
2025-11-28T14:51:32.202914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:32.192276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789008103787389:2108] [node 5] ICS09 handshake done sender: [4:7577789008103787372:2110] self: [4:7577789008103787371:0] peer: [5:7577789008474134225:0] socket: 100 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.206184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789008474134207:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.192474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789006358649186:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.199583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789008027681949:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:32.192315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789008103787389:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:32.208913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.192525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.204173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789008027681871:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.192355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.205074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789004692799157:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789008987766527:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.194818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789006358649186:2048] [node 2] ICP22 created new session: [1:7577789010653616570:2048]
2025-11-28T14:51:32.194959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789008103787366:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.204679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681958:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-28T14:51:32.208577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789008987766536:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42271
2025-11-28T14:51:32.195264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789010653616570:2048] [node 2] ICS09 handshake done sender: [1:7577789010653616555:2113] self: [1:7577789010653616552:0] peer: [2:7577789008987766534:0] socket: 92 qp: -1
2025-11-28T14:51:32.209230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008474134207:2108] [node 4] ICP22 created new session: [5:7577789008474134228:2108]
2025-11-28T14:51:32.198940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.204755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681957:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-28T14:51:32.209057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789008987766538:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42271
2025-11-28T14:51:32.198990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789008103787366:2109] [node 6] ICP22 created new session: [4:7577789008103787392:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.196603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789010653616570:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.209475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789008987766527:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:32.200458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789008103787392:2109] [node 6] ICS09 handshake done sender: [4:7577789008103787374:2112] self: [4:7577789008103787370:0] peer: [6:7577789008027681954:0] socket: 104 qp: -1
2025-11-28T14:51:32.210359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008474134228:2108] [node 4] ICS09 handshake done sender: [5:7577789008474134222:2116] self: [5:7577789008474134225:0] peer: [4:7577789008103787371:0] socket: 105 qp: -1
2025-11-28T14:51:32.205636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.212132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789008987766538:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.197043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:32.208166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789008027681871:2048] [node 1] ICP22 created new session: [6:7577789008027681959:2048]
2025-11-28T14:51:32.210406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008474134228:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:32.212396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789004692799157:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.210352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789010653616573:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22645
2025-11-28T14:51:32.203578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789008103787392:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:32.209044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789008027681950:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:32.210851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008474134227:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46367
2025-11-28T14:51:32.205204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:32.209057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789008027681952:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:32.211542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:32.209087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789008027681959:2048] [node 1] ICS09 handshake done sender: [6:7577789008027681949:2115] self: [6:7577789008027681953:0] peer: [1:7577789010653616551:0] socket: 97 qp: -1
2025-11-28T14:51:32.214187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.210390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789006358649186:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(16414655ns) {HandleHandshakeStatus:329(16412430ns) {DropIncomingHandshake:442(13201ns) DropOutgoingHandshake:457(289ns) SwitchToState:179(330ns) ProcessPendingSessionEvents:495(9216385ns) {ForwardSessionEventToSession:612(5066172ns) ForwardSessionEventToSession:612(489021ns) ForwardSessionEventToSession:612(4806ns) ForwardSessionEventToSession:612(3354ns) ForwardSessionEventToSession:612(393705ns) ForwardSessionEventToSession:612(3249ns) ForwardSessionEventToSession:612(328200ns) ForwardSessionEventToSession:612(5624ns) ForwardSessionEventToSession:612(2324ns) ForwardSessionEventToSession:612(2900220ns)}}}
2025-11-28T14:51:32.212874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008474134229:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46367
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.209137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789008027681959:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:32.216759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789004692799157:2048] [node 1] ICP22 created new session: [2:7577789008987766543:2048]
2025-11-28T14:51:32.215843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008474134227:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.217133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789008987766543:2048] [node 1] ICS09 handshake done sender: [2:7577789008987766527:2114] self: [2:7577789008987766534:0] peer: [1:7577789010653616552:0] socket: 102 qp: -1
2025-11-28T14:51:32.210137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789008027681872:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.210369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789008027681935:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.219805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789008987766543:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:32.220237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789009081330453:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23834
2025-11-28T14:51:32.219893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:32.210381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:32.216208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008474134229:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.211824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.219729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789008474134143:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.220621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789010653616558:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.212000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.220532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789008474134144:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.220675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789010653616558:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.212031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789008027681872:2049] [node 2] ICP22 created new session: [6:7577789008027681961:2049]
2025-11-28T14:51:32.212378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789008027681935:2108] [node 4] ICP22 created new session: [6:7577789008027681962:2108]
2025-11-28T14:51:32.222701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789008987766526:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.212415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789008027681961:2049] [node 2] ICS09 handshake done sender: [6:7577789008027681952:2118] self: [6:7577789008027681955:0] peer: [2:7577789008987766519:0] socket: 128 qp: -1
2025-11-28T14:51:32.222737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789008987766526:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.226172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789008987766526:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:32.212965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789008027681961:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:32.213911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.226742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789010653616558:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:32.216652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789008027681962:2108] [node 4] ICS09 handshake done sender: [6:7577789008027681950:2116] self: [6:7577789008027681954:0] peer: [4:7577789008103787370:0] socket: 106 qp: -1
2025-11-28T14:51:32.216784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789008027681962:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:32.217027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.227148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008474134233:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46367
2025-11-28T14:51:32.227264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789008987766518:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.227343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:32.227451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008474134234:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46367
2025-11-28T14:51:32.227891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008474134227:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:32.228174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789008474134143:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.227389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789008987766518:2108] [node 5] ICP22 created new session: [2:7577789008987766546:2108]
2025-11-28T14:51:32.229081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789010653616548:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.228218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.227489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789008987766546:2108] [node 5] ICS09 handshake done sender: [2:7577789008987766526:2113] self: [2:7577789008987766522:0] peer: [5:7577789008474134232:0] socket: 90 qp: -1
2025-11-28T14:51:32.228257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008474134143:2048] [node 1] ICP22 created new session: [5:7577789008474134235:2048]
2025-11-28T14:51:32.229182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:32.227538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789008987766546:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:32.228292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008474134235:2048] [node 1] ICS09 handshake done sender: [5:7577789008474134227:2119] self: [5:7577789008474134231:0] peer: [1:7577789010653616556:0] socket: 147 qp: -1
2025-11-28T14:51:32.229311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789010653616548:2108] [node 5] ICP22 created new session: [1:7577789010653616575:2108]
2025-11-28T14:51:32.229199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.228351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008474134235:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:32.229367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789010653616575:2108] [node 5] ICS09 handshake done sender: [1:7577789010653616558:2115] self: [1:7577789010653616556:0] peer: [5:7577789008474134231:0] socket: 146 qp: -1
2025-11-28T14:51:32.229407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789010653616575:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:32.229444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.228416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:32.229281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008474134229:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:32.229564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789008474134144:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.229608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.229645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008474134144:2049] [node 2] ICP22 created new session: [5:7577789008474134237:2049]
2025-11-28T14:51:32.229696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008474134237:2049] [node 2] ICS09 handshake done sender: [5:7577789008474134229:2120] self: [5:7577789008474134232:0] peer: [2:7577789008987766522:0] socket: 111 qp: -1
2025-11-28T14:51:32.229729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008474134237:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:32.229754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:32.234223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789008474134214:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.245086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789010653616562:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.245320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789010653616548:2108] [node 5] ICP18 handshake (actor [1:7577789010653616562:2116]) is too old
2025-11-28T14:51:32.245846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789008474134215:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.253538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789008103787381:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.254662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789008103787365:2108] [node 5] ICP18 handshake (actor [4:7577789008103787381:2116]) is too old
2025-11-28T14:51:32.254707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789009081330454:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23834
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.255594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681966:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-28T14:51:32.259635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789008474134211:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.267244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789008987766533:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.267499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789008987766518:2108] [node 5] ICP18 handshake (actor [2:7577789008987766533:2116]) is too old
2025-11-28T14:51:32.269896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789010653616550:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.277522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789009081330449:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.277699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789004786363071:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.278097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789010653616550:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.278754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789009081330456:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23834
2025-11-28T14:51:32.279022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789010653616550:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:32.279245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789006358649187:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.279338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789009081330449:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:32.279362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.279437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789006358649187:2049] [node 3] ICP22 created new session: [1:7577789010653616578:2049]
2025-11-28T14:51:32.279503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789010653616578:2049] [node 3] ICS09 handshake done sender: [1:7577789010653616550:2111] self: [1:7577789010653616549:0] peer: [3:7577789009081330455:0] socket: 94 qp: -1
2025-11-28T14:51:32.279647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789010653616578:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:32.279700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:32.279794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789004786363071:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.279901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.279992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789004786363071:2048] [node 1] ICP22 created new session: [3:7577789009081330457:2048]
2025-11-28T14:51:32.280046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789009081330457:2048] [node 1] ICS09 handshake done sender: [3:7577789009081330449:2118] self: [3:7577789009081330455:0] peer: [1:7577789010653616549:0] socket: 108 qp: -1
2025-11-28T14:51:32.280081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789009081330457:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:32.280167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.280936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789008987766548:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42271
2025-11-28T14:51:32.297349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789009081330439:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.305456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789008027681951:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.305631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789008027681873:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.305849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789009081330439:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.306370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789009081330439:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.306600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789009081330433:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.306692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.306646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681968:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-28T14:51:32.306722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789009081330433:2109] [node 6] ICP22 created new session: [3:7577789009081330461:2109]
2025-11-28T14:51:32.306773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789009081330461:2109] [node 6] ICS09 handshake done sender: [3:7577789009081330439:2113] self: [3:7577789009081330435:0] peer: [6:7577789008027681967:0] socket: 114 qp: -1
2025-11-28T14:51:32.306809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789009081330461:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:32.306842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:32.306901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789008027681951:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:32.307131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789008027681873:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.307497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.307807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789008027681873:2050] [node 3] ICP22 created new session: [6:7577789008027681969:2050]
2025-11-28T14:51:32.307870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789008027681969:2050] [node 3] ICS09 handshake done sender: [6:7577789008027681951:2117] self: [6:7577789008027681967:0] peer: [3:7577789009081330435:0] socket: 122 qp: -1
2025-11-28T14:51:32.307910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789008027681969:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:32.308001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:32.316677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789009081330441:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.329258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789009081330441:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.329685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789009081330441:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.329882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789009081330432:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.329945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:32.330000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789009081330432:2108] [node 5] ICP22 created new session: [3:7577789009081330464:2108]
2025-11-28T14:51:32.330056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789009081330464:2108] [node 5] ICS09 handshake done sender: [3:7577789009081330441:2114] self: [3:7577789009081330437:0] peer: [5:7577789008474134241:0] socket: 116 qp: -1
2025-11-28T14:51:32.330090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789009081330464:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:32.330137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.328685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008474134223:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.328956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789008474134145:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.330196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008474134242:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46367
2025-11-28T14:51:32.330684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008474134223:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:32.330823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789008474134145:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.330946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.330998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008474134145:2050] [node 3] ICP22 created new session: [5:7577789008474134243:2050]
2025-11-28T14:51:32.331123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008474134243:2050] [node 3] ICS09 handshake done sender: [5:7577789008474134223:2117] self: [5:7577789008474134241:0] peer: [3:7577789009081330437:0] socket: 117 qp: -1
2025-11-28T14:51:32.331156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008474134243:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:32.331205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:32.341567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789008987766525:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.363578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789008103787383:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.363744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789003808820007:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.363943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789008987766525:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.364427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789008987766525:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.364722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789008987766516:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.364670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789008103787395:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28912
2025-11-28T14:51:32.364832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.364899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789008987766516:2107] [node 4] ICP22 created new session: [2:7577789008987766550:2107]
2025-11-28T14:51:32.364892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789008103787383:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:32.364956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789008987766550:2107] [node 4] ICS09 handshake done sender: [2:7577789008987766525:2112] self: [2:7577789008987766524:0] peer: [4:7577789008103787394:0] socket: 127 qp: -1
2025-11-28T14:51:32.365091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789008987766550:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:32.365035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789003808820007:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.365390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:32.365277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.365443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789003808820007:2049] [node 2] ICP22 created new session: [4:7577789008103787396:2049]
2025-11-28T14:51:32.365497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789008103787396:2049] [node 2] ICS09 handshake done sender: [4:7577789008103787383:2118] self: [4:7577789008103787394:0] peer: [2:7577789008987766524:0] socket: 131 qp: -1
2025-11-28T14:51:32.365531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789008103787396:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:32.365576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:32.369897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789009081330443:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.380720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789008103787382:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.380938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789003808820008:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.381136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789009081330443:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.381602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789009081330443:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.381818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789009081330434:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.381757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789008103787399:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28912
2025-11-28T14:51:32.381998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.382055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789009081330434:2107] [node 4] ICP22 created new session: [3:7577789009081330467:2107]
2025-11-28T14:51:32.382088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789008103787382:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:32.382101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789009081330467:2107] [node 4] ICS09 handshake done sender: [3:7577789009081330443:2115] self: [3:7577789009081330436:0] peer: [4:7577789008103787398:0] socket: 121 qp: -1
2025-11-28T14:51:32.382472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789003808820008:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.382539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.382139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789009081330467:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:32.382341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:32.382701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789003808820008:2050] [node 3] ICP22 created new session: [4:7577789008103787400:2050]
2025-11-28T14:51:32.382795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789008103787400:2050] [node 3] ICS09 handshake done sender: [4:7577789008103787382:2117] self: [4:7577789008103787398:0] peer: [3:7577789009081330436:0] socket: 125 qp: -1
2025-11-28T14:51:32.383053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789008103787400:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:32.383118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:32.383365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789003808820008:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:32.384832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789010653616557:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.410453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789008103787387:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.410690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789003808820006:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.410962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789010653616557:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.411625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789010653616557:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.411779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789008103787403:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28912
2025-11-28T14:51:32.411866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789010653616547:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.412053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.412110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789010653616547:2107] [node 4] ICP22 created new session: [1:7577789010653616581:2107]
2025-11-28T14:51:32.412176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789010653616581:2107] [node 4] ICS09 handshake done sender: [1:7577789010653616557:2114] self: [1:7577789010653616554:0] peer: [4:7577789008103787402:0] socket: 137 qp: -1
2025-11-28T14:51:32.412096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789008103787387:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:32.412230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789010653616581:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:32.412257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789003808820006:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.412323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:32.412551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.412615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789003808820006:2048] [node 1] ICP22 created new session: [4:7577789008103787404:2048]
2025-11-28T14:51:32.412673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789008103787404:2048] [node 1] ICS09 handshake done sender: [4:7577789008103787387:2119] self: [4:7577789008103787402:0] peer: [1:7577789010653616554:0] socket: 141 qp: -1
2025-11-28T14:51:32.412707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789008103787404:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:32.412765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:32.413840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789008103787376:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.418425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789010653616573:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.418613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789010653616547:2107] [node 4] ICP18 handshake (actor [1:7577789010653616573:2118]) is too old
2025-11-28T14:51:32.427126Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789008987766530:2115] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.435334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789009081330453:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.435591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789004786363072:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.435856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789008987766530:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.436356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789008987766530:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.436613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789004692799158:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.436794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.436785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789009081330470:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23834
2025-11-28T14:51:32.436878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789004692799158:2049] [node 3] ICP22 created new session: [2:7577789008987766553:2049]
2025-11-28T14:51:32.436927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789008987766553:2049] [node 3] ICS09 handshake done sender: [2:7577789008987766530:2115] self: [2:7577789008987766528:0] peer: [3:7577789009081330469:0] socket: 148 qp: -1
2025-11-28T14:51:32.436963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789008987766553:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:32.437025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:32.437059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789009081330453:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:32.437199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789004786363072:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.437254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.437371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789004786363072:2049] [node 2] ICP22 created new session: [3:7577789009081330471:2049]
2025-11-28T14:51:32.437445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789009081330471:2049] [node 2] ICS09 handshake done sender: [3:7577789009081330453:2120] self: [3:7577789009081330469:0] peer: [2:7577789008987766528:0] socket: 149 qp: -1
2025-11-28T14:51:32.437486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789009081330471:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:32.437526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:32.438785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789008474134217:2114] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.446967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789009081330454:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.447182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789009081330432:2108] [node 5] ICP18 handshake (actor [3:7577789009081330454:2121]) is too old
2025-11-28T14:51:32.451029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789008474134216:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.459894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789008027681966:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.460140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789008027681934:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.460511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008474134216:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.461036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008474134216:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.461329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789008027681972:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-28T14:51:32.461253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789008474134206:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.461593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.461682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789008027681966:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:32.461643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008474134206:2109] [node 6] ICP22 created new session: [5:7577789008474134246:2109]
2025-11-28T14:51:32.461701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008474134246:2109] [node 6] ICS09 handshake done sender: [5:7577789008474134216:2113] self: [5:7577789008474134212:0] peer: [6:7577789008027681971:0] socket: 115 qp: -1
2025-11-28T14:51:32.462103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789008027681934:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8117) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789008027681966:2122] held: no
2025-11-28T14:51:32.461738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008474134246:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:32.461840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:32.462988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789008027681934:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.463287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:32.463337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789008027681934:2107] [node 5] ICP22 created new session: [6:7577789008027681973:2107]
2025-11-28T14:51:32.463397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789008027681973:2107] [node 5] ICS09 handshake done sender: [6:7577789008027681966:2122] self: [6:7577789008027681971:0] peer: [5:7577789008474134212:0] socket: 119 qp: -1
2025-11-28T14:51:32.463436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789008027681973:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:32.463479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.464757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789009081330444:2116] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:32.466390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789008987766548:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.466569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789004692799158:2049] [node 3] ICP18 handshake (actor [2:7577789008987766548:2119]) is too old
2025-11-28T14:51:32.493385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:32.493656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789008987766543:2048] [node 1] ICS01 socket: 102 reason# EndOfStream
2025-11-28T14:51:32.493703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789004692799157:2048] [node 1] ICP30 unregister session Session# [2:7577789008987766543:2048] VirtualId# [2:7577789008987766534:0]
2025-11-28T14:51:32.493750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789008987766543:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.493862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:32.493942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:32.494583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:32.494816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789009081330457:2048] [node 1] ICS07 socket disconnect 108 reason# EndOfStream
2025-11-28T14:51:32.494850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789009081330457:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.494931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:32.494988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789009081330457:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:32.495826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789004786363071:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.496093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789009081330457:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.496149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789004786363071:2048] [node 1] ICP30 unregister session Session# [3:7577789009081330457:2048] VirtualId# [3:7577789009081330455:0]
2025-11-28T14:51:32.496220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:32.496257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789004786363071:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.496277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.497282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:32.497483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789008103787404:2048] [node 1] ICS07 socket disconnect 141 reason# EndOfStream
2025-11-28T14:51:32.497516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789008103787404:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.497594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:32.497641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789008103787404:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:32.498238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:32.498353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789003808820006:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.498373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789008474134235:2048] [node 1] ICS01 socket: 147 reason# EndOfStream
2025-11-28T14:51:32.498408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789008103787404:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.498402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789008474134143:2048] [node 1] ICP30 unregister session Session# [5:7577789008474134235:2048] VirtualId# [5:7577789008474134231:0]
2025-11-28T14:51:32.498477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789003808820006:2048] [node 1] ICP30 unregister session Session# [4:7577789008103787404:2048] VirtualId# [4:7577789008103787402:0]
2025-11-28T14:51:32.498426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789008474134235:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.498524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:32.498498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:32.498556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789003808820006:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.498535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:32.498569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.499103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:32.499315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789008027681959:2048] [node 1] ICS07 socket disconnect 97 reason# EndOfStream
2025-11-28T14:51:32.499352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789008027681959:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.499443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:32.499503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789008027681959:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:32.500086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789008027681871:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.500132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789008027681959:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.500194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789008027681871:2048] [node 1] ICP30 unregister session Session# [6:7577789008027681959:2048] VirtualId# [6:7577789008027681953:0]
2025-11-28T14:51:32.500251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:32.500281Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789008027681871:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.500297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.506278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789004786363071:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:32.508376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789003808820006:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:32.510334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789008027681871:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:32.512738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:32.512986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789009081330471:2049] [node 2] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-28T14:51:32.513018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789009081330471:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.513158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:32.513214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789009081330471:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:32.514060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789004786363072:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8101) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.514193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789009081330471:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.514238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789004786363072:2049] [node 2] ICP30 unregister session Session# [3:7577789009081330471:2049] VirtualId# [3:7577789009081330469:0]
2025-11-28T14:51:32.514289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:32.514331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789004786363072:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.514345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.515660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:32.515867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789008103787396:2049] [node 2] ICS07 socket disconnect 131 reason# EndOfStream
2025-11-28T14:51:32.515897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789008103787396:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.515992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:32.516037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789008103787396:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:32.516399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:32.516591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789008474134237:2049] [node 2] ICS01 socket: 111 reason# EndOfStream
2025-11-28T14:51:32.516637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789008474134144:2049] [node 2] ICP30 unregister session Session# [5:7577789008474134237:2049] VirtualId# [5:7577789008474134232:0]
2025-11-28T14:51:32.516679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789008474134237:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.516745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:32.516803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:32.516809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789003808820007:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8101) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.516871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789008103787396:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.516956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789003808820007:2049] [node 2] ICP30 unregister session Session# [4:7577789008103787396:2049] VirtualId# [4:7577789008103787394:0]
2025-11-28T14:51:32.517027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:32.517108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789003808820007:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.517128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.517275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:32.517502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789008027681961:2049] [node 2] ICS07 socket disconnect 128 reason# EndOfStream
2025-11-28T14:51:32.517529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789008027681961:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.517605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:32.517663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789008027681961:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:32.518252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789008027681872:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8101) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.518331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789008027681961:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.518381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789008027681872:2049] [node 2] ICP30 unregister session Session# [6:7577789008027681961:2049] VirtualId# [6:7577789008027681955:0]
2025-11-28T14:51:32.518423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:32.518468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789008027681872:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.518486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.526751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789003808820007:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:32.526846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:32.527027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789008103787400:2050] [node 3] ICS07 socket disconnect 125 reason# EndOfStream
2025-11-28T14:51:32.527057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789008103787400:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.527142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:32.527198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789008103787400:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:32.527858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:32.527933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789003808820008:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6749) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.528055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789008474134243:2050] [node 3] ICS07 socket disconnect 117 reason# EndOfStream
2025-11-28T14:51:32.528005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789008103787400:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.528095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789008474134243:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.528214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789003808820008:2050] [node 3] ICP30 unregister session Session# [4:7577789008103787400:2050] VirtualId# [4:7577789008103787398:0]
2025-11-28T14:51:32.528319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:32.528356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789003808820008:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.528157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:32.528202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789008474134243:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:32.528388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.528777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789008027681872:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:32.529453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:32.529443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789008474134145:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6749) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.529604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789008474134243:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.529660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789008474134145:2050] [node 3] ICP30 unregister session Session# [5:7577789008474134243:2050] VirtualId# [5:7577789008474134241:0]
2025-11-28T14:51:32.529677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789008027681969:2050] [node 3] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-28T14:51:32.529694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:32.529717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789008027681969:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.529718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789008474134145:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.529731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.529788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:32.529840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789008027681969:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:32.530459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789008027681873:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6749) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.530519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789008027681969:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.530550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789008027681873:2050] [node 3] ICP30 unregister session Session# [6:7577789008027681969:2050] VirtualId# [6:7577789008027681967:0]
2025-11-28T14:51:32.530602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:32.530628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789008027681873:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.530664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.540067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789008027681873:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:32.540070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789008474134145:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:32.561530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:32.561720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789008474134228:2108] [node 4] ICS07 socket disconnect 105 reason# EndOfStream
2025-11-28T14:51:32.561745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789008474134228:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.561820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:32.561857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789008474134228:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:32.562523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:32.562674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789008474134207:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61552) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:32.562713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789008474134228:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:32.562751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789008474134207:2108] [node 4] ICP30 unregister session Session# [5:7577789008474134228:2108] VirtualId# [5:7577789008474134225:0]
2025-11-28T14:51:32.562809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:32.562739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789008027681962:2108] [node 4] ICS01 socket: 106 reason# EndOfStream
2025-11-28T14:51:32.562834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789008474134207:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61552) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.562875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789008027681935:2108] [node 4] ICP30 unregister session Session# [6:7577789008027681962:2108] VirtualId# [6:7577789008027681954:0]
2025-11-28T14:51:32.562848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61552) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:32.562905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789008027681962:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.562969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:32.563026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:32.574252Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:32.574508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789008027681973:2107] [node 5] ICS01 socket: 119 reason# EndOfStream
2025-11-28T14:51:32.574579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789008027681934:2107] [node 5] ICP30 unregister session Session# [6:7577789008027681973:2107] VirtualId# [6:7577789008027681971:0]
2025-11-28T14:51:32.574612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789008027681973:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:32.574693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:32.574818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:32.639642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789011182752246:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:32.639691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789011182752247:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:32.662934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789007305076119:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:32.662970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789007305076120:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:32.686048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789008727375311:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:32.686107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789008727375312:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:32.709262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789010779530404:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:32.709326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789010779530405:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:32.709360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789010779530406:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:32.729525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008789989280:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:32.729570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008789989281:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:32.729601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008789989282:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:32.752645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789009522158720:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:32.752705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789009522158721:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:32.752735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789009522158722:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:32.911098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789011182752309:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:32.911204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789009522158783:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:32.911387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789008727375376:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:32.911513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789010779530468:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:32.911574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789007305076183:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:32.912564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789011182752310:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:32.913160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789009522158784:2109] [node 4] ICP01 ready to work
2025-11-28T14:51:32.913206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008789989343:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:32.913234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789008727375377:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:32.913376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789010779530469:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:32.913672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789007305076184:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:32.913821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789011182752311:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:32.914073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789008789989344:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:32.914213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789008727375378:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:32.914253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789007305076185:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.925911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008789989358:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4927
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.926279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789011182752325:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47447
2025-11-28T14:51:32.926332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789008727375389:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7154
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:32.928719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789008727375393:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7154
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.928829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789011182752326:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47447
2025-11-28T14:51:32.928792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789007305076196:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-28T14:51:32.929302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789011182752327:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47447
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.929306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789007305076197:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-28T14:51:32.929419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789007305076198:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.929518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789007305076199:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-28T14:51:32.929306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789008727375394:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7154
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.932969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789011182752328:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47447
2025-11-28T14:51:32.932986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789007305076203:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-28T14:51:32.937268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008789989358:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.938038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789010779530483:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33299
2025-11-28T14:51:32.938116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789010779530484:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33299
2025-11-28T14:51:32.938167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789010779530485:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33299
2025-11-28T14:51:32.938980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752327:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.939128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752326:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.939399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789011182752329:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47447
2025-11-28T14:51:32.939666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789010779530483:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.939832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789007305076199:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.939812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752325:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.939950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789010779530404:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:51:32.939933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789007305076198:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:51:32.940139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752317:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.940104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789008727375395:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7154
2025-11-28T14:51:32.940163Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789011182752317:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.940231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789007305076197:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.940266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789010779530485:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.940348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789010779530405:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.940421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789008727375389:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.940553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789007305076196:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.940761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752329:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.940767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789007305076187:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.940790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789007305076187:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.940856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789008727375393:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.940978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789008727375395:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.941061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789008727375394:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.941230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789007305076119:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.941285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789008727375312:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.941320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789007305076203:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.941350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789008727375311:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.941586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789007305076191:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.941601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789007305076191:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.941675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752328:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.941689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789010779530484:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.941791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789011182752317:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:32.942180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789007305076187:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.942569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752313:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.942572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789010779530406:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.942592Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789011182752313:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.942615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789007305076183:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.942654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789010779530488:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33299
2025-11-28T14:51:32.942716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.942780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789007305076183:2107] [node 4] ICP22 created new session: [2:7577789007305076211:2107]
2025-11-28T14:51:32.942824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789007305076211:2107] [node 4] ICS09 handshake done sender: [2:7577789007305076187:2111] self: [2:7577789007305076186:0] peer: [4:7577789010779530487:0] socket: 127 qp: -1
2025-11-28T14:51:32.942855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789007305076211:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:32.942895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.943598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789010779530489:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33299
2025-11-28T14:51:32.943607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752315:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.943627Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789011182752315:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.944251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789011182752309:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.944340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.944397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789011182752309:2107] [node 4] ICP22 created new session: [1:7577789011182752337:2107]
2025-11-28T14:51:32.944441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789011182752337:2107] [node 4] ICS09 handshake done sender: [1:7577789011182752317:2112] self: [1:7577789011182752316:0] peer: [4:7577789010779530486:0] socket: 111 qp: -1
2025-11-28T14:51:32.944469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789011182752337:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:32.944521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:32.944479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789008727375384:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.944517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789008727375384:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.944569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789007305076191:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.945653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789010779530483:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:32.945682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789008727375400:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7154
2025-11-28T14:51:32.945712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789007305076120:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.945878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.945909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789007305076120:2049] [node 3] ICP22 created new session: [2:7577789007305076214:2049]
2025-11-28T14:51:32.945936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789007305076214:2049] [node 3] ICS09 handshake done sender: [2:7577789007305076191:2113] self: [2:7577789007305076190:0] peer: [3:7577789008727375398:0] socket: 129 qp: -1
2025-11-28T14:51:32.945960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789007305076214:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:32.946014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:32.945982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789010779530485:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:32.946325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789010779530405:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789010779530485:2118] held: no
2025-11-28T14:51:32.946444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789010779530405:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.946528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.946682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789010779530405:2049] [node 2] ICP22 created new session: [4:7577789010779530491:2049]
2025-11-28T14:51:32.946687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789008727375395:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:32.946785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789010779530491:2049] [node 2] ICS09 handshake done sender: [4:7577789010779530485:2118] self: [4:7577789010779530487:0] peer: [2:7577789007305076186:0] socket: 134 qp: -1
2025-11-28T14:51:32.946877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789010779530491:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:32.947057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:32.947046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789011182752313:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.947257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008789989360:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4927
2025-11-28T14:51:32.947311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789008727375384:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:32.947341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789011182752315:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:32.947678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789011182752246:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.947760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.947752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789008727375312:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.947797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789011182752246:2048] [node 2] ICP22 created new session: [1:7577789011182752341:2048]
2025-11-28T14:51:32.947803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.947767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789010779530404:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.947838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789011182752341:2048] [node 2] ICS09 handshake done sender: [1:7577789011182752313:2110] self: [1:7577789011182752312:0] peer: [2:7577789007305076207:0] socket: 102 qp: -1
2025-11-28T14:51:32.947857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789008727375312:2049] [node 2] ICP22 created new session: [3:7577789008727375403:2049]
2025-11-28T14:51:32.947832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.947874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789011182752341:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:32.947871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789010779530404:2048] [node 1] ICP22 created new session: [4:7577789010779530493:2048]
2025-11-28T14:51:32.947899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789008727375403:2049] [node 2] ICS09 handshake done sender: [3:7577789008727375395:2120] self: [3:7577789008727375398:0] peer: [2:7577789007305076190:0] socket: 133 qp: -1
2025-11-28T14:51:32.947909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:32.947910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789010779530493:2048] [node 1] ICS09 handshake done sender: [4:7577789010779530483:2116] self: [4:7577789010779530486:0] peer: [1:7577789011182752316:0] socket: 130 qp: -1
2025-11-28T14:51:32.947974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789008727375403:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:32.947935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789010779530493:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:32.948009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:32.947965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.948147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158798:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.948253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789011182752247:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.948319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.948357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789011182752247:2049] [node 3] ICP22 created new session: [1:7577789011182752343:2049]
2025-11-28T14:51:32.948411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789011182752343:2049] [node 3] ICS09 handshake done sender: [1:7577789011182752315:2111] self: [1:7577789011182752314:0] peer: [3:7577789008727375399:0] socket: 107 qp: -1
2025-11-28T14:51:32.948440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789011182752343:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:32.948488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.949502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789007305076216:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.950422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789010779530404:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:32.950464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008789989361:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4927
2025-11-28T14:51:32.950485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789008727375376:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.950568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.950605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789008727375376:2107] [node 4] ICP22 created new session: [3:7577789008727375405:2107]
2025-11-28T14:51:32.950649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789008727375405:2107] [node 4] ICS09 handshake done sender: [3:7577789008727375384:2114] self: [3:7577789008727375383:0] peer: [4:7577789010779530490:0] socket: 123 qp: -1
2025-11-28T14:51:32.950680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789008727375405:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:32.950717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.950824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789010779530495:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33299
2025-11-28T14:51:32.951022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008789989360:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.951529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789008727375312:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.951905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789008727375402:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7154
2025-11-28T14:51:32.951936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008789989362:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4927
2025-11-28T14:51:32.951995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789008789989281:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.952128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789010779530484:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:32.953008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789010779530406:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.951980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789007305076197:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:32.953118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.952270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789007305076119:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22457) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789007305076197:2117] held: no
2025-11-28T14:51:32.952945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789007305076119:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.953016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.953057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789007305076119:2048] [node 1] ICP22 created new session: [2:7577789007305076218:2048]
2025-11-28T14:51:32.953122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789007305076218:2048] [node 1] ICS09 handshake done sender: [2:7577789007305076197:2117] self: [2:7577789007305076207:0] peer: [1:7577789011182752312:0] socket: 113 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.953484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789010779530406:2050] [node 3] ICP22 created new session: [4:7577789010779530498:2050]
2025-11-28T14:51:32.953497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789007305076218:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.953672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:32.953949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789010779530498:2050] [node 3] ICS09 handshake done sender: [4:7577789010779530484:2117] self: [4:7577789010779530490:0] peer: [3:7577789008727375383:0] socket: 131 qp: -1
2025-11-28T14:51:32.953709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158799:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.954769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158800:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.953983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789010779530498:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:32.954025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:32.955252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789007305076193:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.955288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789007305076193:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.956065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008789989361:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.956256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789008789989280:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.956480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789008727375311:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22457) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789008727375394:2119] held: no
2025-11-28T14:51:32.956993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789008727375394:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.958042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789008727375311:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.958039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008789989365:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4927
2025-11-28T14:51:32.958141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.958152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752319:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.958191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789008727375311:2048] [node 1] ICP22 created new session: [3:7577789008727375409:2048]
2025-11-28T14:51:32.958191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789011182752319:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.958264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789008727375409:2048] [node 1] ICS09 handshake done sender: [3:7577789008727375394:2119] self: [3:7577789008727375399:0] peer: [1:7577789011182752314:0] socket: 118 qp: -1
2025-11-28T14:51:32.958292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789008727375409:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:32.958320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:32.958437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789007305076193:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.958450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789009522158798:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.959620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008789989366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4927
2025-11-28T14:51:32.960274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789007305076184:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.960409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.960434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789009522158800:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.960455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789007305076184:2108] [node 5] ICP22 created new session: [2:7577789007305076221:2108]
2025-11-28T14:51:32.960498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789007305076221:2108] [node 5] ICS09 handshake done sender: [2:7577789007305076193:2114] self: [2:7577789007305076192:0] peer: [5:7577789008789989363:0] socket: 145 qp: -1
2025-11-28T14:51:32.960537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789007305076221:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:32.960649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.960597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158801:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.960687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789009522158721:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.960872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008789989362:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.960928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008789989360:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:32.961013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789009522158799:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.961102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789009522158720:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.961104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789011182752319:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:32.961265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789008789989281:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.961276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789009522158784:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.961621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.961635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789007305076195:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.961670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008789989281:2049] [node 2] ICP22 created new session: [5:7577789008789989368:2049]
2025-11-28T14:51:32.961614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789010779530479:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.961660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789007305076195:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.961648Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789010779530479:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.962151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008789989368:2049] [node 2] ICS09 handshake done sender: [5:7577789008789989360:2116] self: [5:7577789008789989363:0] peer: [2:7577789007305076192:0] socket: 146 qp: -1
2025-11-28T14:51:32.962226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789011182752321:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.962268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008789989368:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:32.963134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:32.962256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789011182752321:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.963089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789011182752310:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.963215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:32.963289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789011182752310:2108] [node 5] ICP22 created new session: [1:7577789011182752347:2108]
2025-11-28T14:51:32.963474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789011182752347:2108] [node 5] ICS09 handshake done sender: [1:7577789011182752319:2113] self: [1:7577789011182752318:0] peer: [5:7577789008789989364:0] socket: 149 qp: -1
2025-11-28T14:51:32.963512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789011182752347:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:32.963595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.963613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789007305076195:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:32.963693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789010779530479:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.964376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158805:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.964456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789010779530469:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.964517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.964587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789010779530469:2109] [node 6] ICP22 created new session: [4:7577789010779530501:2109]
2025-11-28T14:51:32.964631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789010779530501:2109] [node 6] ICS09 handshake done sender: [4:7577789010779530479:2115] self: [4:7577789010779530478:0] peer: [6:7577789009522158804:0] socket: 126 qp: -1
2025-11-28T14:51:32.964610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789007305076185:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.964672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789010779530501:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:32.964695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:32.964690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.964738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789007305076185:2110] [node 6] ICP22 created new session: [2:7577789007305076224:2110]
2025-11-28T14:51:32.964818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789007305076224:2110] [node 6] ICS09 handshake done sender: [2:7577789007305076195:2115] self: [2:7577789007305076194:0] peer: [6:7577789009522158802:0] socket: 147 qp: -1
2025-11-28T14:51:32.964851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789007305076224:2110] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.964880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:32.966199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158806:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.966625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789009522158798:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:32.966618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789011182752321:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:32.966911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789008789989343:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.966973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789011182752311:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.967057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789008789989281:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:32.967066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.967100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789009522158801:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.967097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789011182752311:2109] [node 6] ICP22 created new session: [1:7577789011182752350:2109]
2025-11-28T14:51:32.967125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789011182752350:2109] [node 6] ICS09 handshake done sender: [1:7577789011182752321:2114] self: [1:7577789011182752320:0] peer: [6:7577789009522158803:0] socket: 128 qp: -1
2025-11-28T14:51:32.967150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789011182752350:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:32.967176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:32.967289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789009522158721:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.967328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:32.967407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789009522158721:2049] [node 2] ICP22 created new session: [6:7577789009522158808:2049]
2025-11-28T14:51:32.967516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789009522158808:2049] [node 2] ICS09 handshake done sender: [6:7577789009522158798:2115] self: [6:7577789009522158802:0] peer: [2:7577789007305076194:0] socket: 148 qp: -1
2025-11-28T14:51:32.967564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789009522158808:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:32.967598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.968187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008789989367:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4927
2025-11-28T14:51:32.968625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008789989365:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.968734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789008789989280:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22457) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789008789989361:2117] held: no
2025-11-28T14:51:32.968866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789010779530477:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.968891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789010779530477:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.968935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008789989361:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:32.969768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789009522158799:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:32.969999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789008789989280:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.970099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.970140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008789989280:2048] [node 1] ICP22 created new session: [5:7577789008789989373:2048]
2025-11-28T14:51:32.970186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008789989373:2048] [node 1] ICS09 handshake done sender: [5:7577789008789989361:2117] self: [5:7577789008789989364:0] peer: [1:7577789011182752318:0] socket: 96 qp: -1
2025-11-28T14:51:32.970226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008789989373:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:32.970222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789010779530477:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:32.970274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:32.971064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789010779530468:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.971169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:32.971220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789010779530468:2108] [node 5] ICP22 created new session: [4:7577789010779530505:2108]
2025-11-28T14:51:32.971257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789010779530505:2108] [node 5] ICS09 handshake done sender: [4:7577789010779530477:2114] self: [4:7577789010779530476:0] peer: [5:7577789008789989370:0] socket: 132 qp: -1
2025-11-28T14:51:32.971287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789010779530505:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:32.971263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789008789989282:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.971289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789009522158721:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:32.971349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.971451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789009522158784:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.971486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.971515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789009522158784:2109] [node 4] ICP22 created new session: [6:7577789009522158810:2109]
2025-11-28T14:51:32.971551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789009522158810:2109] [node 4] ICS09 handshake done sender: [6:7577789009522158799:2116] self: [6:7577789009522158804:0] peer: [4:7577789010779530478:0] socket: 151 qp: -1
2025-11-28T14:51:32.971590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789009522158810:2109] [node 4] ICS10 traffic start
2025-11-28T14:51:32.971616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.971741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789008727375386:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.971846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789010779530504:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33299
2025-11-28T14:51:32.971774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789008727375386:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.972407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158807:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.972589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789008727375386:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:32.973187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789009522158720:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22457) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789009522158800:2117] held: no
2025-11-28T14:51:32.973247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789008727375377:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.973351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:32.973388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789008727375377:2108] [node 5] ICP22 created new session: [3:7577789008727375412:2108]
2025-11-28T14:51:32.973430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789008727375412:2108] [node 5] ICS09 handshake done sender: [3:7577789008727375386:2115] self: [3:7577789008727375385:0] peer: [5:7577789008789989375:0] socket: 104 qp: -1
2025-11-28T14:51:32.973457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789008727375412:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:32.973492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.973719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789009522158722:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.973987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789009522158800:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.974462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008789989376:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4927
2025-11-28T14:51:32.974635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789009522158720:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.974636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789008727375388:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.974662Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789008727375388:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.974700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:32.974821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789009522158720:2048] [node 1] ICP22 created new session: [6:7577789009522158816:2048]
2025-11-28T14:51:32.974882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789009522158816:2048] [node 1] ICS09 handshake done sender: [6:7577789009522158800:2117] self: [6:7577789009522158803:0] peer: [1:7577789011182752320:0] socket: 152 qp: -1
2025-11-28T14:51:32.974905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789009522158816:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:32.975148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:32.975371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789010779530504:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.975471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789010779530468:2108] [node 5] ICP18 handshake (actor [4:7577789010779530504:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.976318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158812:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.976546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789008727375388:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:32.977053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789008789989282:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61979) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789008789989365:2119] held: no
2025-11-28T14:51:32.977076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789008727375378:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.977155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.977197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789008727375378:2109] [node 6] ICP22 created new session: [3:7577789008727375416:2109]
2025-11-28T14:51:32.977238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789008727375416:2109] [node 6] ICS09 handshake done sender: [3:7577789008727375388:2116] self: [3:7577789008727375387:0] peer: [6:7577789009522158813:0] socket: 120 qp: -1
2025-11-28T14:51:32.977262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789008727375416:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:32.977286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.978251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789008727375415:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7154
2025-11-28T14:51:32.978550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158818:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.978794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008789989362:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:32.979173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789008789989343:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4994) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789008789989362:2118] held: no
2025-11-28T14:51:32.979179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789008727375415:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.979195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789009522158812:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.979278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789008727375378:2109] [node 6] ICP18 handshake (actor [3:7577789008727375415:2123]) is too old
2025-11-28T14:51:32.979269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789008789989343:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.979245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789009522158801:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:32.979367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:32.979413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008789989343:2108] [node 4] ICP22 created new session: [5:7577789008789989378:2108]
2025-11-28T14:51:32.979467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008789989378:2108] [node 4] ICS09 handshake done sender: [5:7577789008789989362:2118] self: [5:7577789008789989370:0] peer: [4:7577789010779530476:0] socket: 150 qp: -1
2025-11-28T14:51:32.979505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008789989378:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:32.979536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789009522158722:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.979554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:32.979581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.979693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789009522158722:2050] [node 3] ICP22 created new session: [6:7577789009522158819:2050]
2025-11-28T14:51:32.979748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789009522158819:2050] [node 3] ICS09 handshake done sender: [6:7577789009522158801:2118] self: [6:7577789009522158813:0] peer: [3:7577789008727375387:0] socket: 155 qp: -1
2025-11-28T14:51:32.979787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789009522158819:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:32.979813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.980140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789008789989377:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4927
2025-11-28T14:51:32.980801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008789989365:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:32.981264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789008789989282:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.981456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:32.981509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008789989282:2050] [node 3] ICP22 created new session: [5:7577789008789989380:2050]
2025-11-28T14:51:32.981546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008789989380:2050] [node 3] ICS09 handshake done sender: [5:7577789008789989365:2119] self: [5:7577789008789989375:0] peer: [3:7577789008727375385:0] socket: 98 qp: -1
2025-11-28T14:51:32.981582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008789989380:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:32.981608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:32.981658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789009522158783:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:32.982108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789008789989354:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:32.982134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789008789989354:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:32.982983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789008789989354:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:32.983191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789009522158822:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39011
2025-11-28T14:51:32.983274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789008789989344:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:32.983364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:32.983423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789008789989344:2109] [node 6] ICP22 created new session: [5:7577789008789989383:2109]
2025-11-28T14:51:32.983462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789008789989383:2109] [node 6] ICS09 handshake done sender: [5:7577789008789989354:2114] self: [5:7577789008789989353:0] peer: [6:7577789009522158821:0] socket: 119 qp: -1
2025-11-28T14:51:32.983491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789008789989383:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:32.983570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:32.983623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789009522158812:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:32.983804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789009522158783:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:32.983845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:32.983880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789009522158783:2107] [node 5] ICP22 created new session: [6:7577789009522158823:2107]
2025-11-28T14:51:32.983912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789009522158823:2107] [node 5] ICS09 handshake done sender: [6:7577789009522158812:2122] self: [6:7577789009522158821:0] peer: [5:7577789008789989353:0] socket: 161 qp: -1
2025-11-28T14:51:32.983942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789009522158823:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:32.983976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:32.999990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:33.000226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789007305076218:2048] [node 1] ICS01 socket: 113 reason# EndOfStream
2025-11-28T14:51:33.000277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789007305076119:2048] [node 1] ICP30 unregister session Session# [2:7577789007305076218:2048] VirtualId# [2:7577789007305076207:0]
2025-11-28T14:51:33.000313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789007305076218:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.000437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.000487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.000988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:33.001168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789008727375409:2048] [node 1] ICS01 socket: 118 reason# EndOfStream
2025-11-28T14:51:33.001199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789008727375311:2048] [node 1] ICP30 unregister session Session# [3:7577789008727375409:2048] VirtualId# [3:7577789008727375399:0]
2025-11-28T14:51:33.001220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789008727375409:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.001279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.001315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.003883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:33.004087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789010779530493:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-28T14:51:33.004126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789010779530404:2048] [node 1] ICP30 unregister session Session# [4:7577789010779530493:2048] VirtualId# [4:7577789010779530486:0]
2025-11-28T14:51:33.004150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789010779530493:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.004212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.004252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.004798Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:33.005006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789008789989373:2048] [node 1] ICS01 socket: 96 reason# EndOfStream
2025-11-28T14:51:33.005040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789008789989280:2048] [node 1] ICP30 unregister session Session# [5:7577789008789989373:2048] VirtualId# [5:7577789008789989364:0]
2025-11-28T14:51:33.005078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789008789989373:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.005152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.005205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.005582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:33.005779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789009522158816:2048] [node 1] ICS01 socket: 152 reason# EndOfStream
2025-11-28T14:51:33.005816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789009522158720:2048] [node 1] ICP30 unregister session Session# [6:7577789009522158816:2048] VirtualId# [6:7577789009522158803:0]
2025-11-28T14:51:33.005842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789009522158816:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.005911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.005955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.026086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:33.026285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789008727375403:2049] [node 2] ICS01 socket: 133 reason# EndOfStream
2025-11-28T14:51:33.026338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789008727375312:2049] [node 2] ICP30 unregister session Session# [3:7577789008727375403:2049] VirtualId# [3:7577789008727375398:0]
2025-11-28T14:51:33.026372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789008727375403:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.026461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:33.026530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:33.028435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:33.028636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789010779530491:2049] [node 2] ICS01 socket: 134 reason# EndOfStream
2025-11-28T14:51:33.028683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789010779530405:2049] [node 2] ICP30 unregister session Session# [4:7577789010779530491:2049] VirtualId# [4:7577789010779530487:0]
2025-11-28T14:51:33.028715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789010779530491:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.028784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:33.028844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:33.029314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:33.029490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789008789989368:2049] [node 2] ICS07 socket disconnect 146 reason# EndOfStream
2025-11-28T14:51:33.029516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789008789989368:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.029577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:33.029639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789008789989368:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:33.030103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:33.030312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789009522158808:2049] [node 2] ICS07 socket disconnect 148 reason# EndOfStream
2025-11-28T14:51:33.030362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789009522158808:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.030449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:33.030493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789009522158808:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:33.030576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789008789989281:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.030645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789008789989368:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.030675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789008789989281:2049] [node 2] ICP30 unregister session Session# [5:7577789008789989368:2049] VirtualId# [5:7577789008789989363:0]
2025-11-28T14:51:33.030714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:33.030739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789008789989281:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.030749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.031347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789009522158721:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.031386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789009522158808:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.031413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789009522158721:2049] [node 2] ICP30 unregister session Session# [6:7577789009522158808:2049] VirtualId# [6:7577789009522158802:0]
2025-11-28T14:51:33.031449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:33.031474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789009522158721:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.031485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.040883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789008789989281:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:33.041819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789009522158721:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:33.050951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:33.051224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789010779530498:2050] [node 3] ICS01 socket: 131 reason# EndOfStream
2025-11-28T14:51:33.051279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789010779530406:2050] [node 3] ICP30 unregister session Session# [4:7577789010779530498:2050] VirtualId# [4:7577789010779530490:0]
2025-11-28T14:51:33.051317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789010779530498:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.051425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:33.051485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:33.051644Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:33.051904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789008789989380:2050] [node 3] ICS01 socket: 98 reason# EndOfStream
2025-11-28T14:51:33.051940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789008789989282:2050] [node 3] ICP30 unregister session Session# [5:7577789008789989380:2050] VirtualId# [5:7577789008789989375:0]
2025-11-28T14:51:33.051999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789008789989380:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.052088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:33.052141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:33.052438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:33.052627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789009522158819:2050] [node 3] ICS07 socket disconnect 155 reason# EndOfStream
2025-11-28T14:51:33.052672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789009522158819:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.052757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:33.052835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789009522158819:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:33.053605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789009522158722:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61979) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.053652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789009522158819:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.053680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789009522158722:2050] [node 3] ICP30 unregister session Session# [6:7577789009522158819:2050] VirtualId# [6:7577789009522158813:0]
2025-11-28T14:51:33.053714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:33.053743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789009522158722:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61979) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.053754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61979) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.063315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789009522158722:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:33.074973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:33.075212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789008789989378:2108] [node 4] ICS01 socket: 150 reason# EndOfStream
2025-11-28T14:51:33.075283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789008789989343:2108] [node 4] ICP30 unregister session Session# [5:7577789008789989378:2108] VirtualId# [5:7577789008789989370:0]
2025-11-28T14:51:33.075324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789008789989378:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.075416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:33.075477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:33.075852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:33.076078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789009522158810:2109] [node 4] ICS07 socket disconnect 151 reason# EndOfStream
2025-11-28T14:51:33.076114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789009522158810:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.076185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:33.076235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789009522158810:2109] [node 4] ICS15 start handshake
2025-11-28T14:51:33.077016Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789009522158784:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4994) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.077068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789009522158810:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.077116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789009522158784:2109] [node 4] ICP30 unregister session Session# [6:7577789009522158810:2109] VirtualId# [6:7577789009522158804:0]
2025-11-28T14:51:33.077251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:33.077274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789009522158784:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4994) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.077282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4994) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.086921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789009522158784:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:51:33.096395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:33.096641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789009522158823:2107] [node 5] ICS01 socket: 161 reason# EndOfStream
2025-11-28T14:51:33.096691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789009522158783:2107] [node 5] ICP30 unregister session Session# [6:7577789009522158823:2107] VirtualId# [6:7577789009522158821:0]
2025-11-28T14:51:33.096720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789009522158823:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.096821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:33.096891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:33.165152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789012770845434:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:33.165203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789012770845435:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:33.187239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789012231449419:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.187472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789012231449420:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:33.209957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789014111888998:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.209972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789014111888999:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:33.232578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789011461036506:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:33.232615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789011461036507:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:33.234821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789011461036505:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.263008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789012992964916:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.263132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789012992964918:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:33.264096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789012992964917:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:33.287065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789015665599896:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.287090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789015665599897:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:33.287104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789015665599898:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:33.379304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789015665599959:2106] [node 4] ICP01 ready to work
2025-11-28T14:51:33.379655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789011461036568:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:33.379658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789015665599960:2104] [node 5] ICP01 ready to work
2025-11-28T14:51:33.379754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789012992964979:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:33.379702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789014111889061:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:33.379760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789012992964980:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:33.380100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789011461036569:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:33.380233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789014111889062:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:33.380551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789014111889063:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:33.381947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789011461036505:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32036538ns) {Configure:104(32024157ns) {ProcessConfigured:135(205233ns) {StartInitialHandshake:157(202639ns) {DropHandshakes:470(2350ns) {DropIncomingHandshake:442(614ns) DropOutgoingHandshake:457(377ns)} GenerateSessionVirtualId:988(687ns) PrepareNewSessionHandshake:477(1110ns) {SwitchToState:179(349ns)}}}}}
2025-11-28T14:51:33.382202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789012231449420:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41291984ns) {Configure:104(37940300ns) {ProcessConfigured:135(220387ns) {StartInitialHandshake:157(218400ns) {DropHandshakes:470(2643ns) {DropIncomingHandshake:442(753ns) DropOutgoingHandshake:457(438ns)} GenerateSessionVirtualId:988(654ns) PrepareNewSessionHandshake:477(1273ns) {SwitchToState:179(697ns)}}}}}
2025-11-28T14:51:33.382260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789014111888998:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39850053ns) {Configure:104(39837841ns) {ProcessConfigured:135(179970ns) {StartInitialHandshake:157(177477ns) {DropHandshakes:470(1321ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(679ns) PrepareNewSessionHandshake:477(1384ns) {SwitchToState:179(715ns)}}}}}
2025-11-28T14:51:33.382267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789012770845434:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43582065ns) {Configure:104(43578283ns) {ProcessConfigured:135(230360ns) {StartInitialHandshake:157(227909ns) {DropHandshakes:470(2179ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(809ns)} GenerateSessionVirtualId:988(628ns) PrepareNewSessionHandshake:477(920ns) {SwitchToState:179(402ns)}}}}}
2025-11-28T14:51:33.382446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789012231449419:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32583865ns) {Configure:104(32572659ns) {ProcessConfigured:135(65214ns) {StartInitialHandshake:157(64245ns) {DropHandshakes:470(3642ns) {DropIncomingHandshake:442(213ns) DropOutgoingHandshake:457(158ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:477(790ns) {SwitchToState:179(284ns)}}}}}
2025-11-28T14:51:33.382490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789012770845435:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(38260170ns) {Configure:104(38253486ns) {ProcessConfigured:135(129186ns) {StartInitialHandshake:157(127237ns) {DropHandshakes:470(7336ns) {DropIncomingHandshake:442(373ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(447ns) PrepareNewSessionHandshake:477(2520ns) {SwitchToState:179(1205ns)}}}}}
2025-11-28T14:51:33.382647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789012770845503:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:33.382781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789012231449487:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:33.382907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789012231449488:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:33.383213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789012770845504:2111] [node 6] ICP01 ready to work
2025-11-28T14:51:33.383346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789012770845505:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:33.383356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789012231449489:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.389173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789011461036583:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20014
2025-11-28T14:51:33.389138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789014111889077:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51051
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.391090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789014111889078:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51051
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.393046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789012231449497:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.393391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789012770845512:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47657
2025-11-28T14:51:33.393390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789012231449496:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21521
2025-11-28T14:51:33.394284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789012770845511:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47657
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.398654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789012770845516:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47657
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.399771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789012231449496:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.399817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845511:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.400502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789014111889079:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51051
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.401738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789014111889077:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.402599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789011461036583:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.403013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789012231449500:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.403039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789011461036585:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20014
2025-11-28T14:51:33.403253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789011461036584:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20014
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.403645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789011461036507:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.403739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599974:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-11-28T14:51:33.403588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789012231449497:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.404025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845512:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.404007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789012231449502:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21521
No RDMA context found for GID: 2025-11-28T14:51:33.404629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789014111889069:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:51:33.404686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789014111889069:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.407777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789012231449503:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21521
2025-11-28T14:51:33.407803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789011461036584:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.408301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789012770845518:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47657
2025-11-28T14:51:33.408467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789014111889069:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:33.409079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789012992964994:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35163
2025-11-28T14:51:33.409665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789012770845519:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47657
2025-11-28T14:51:33.409009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789014111889061:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.409329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789015665599974:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.418295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789015665599960:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.409329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:33.418337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789014111889061:2107] [node 4] ICP22 created new session: [3:7577789014111889082:2107]
2025-11-28T14:51:33.418272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845518:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.418503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789014111889082:2107] [node 4] ICS09 handshake done sender: [3:7577789014111889069:2112] self: [3:7577789014111889068:0] peer: [4:7577789011461036586:0] socket: 105 qp: -1
2025-11-28T14:51:33.418760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789012992964987:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.418790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789012992964987:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:33.418551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789014111889082:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:33.418502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845519:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.419210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:33.419404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789011461036507:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27591) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789011461036583:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.420194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789012231449502:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.420668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789011461036587:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20014
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.422140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789012992964987:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:33.422749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789011461036583:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:33.423051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789011461036507:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.422994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789012992964995:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35163
2025-11-28T14:51:33.423158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789012992964979:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.423143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:33.423237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789011461036507:2050] [node 3] ICP22 created new session: [4:7577789011461036590:2050]
2025-11-28T14:51:33.423256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:33.423356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789012992964979:2109] [node 6] ICP22 created new session: [5:7577789012992964997:2109]
2025-11-28T14:51:33.423356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789011461036590:2050] [node 3] ICS09 handshake done sender: [4:7577789011461036583:2115] self: [4:7577789011461036586:0] peer: [3:7577789014111889068:0] socket: 110 qp: -1
2025-11-28T14:51:33.423402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789011461036590:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:33.423405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789012992964997:2109] [node 6] ICS09 handshake done sender: [5:7577789012992964987:2112] self: [5:7577789012992964985:0] peer: [6:7577789015665599975:0] socket: 126 qp: -1
2025-11-28T14:51:33.423222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599976:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-11-28T14:51:33.423483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:33.423446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789012992964997:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:33.423489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:33.423710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789012992964995:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.424191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789015665599974:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.424284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789012992964918:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.424325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789012992964999:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35163
2025-11-28T14:51:33.424730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789015665599960:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.424816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:33.424753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789014111889071:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.424981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789015665599960:2104] [node 5] ICP22 created new session: [6:7577789015665599977:2104]
2025-11-28T14:51:33.424822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789014111889071:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:33.425288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789015665599977:2104] [node 5] ICS09 handshake done sender: [6:7577789015665599974:2115] self: [6:7577789015665599975:0] peer: [5:7577789012992964985:0] socket: 134 qp: -1
2025-11-28T14:51:33.425584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789011461036592:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20014
2025-11-28T14:51:33.425492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789015665599977:2104] [node 5] ICS10 traffic start
2025-11-28T14:51:33.425906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789014111889071:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:33.425742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:33.426272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789012992964999:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.426134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789014111889062:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.426216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:33.426323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789014111889062:2108] [node 5] ICP22 created new session: [3:7577789014111889086:2108]
2025-11-28T14:51:33.426413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789014111889086:2108] [node 5] ICS09 handshake done sender: [3:7577789014111889071:2113] self: [3:7577789014111889070:0] peer: [5:7577789012992965001:0] socket: 109 qp: -1
2025-11-28T14:51:33.426451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789014111889086:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.426487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:33.426546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789012992965002:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35163
2025-11-28T14:51:33.427357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789012992964980:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.427687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599979:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-11-28T14:51:33.428021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789012992964995:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:33.427998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789011461036577:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.428018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789011461036577:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.428940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789012992964918:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.428973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789014111889089:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51051
2025-11-28T14:51:33.428998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:33.429030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789011461036577:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:33.429042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789012992964918:2050] [node 3] ICP22 created new session: [5:7577789012992965004:2050]
2025-11-28T14:51:33.429073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789012992965004:2050] [node 3] ICS09 handshake done sender: [5:7577789012992964995:2116] self: [5:7577789012992965001:0] peer: [3:7577789014111889070:0] socket: 146 qp: -1
2025-11-28T14:51:33.429102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789012992965004:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:33.429126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:33.429230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789011461036568:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.429798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:33.429837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789011461036568:2108] [node 5] ICP22 created new session: [4:7577789011461036594:2108]
2025-11-28T14:51:33.430245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789011461036594:2108] [node 5] ICS09 handshake done sender: [4:7577789011461036577:2113] self: [4:7577789011461036576:0] peer: [5:7577789012992965003:0] socket: 148 qp: -1
2025-11-28T14:51:33.430291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789011461036594:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:33.430344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:33.431464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789014111889089:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.431489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789015665599979:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.431631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789015665599898:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.431994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789014111889076:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.432019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789014111889076:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:33.432103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599982:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-11-28T14:51:33.432100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789012992965006:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35163
2025-11-28T14:51:33.433428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789014111889076:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:33.433589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789014111889063:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.433675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:33.433748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789014111889063:2109] [node 6] ICP22 created new session: [3:7577789014111889092:2109]
2025-11-28T14:51:33.433963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789014111889092:2109] [node 6] ICS09 handshake done sender: [3:7577789014111889076:2114] self: [3:7577789014111889075:0] peer: [6:7577789015665599981:0] socket: 154 qp: -1
2025-11-28T14:51:33.434017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789014111889092:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:33.434062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:33.434831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789012992964999:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.435140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789012992964980:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.435127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599983:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-11-28T14:51:33.435190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:33.435219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789012992964980:2108] [node 4] ICP22 created new session: [5:7577789012992965007:2108]
2025-11-28T14:51:33.435258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789012992965007:2108] [node 4] ICS09 handshake done sender: [5:7577789012992964999:2117] self: [5:7577789012992965003:0] peer: [4:7577789011461036576:0] socket: 149 qp: -1
2025-11-28T14:51:33.435281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789012992965007:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:33.435337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:33.436191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789015665599898:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27591) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789015665599979:2117] held: no
2025-11-28T14:51:33.436455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789015665599982:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.436558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789015665599979:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:33.437431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789015665599963:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.438055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789015665599898:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.438127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:33.438176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789015665599898:2050] [node 3] ICP22 created new session: [6:7577789015665599984:2050]
2025-11-28T14:51:33.438218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789015665599984:2050] [node 3] ICS09 handshake done sender: [6:7577789015665599979:2117] self: [6:7577789015665599981:0] peer: [3:7577789014111889075:0] socket: 155 qp: -1
2025-11-28T14:51:33.438248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789015665599959:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.438250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789015665599984:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:33.438282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:33.438516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789011461036579:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.438554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789011461036579:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:33.446110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789011461036579:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:33.446369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789011461036569:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.446458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:33.446511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789011461036569:2109] [node 6] ICP22 created new session: [4:7577789011461036598:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.446608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789011461036598:2109] [node 6] ICS09 handshake done sender: [4:7577789011461036579:2114] self: [4:7577789011461036578:0] peer: [6:7577789015665599985:0] socket: 160 qp: -1
2025-11-28T14:51:33.446683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789011461036598:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:33.446717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:33.446709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599987:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-11-28T14:51:33.447547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789015665599959:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11854) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789015665599982:2118] held: no
2025-11-28T14:51:33.447721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789015665599982:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:33.448299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789015665599959:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.448381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:33.448424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789015665599959:2106] [node 4] ICP22 created new session: [6:7577789015665599988:2106]
2025-11-28T14:51:33.448461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789015665599988:2106] [node 4] ICS09 handshake done sender: [6:7577789015665599982:2118] self: [6:7577789015665599985:0] peer: [4:7577789011461036578:0] socket: 161 qp: -1
2025-11-28T14:51:33.448500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789015665599988:2106] [node 4] ICS10 traffic start
2025-11-28T14:51:33.448555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:33.458263Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789012770845502:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.466172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789014111889079:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.466381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789014111888998:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.474217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789012770845508:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.481912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845502:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.481893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789011461036585:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.482109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789011461036505:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.482378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845508:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.482592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789012770845502:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.482898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789014111889095:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51051
2025-11-28T14:51:33.483126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789012770845435:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.483193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:33.483197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789014111889079:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:33.483243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789012770845435:2049] [node 3] ICP22 created new session: [1:7577789012770845526:2049]
2025-11-28T14:51:33.483411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789012770845526:2049] [node 3] ICS09 handshake done sender: [1:7577789012770845502:2113] self: [1:7577789012770845501:0] peer: [3:7577789014111889094:0] socket: 122 qp: -1
2025-11-28T14:51:33.483527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789014111888998:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.483519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789012770845526:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:33.483592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:33.483564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:33.483647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789014111888998:2048] [node 1] ICP22 created new session: [3:7577789014111889096:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.483692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789014111889096:2048] [node 1] ICS09 handshake done sender: [3:7577789014111889079:2117] self: [3:7577789014111889094:0] peer: [1:7577789012770845501:0] socket: 130 qp: -1
2025-11-28T14:51:33.483731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789014111889096:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:33.483788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:33.483762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789011461036601:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20014
2025-11-28T14:51:33.486136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789012770845508:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:33.487529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789011461036585:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:33.488087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789011461036505:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.488199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:33.488250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789011461036505:2048] [node 1] ICP22 created new session: [4:7577789011461036602:2048]
2025-11-28T14:51:33.488303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789011461036602:2048] [node 1] ICS09 handshake done sender: [4:7577789011461036585:2117] self: [4:7577789011461036600:0] peer: [1:7577789012770845506:0] socket: 141 qp: -1
2025-11-28T14:51:33.488341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789011461036602:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:33.488397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:33.488332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789012770845503:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.488406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:33.488441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789012770845503:2107] [node 4] ICP22 created new session: [1:7577789012770845529:2107]
2025-11-28T14:51:33.488475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789012770845529:2107] [node 4] ICS09 handshake done sender: [1:7577789012770845508:2114] self: [1:7577789012770845506:0] peer: [4:7577789011461036600:0] socket: 131 qp: -1
2025-11-28T14:51:33.488500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789012770845529:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:33.488525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.490966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599990:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.493580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789012231449484:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.493702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789012992965009:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35163
2025-11-28T14:51:33.503115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789012231449503:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.503108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789014111889078:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.503344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789014111888999:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.515631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789012231449492:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.522755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789012231449484:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.522777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789012992964994:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.522936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789012992964917:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.523176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789012231449492:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.523387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789012231449484:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.523550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789014111889099:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51051
2025-11-28T14:51:33.523855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789012231449420:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.523847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789014111889078:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:33.523915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:33.524242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789012231449420:2049] [node 3] ICP22 created new session: [2:7577789012231449510:2049]
2025-11-28T14:51:33.524427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789014111888999:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.524435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789012231449510:2049] [node 3] ICS09 handshake done sender: [2:7577789012231449484:2111] self: [2:7577789012231449483:0] peer: [3:7577789014111889098:0] socket: 116 qp: -1
2025-11-28T14:51:33.524696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789012231449510:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:33.524468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:33.524740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:33.524666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789014111888999:2049] [node 2] ICP22 created new session: [3:7577789014111889100:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.524711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789014111889100:2049] [node 2] ICS09 handshake done sender: [3:7577789014111889078:2116] self: [3:7577789014111889098:0] peer: [2:7577789012231449483:0] socket: 119 qp: -1
2025-11-28T14:51:33.524744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789014111889100:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:33.524784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:33.524886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789012992965011:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35163
2025-11-28T14:51:33.526475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789012231449492:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:33.527863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789012231449488:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.527954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789012992964994:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:33.527933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:33.527960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789012231449488:2109] [node 5] ICP22 created new session: [2:7577789012231449513:2109]
2025-11-28T14:51:33.527989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789012231449513:2109] [node 5] ICS09 handshake done sender: [2:7577789012231449492:2113] self: [2:7577789012231449491:0] peer: [5:7577789012992965010:0] socket: 129 qp: -1
2025-11-28T14:51:33.528028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789012231449513:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:33.528055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:33.528342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789012992964917:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.528398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:33.528429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789012992964917:2049] [node 2] ICP22 created new session: [5:7577789012992965012:2049]
2025-11-28T14:51:33.528468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789012992965012:2049] [node 2] ICS09 handshake done sender: [5:7577789012992964994:2115] self: [5:7577789012992965010:0] peer: [2:7577789012231449491:0] socket: 137 qp: -1
2025-11-28T14:51:33.528499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789012992965012:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:33.528555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.529066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789011461036604:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20014
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.530280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599991:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-11-28T14:51:33.539373Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789012992964988:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.547434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789011461036592:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.547631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789011461036568:2108] [node 5] ICP18 handshake (actor [4:7577789011461036592:2119]) is too old
2025-11-28T14:51:33.559944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789012770845500:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.581341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789012770845509:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.589643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789015665599990:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.589886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789015665599896:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.590195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845509:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.590630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789012770845509:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.590967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599993:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-11-28T14:51:33.591010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789012770845504:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.591078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:33.591133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789012770845504:2111] [node 6] ICP22 created new session: [1:7577789012770845533:2111]
2025-11-28T14:51:33.591180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789012770845533:2111] [node 6] ICS09 handshake done sender: [1:7577789012770845509:2115] self: [1:7577789012770845507:0] peer: [6:7577789015665599992:0] socket: 102 qp: -1
2025-11-28T14:51:33.591224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789012770845533:2111] [node 6] ICS10 traffic start
2025-11-28T14:51:33.591227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789015665599990:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:33.591255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:33.591923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789015665599896:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.591986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:33.592237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789012231449486:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.592262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789015665599896:2048] [node 1] ICP22 created new session: [6:7577789015665599994:2048]
2025-11-28T14:51:33.592913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789015665599994:2048] [node 1] ICS09 handshake done sender: [6:7577789015665599990:2121] self: [6:7577789015665599992:0] peer: [1:7577789012770845507:0] socket: 104 qp: -1
2025-11-28T14:51:33.592934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789012231449500:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.592952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789015665599994:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:33.599012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:33.599918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789012231449419:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.600560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845516:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.600650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845500:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.601347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789012770845500:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:33.601632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789012770845434:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.601684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:33.601724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789012770845434:2048] [node 2] ICP22 created new session: [1:7577789012770845537:2048]
2025-11-28T14:51:33.601773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789012770845537:2048] [node 2] ICS09 handshake done sender: [1:7577789012770845500:2112] self: [1:7577789012770845499:0] peer: [2:7577789012231449516:0] socket: 118 qp: -1
2025-11-28T14:51:33.601810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789012770845537:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:33.601844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:33.603997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789012231449493:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.632476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789011461036604:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.632681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789011461036506:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.633110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789012231449419:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789012231449500:2118] held: no
2025-11-28T14:51:33.633221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789012231449493:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.633384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789012231449517:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21521
2025-11-28T14:51:33.633790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789012231449500:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.634248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789012231449493:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:33.634265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789011461036606:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20014
2025-11-28T14:51:33.634448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789011461036604:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:33.634554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789012231449419:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.634596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:33.635156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789011461036506:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.634679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789012231449419:2048] [node 1] ICP22 created new session: [2:7577789012231449519:2048]
2025-11-28T14:51:33.635257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:33.635223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789012231449519:2048] [node 1] ICS09 handshake done sender: [2:7577789012231449500:2118] self: [2:7577789012231449516:0] peer: [1:7577789012770845499:0] socket: 121 qp: -1
2025-11-28T14:51:33.635272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789012231449519:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:33.635294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789011461036506:2049] [node 2] ICP22 created new session: [4:7577789011461036607:2049]
2025-11-28T14:51:33.635307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:33.635346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789011461036607:2049] [node 2] ICS09 handshake done sender: [4:7577789011461036604:2121] self: [4:7577789011461036605:0] peer: [2:7577789012231449490:0] socket: 111 qp: -1
2025-11-28T14:51:33.635372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789011461036607:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:33.635395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:33.636154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789012231449487:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.636211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:33.636249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789012231449487:2107] [node 4] ICP22 created new session: [2:7577789012231449521:2107]
2025-11-28T14:51:33.636284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789012231449521:2107] [node 4] ICS09 handshake done sender: [2:7577789012231449493:2114] self: [2:7577789012231449490:0] peer: [4:7577789011461036605:0] socket: 103 qp: -1
2025-11-28T14:51:33.636301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789012231449521:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:33.636324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:33.637941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789012770845514:2118] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.645460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789012992965009:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.645683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789012992964916:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.645977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789012770845514:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.646459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789012770845514:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:33.646652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789012770845505:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:33.646677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789012992965015:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35163
2025-11-28T14:51:33.646725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:33.646788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789012770845505:2109] [node 5] ICP22 created new session: [1:7577789012770845540:2109]
2025-11-28T14:51:33.646835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789012770845540:2109] [node 5] ICS09 handshake done sender: [1:7577789012770845514:2118] self: [1:7577789012770845513:0] peer: [5:7577789012992965014:0] socket: 108 qp: -1
2025-11-28T14:51:33.646874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789012770845540:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:33.646916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:33.646897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789012992965009:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:33.647164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789012992964916:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.647209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:33.647264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789012992964916:2048] [node 1] ICP22 created new session: [5:7577789012992965016:2048]
2025-11-28T14:51:33.647483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789012992965016:2048] [node 1] ICS09 handshake done sender: [5:7577789012992965009:2120] self: [5:7577789012992965014:0] peer: [1:7577789012770845513:0] socket: 112 qp: -1
2025-11-28T14:51:33.647504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789012992965016:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:33.647532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:33.649334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789012231449499:2117] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:33.650032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789015665599991:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.650228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789015665599897:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:33.650423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789012231449499:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:33.650705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789012231449499:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:33.650847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789012231449489:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:33.650903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:33.650930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789012231449489:2110] [node 6] ICP22 created new session: [2:7577789012231449524:2110]
2025-11-28T14:51:33.650965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789012231449524:2110] [node 6] ICS09 handshake done sender: [2:7577789012231449499:2117] self: [2:7577789012231449498:0] peer: [6:7577789015665599996:0] socket: 132 qp: -1
2025-11-28T14:51:33.650990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789012231449524:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:33.651024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:33.651027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789015665599997:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-11-28T14:51:33.651250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789015665599991:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:33.651662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789015665599897:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4436) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789015665599991:2122] held: no
2025-11-28T14:51:33.651772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789015665599897:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:33.651813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:33.651850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789015665599897:2049] [node 2] ICP22 created new session: [6:7577789015665599998:2049]
2025-11-28T14:51:33.651892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789015665599998:2049] [node 2] ICS09 handshake done sender: [6:7577789015665599991:2122] self: [6:7577789015665599996:0] peer: [2:7577789012231449498:0] socket: 170 qp: -1
2025-11-28T14:51:33.651930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789015665599998:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:33.651986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:33.654019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:33.654224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789012231449519:2048] [node 1] ICS01 socket: 121 reason# EndOfStream
2025-11-28T14:51:33.654263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789012231449419:2048] [node 1] ICP30 unregister session Session# [2:7577789012231449519:2048] VirtualId# [2:7577789012231449516:0]
2025-11-28T14:51:33.654297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789012231449519:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.654368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.654422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.655130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:33.655335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789014111889096:2048] [node 1] ICS07 socket disconnect 130 reason# EndOfStream
2025-11-28T14:51:33.655360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789014111889096:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.655446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.655494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789014111889096:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:33.656267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789014111888998:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.656905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789014111889096:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.656949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789014111888998:2048] [node 1] ICP30 unregister session Session# [3:7577789014111889096:2048] VirtualId# [3:7577789014111889094:0]
2025-11-28T14:51:33.656990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.657098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789014111888998:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.657118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.658467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:33.658654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789011461036602:2048] [node 1] ICS01 socket: 141 reason# EndOfStream
2025-11-28T14:51:33.658685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789011461036505:2048] [node 1] ICP30 unregister session Session# [4:7577789011461036602:2048] VirtualId# [4:7577789011461036600:0]
2025-11-28T14:51:33.658714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789011461036602:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.658778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.658843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.659347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:33.659557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789012992965016:2048] [node 1] ICS07 socket disconnect 112 reason# EndOfStream
2025-11-28T14:51:33.659584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789012992965016:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.659642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.659698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789012992965016:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:33.660099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:33.660282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789012992964916:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.660343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789015665599994:2048] [node 1] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-28T14:51:33.660397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789015665599994:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.660316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789012992965016:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.660480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:33.660338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789012992964916:2048] [node 1] ICP30 unregister session Session# [5:7577789012992965016:2048] VirtualId# [5:7577789012992965014:0]
2025-11-28T14:51:33.660415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.660540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789015665599994:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:33.660433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789012992964916:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.660442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.661347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789015665599896:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.661457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789015665599994:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.661490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789015665599896:2048] [node 1] ICP30 unregister session Session# [6:7577789015665599994:2048] VirtualId# [6:7577789015665599992:0]
2025-11-28T14:51:33.661528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:33.661561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789015665599896:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.661574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10682) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.666447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789014111888998:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:33.666494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:33.666695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789014111889100:2049] [node 2] ICS01 socket: 119 reason# EndOfStream
2025-11-28T14:51:33.666741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789014111888999:2049] [node 2] ICP30 unregister session Session# [3:7577789014111889100:2049] VirtualId# [3:7577789014111889098:0]
2025-11-28T14:51:33.666767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789014111889100:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.666865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:33.666956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:33.668389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:33.668566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789011461036607:2049] [node 2] ICS07 socket disconnect 111 reason# EndOfStream
2025-11-28T14:51:33.668597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789011461036607:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.668673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:33.668718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789011461036607:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:33.669187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:33.669340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789012992965012:2049] [node 2] ICS01 socket: 137 reason# EndOfStream
2025-11-28T14:51:33.669372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789012992964917:2049] [node 2] ICP30 unregister session Session# [5:7577789012992965012:2049] VirtualId# [5:7577789012992965010:0]
2025-11-28T14:51:33.669375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789011461036506:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4436) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.669392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789012992965012:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.669466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789011461036607:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.669465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:33.669511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789011461036506:2049] [node 2] ICP30 unregister session Session# [4:7577789011461036607:2049] VirtualId# [4:7577789011461036605:0]
2025-11-28T14:51:33.669508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:33.669569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:33.669608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789011461036506:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4436) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.669620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4436) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.670075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:33.670299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789015665599998:2049] [node 2] ICS01 socket: 170 reason# EndOfStream
2025-11-28T14:51:33.670342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789015665599897:2049] [node 2] ICP30 unregister session Session# [6:7577789015665599998:2049] VirtualId# [6:7577789015665599996:0]
2025-11-28T14:51:33.670373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789015665599998:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.670457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:33.670514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:33.670507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789012992964916:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:33.671522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789015665599896:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:33.679804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789011461036506:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:33.705803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:33.705997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789011461036590:2050] [node 3] ICS01 socket: 110 reason# EndOfStream
2025-11-28T14:51:33.706029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789011461036507:2050] [node 3] ICP30 unregister session Session# [4:7577789011461036590:2050] VirtualId# [4:7577789011461036586:0]
2025-11-28T14:51:33.706050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789011461036590:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.706141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:33.706189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:33.706501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:33.706625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789012992965004:2050] [node 3] ICS07 socket disconnect 146 reason# EndOfStream
2025-11-28T14:51:33.706649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789012992965004:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.706692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:33.706748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789012992965004:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:33.707087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:33.707333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789015665599984:2050] [node 3] ICS01 socket: 155 reason# EndOfStream
2025-11-28T14:51:33.707375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789015665599898:2050] [node 3] ICP30 unregister session Session# [6:7577789015665599984:2050] VirtualId# [6:7577789015665599981:0]
2025-11-28T14:51:33.707371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789012992964918:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27591) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.707406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789015665599984:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.707502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:33.707409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789012992965004:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.707549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:33.707467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789012992964918:2050] [node 3] ICP30 unregister session Session# [5:7577789012992965004:2050] VirtualId# [5:7577789012992965001:0]
2025-11-28T14:51:33.707508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:33.707535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789012992964918:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27591) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.707560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27591) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.717667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789012992964918:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:33.733461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:33.733697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789012992965007:2108] [node 4] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-28T14:51:33.733736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789012992965007:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.733832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:33.733893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789012992965007:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:33.734093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:33.734324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789015665599988:2106] [node 4] ICS01 socket: 161 reason# EndOfStream
2025-11-28T14:51:33.734376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789015665599959:2106] [node 4] ICP30 unregister session Session# [6:7577789015665599988:2106] VirtualId# [6:7577789015665599985:0]
2025-11-28T14:51:33.734411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789015665599988:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.734547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789012992964980:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11854) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.734639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789012992965007:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.734505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:33.734558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:33.734686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789012992964980:2108] [node 4] ICP30 unregister session Session# [5:7577789012992965007:2108] VirtualId# [5:7577789012992965003:0]
2025-11-28T14:51:33.734730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:33.734753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789012992964980:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11854) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.734775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11854) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.757781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:33.757993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789015665599977:2104] [node 5] ICS07 socket disconnect 134 reason# EndOfStream
2025-11-28T14:51:33.758016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789015665599977:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:33.758087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:33.758126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789015665599977:2104] [node 5] ICS15 start handshake
2025-11-28T14:51:33.758865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789015665599960:2104] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23433) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:33.758923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789015665599977:2104] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:33.759010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789015665599960:2104] [node 5] ICP30 unregister session Session# [6:7577789015665599977:2104] VirtualId# [6:7577789015665599975:0]
2025-11-28T14:51:33.759066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:33.759101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789015665599960:2104] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:33.759119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:33.803517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789014765246752:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:33.803587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789014765246751:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:33.820226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789012641025314:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:33.820481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789012641025313:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.838966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789014680944788:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.838999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789014680944789:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:33.858524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789015167732194:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.858557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789015167732195:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:33.858573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789015167732196:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:33.881025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789011552370520:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.881074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789011552370521:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:33.881142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789011552370522:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:33.899095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789014238009655:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:33.899185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789014238009656:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:33.899396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789014238009657:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:33.966989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789014238009715:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:34.004750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789014765246823:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:34.029695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789018975912147:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:34.030972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789012641025376:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:34.032166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789015847337879:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:34.031529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789019462699553:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.035104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789016935992673:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:34.031113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789019060214120:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:34.032400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789018975912148:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:34.035626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789019462699554:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:34.036507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789015847337880:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.038062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789011552370521:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58472200ns) {Configure:104(58468727ns) {ProcessConfigured:135(59540ns) {StartInitialHandshake:157(57472ns) {DropHandshakes:470(5053ns) {DropIncomingHandshake:442(586ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:477(833ns) {SwitchToState:179(289ns)}}}}}
2025-11-28T14:51:34.035826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789019060214121:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.039773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789016935992674:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.040247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789018975912149:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.044596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789014765246752:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(74553834ns) {Configure:104(74550676ns) {ProcessConfigured:135(58233ns) {StartInitialHandshake:157(56328ns) {DropHandshakes:470(2300ns) {DropIncomingHandshake:442(710ns) DropOutgoingHandshake:457(403ns)} GenerateSessionVirtualId:988(929ns) PrepareNewSessionHandshake:477(1190ns) {SwitchToState:179(430ns)}}}}}
2025-11-28T14:51:34.046463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789014765246751:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(78773635ns) {Configure:104(78769933ns) {ProcessConfigured:135(535809ns) {StartInitialHandshake:157(533546ns) {DropHandshakes:470(1182ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(571ns) PrepareNewSessionHandshake:477(2041ns) {SwitchToState:179(860ns)}}}}}
2025-11-28T14:51:34.047008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789014680944788:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49604715ns) {Configure:104(49586869ns) {ProcessConfigured:135(387596ns) {StartInitialHandshake:157(385334ns) {DropHandshakes:470(1726ns) {DropIncomingHandshake:442(389ns) DropOutgoingHandshake:457(383ns)} GenerateSessionVirtualId:988(486ns) PrepareNewSessionHandshake:477(1198ns) {SwitchToState:179(502ns)}}}}}
2025-11-28T14:51:34.047131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789014680944789:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(61737334ns) {Configure:104(61731696ns) {ProcessConfigured:135(499933ns) {StartInitialHandshake:157(499222ns) {DropHandshakes:470(520ns) {DropIncomingHandshake:442(208ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(769ns) {SwitchToState:179(258ns)}}}}}
2025-11-28T14:51:34.047136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789015167732194:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(68861690ns) {Configure:104(68856853ns) {ProcessConfigured:135(404310ns) {StartInitialHandshake:157(398856ns) {DropHandshakes:470(1248ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(815ns) PrepareNewSessionHandshake:477(1108ns) {SwitchToState:179(415ns)}}}}}
2025-11-28T14:51:34.047620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789014238009656:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(64822239ns) {Configure:104(64820256ns) {ProcessConfigured:135(356214ns) {StartInitialHandshake:157(354098ns) {DropHandshakes:470(1243ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(460ns) PrepareNewSessionHandshake:477(1345ns) {SwitchToState:179(603ns)}}}}}
2025-11-28T14:51:34.048339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789014238009655:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(66528143ns) {Configure:104(66524707ns) {ProcessConfigured:135(1038020ns) {StartInitialHandshake:157(1036074ns) {DropHandshakes:470(1002ns) {DropIncomingHandshake:442(231ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(1544ns) {SwitchToState:179(727ns)}}}}}
2025-11-28T14:51:34.048321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789011552370520:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(67504803ns) {Configure:104(67500858ns) {ProcessConfigured:135(757324ns) {StartInitialHandshake:157(620227ns) {DropHandshakes:470(1333ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(258ns)} GenerateSessionVirtualId:988(648ns) PrepareNewSessionHandshake:477(1277ns) {SwitchToState:179(603ns)}}}}}
2025-11-28T14:51:34.048541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789014238009715:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(79616250ns) {Configure:104(77368224ns) {ProcessConfigured:135(576698ns) {StartInitialHandshake:157(574429ns) {DropHandshakes:470(1030ns) {DropIncomingHandshake:442(338ns) DropOutgoingHandshake:457(159ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:477(2534ns) {SwitchToState:179(1348ns)}}}}}
2025-11-28T14:51:34.049728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789018532977021:2109] [node 4] ICP01 ready to work
2025-11-28T14:51:34.050587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789012641025313:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(80411574ns) {Configure:104(80405341ns) {ProcessConfigured:135(608843ns) {StartInitialHandshake:157(606882ns) {DropHandshakes:470(1368ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(504ns)} GenerateSessionVirtualId:988(562ns) PrepareNewSessionHandshake:477(1299ns) {SwitchToState:179(496ns)}}}}}
2025-11-28T14:51:34.050755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789012641025314:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(79342488ns) {Configure:104(79339080ns) {ProcessConfigured:135(566127ns) {StartInitialHandshake:157(564443ns) {DropHandshakes:470(1265ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(498ns) PrepareNewSessionHandshake:477(1592ns) {SwitchToState:179(697ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.056192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019462699561:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54069
2025-11-28T14:51:34.058962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789015847337892:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15683
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.062617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789019060214135:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46951
2025-11-28T14:51:34.064308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019462699561:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.063003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789015847337892:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.063395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789018975912156:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63036
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.063835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789019060214138:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46951
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.064941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789011552370520:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.064491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214135:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.065696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789015167732194:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.065731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789018975912161:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63036
2025-11-28T14:51:34.065857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789015847337895:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15683
2025-11-28T14:51:34.065797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214127:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.069457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789016935992674:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16150324ns) {Configure:104(16145253ns) {ProcessConfigured:135(1802383ns) {StartInitialHandshake:157(1800775ns) {DropHandshakes:470(1791ns) {DropIncomingHandshake:442(556ns) DropOutgoingHandshake:457(435ns)} GenerateSessionVirtualId:988(733ns) PrepareNewSessionHandshake:477(2243ns) {SwitchToState:179(1130ns)}}}}}
2025-11-28T14:51:34.065798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789018975912156:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.065818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789019060214127:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.070307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789015847337895:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.068262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789018975912163:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63036
2025-11-28T14:51:34.068346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789019060214137:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46951
2025-11-28T14:51:34.068731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789014680944788:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.069916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214125:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.069392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789018975912161:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.069952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789019060214125:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.070915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789019060214136:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46951
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.070943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214123:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.070974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789019060214123:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.082679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019462699570:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54069
2025-11-28T14:51:34.072105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214137:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.072353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789019060214127:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:34.083163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019462699571:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54069
2025-11-28T14:51:34.073450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789019060214125:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:34.081775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789019060214123:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.082718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789019060214120:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.082761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789014765246752:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.082784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.083293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789015847337898:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15683
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.082799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.083553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789018975912168:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63036
2025-11-28T14:51:34.082922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789014765246823:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.083894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789016935992683:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40747
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.083804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789015847337892:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.082938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789019060214120:2108] [node 5] ICP22 created new session: [1:7577789019060214144:2108]
2025-11-28T14:51:34.083997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789016935992688:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40747
2025-11-28T14:51:34.083023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789014765246752:2049] [node 3] ICP22 created new session: [1:7577789019060214145:2049]
2025-11-28T14:51:34.084126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789016935992686:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40747
2025-11-28T14:51:34.084403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019462699561:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.084445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789011552370520:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.083051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:34.084147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789016935992687:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40747
2025-11-28T14:51:34.084502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:34.083164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789014765246823:2107] [node 4] ICP22 created new session: [1:7577789019060214146:2107]
2025-11-28T14:51:34.084819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789016935992683:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.085000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789011552370520:2048] [node 1] ICP22 created new session: [5:7577789015847337899:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.085018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789016935992686:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.083263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789019060214145:2049] [node 3] ICS09 handshake done sender: [1:7577789019060214123:2119] self: [1:7577789019060214122:0] peer: [3:7577789018975912166:0] socket: 108 qp: -1
2025-11-28T14:51:34.085932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789015847337899:2048] [node 1] ICS09 handshake done sender: [5:7577789015847337892:2115] self: [5:7577789015847337896:0] peer: [1:7577789019060214126:0] socket: 112 qp: -1
2025-11-28T14:51:34.085970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789015847337899:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.086008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.083274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789019060214146:2107] [node 4] ICS09 handshake done sender: [1:7577789019060214125:2120] self: [1:7577789019060214124:0] peer: [4:7577789019462699569:0] socket: 109 qp: -1
2025-11-28T14:51:34.083308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789019060214145:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:34.083312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789019060214146:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:34.088783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977029:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.083351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.083355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789019060214144:2108] [node 5] ICS09 handshake done sender: [1:7577789019060214127:2121] self: [1:7577789019060214126:0] peer: [5:7577789015847337896:0] socket: 110 qp: -1
2025-11-28T14:51:34.083387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789019060214144:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:34.083428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:34.083781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:34.088258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789019060214150:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46951
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.091921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789016935992694:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40747
2025-11-28T14:51:34.092399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789018532977029:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.092491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789016935992694:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.092591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789014238009655:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.092675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789012641025313:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.092913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214129:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.092959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789019060214129:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.092991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214131:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.093036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789019060214131:2123] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.094306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789019060214129:2122] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:34.094311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789019060214131:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.094555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789014765246751:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.094606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.094659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789014765246751:2048] [node 2] ICP22 created new session: [1:7577789019060214155:2048]
2025-11-28T14:51:34.094868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789019060214155:2048] [node 2] ICS09 handshake done sender: [1:7577789019060214131:2123] self: [1:7577789019060214130:0] peer: [2:7577789016935992695:0] socket: 156 qp: -1
2025-11-28T14:51:34.094911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789019060214155:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:34.095146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.095365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789019060214121:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.095615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977031:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.095475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789016935992696:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40747
2025-11-28T14:51:34.095486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.095763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789019060214121:2109] [node 6] ICP22 created new session: [1:7577789019060214157:2109]
2025-11-28T14:51:34.095925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789019060214157:2109] [node 6] ICS09 handshake done sender: [1:7577789019060214129:2122] self: [1:7577789019060214128:0] peer: [6:7577789018532977030:0] socket: 116 qp: -1
2025-11-28T14:51:34.097777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789014238009655:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789018532977029:2115] held: no
2025-11-28T14:51:34.095960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789019060214157:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.100102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789018532977029:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.098091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789016935992694:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.100493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789012641025313:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.095996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.100703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789014238009655:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.100672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:34.100756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:34.100889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789012641025313:2048] [node 1] ICP22 created new session: [2:7577789016935992697:2048]
2025-11-28T14:51:34.101053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789014238009655:2048] [node 1] ICP22 created new session: [6:7577789018532977033:2048]
2025-11-28T14:51:34.101084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789016935992697:2048] [node 1] ICS09 handshake done sender: [2:7577789016935992694:2119] self: [2:7577789016935992695:0] peer: [1:7577789019060214130:0] socket: 157 qp: -1
2025-11-28T14:51:34.101209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789018532977033:2048] [node 1] ICS09 handshake done sender: [6:7577789018532977029:2115] self: [6:7577789018532977030:0] peer: [1:7577789019060214128:0] socket: 155 qp: -1
2025-11-28T14:51:34.101149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789016935992697:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.104486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789018532977033:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.101196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:34.104882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:34.111496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789019462699556:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.119062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789015167732194:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.119144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:34.119197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789015167732194:2048] [node 1] ICP22 created new session: [4:7577789019462699573:2048]
2025-11-28T14:51:34.119238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019462699573:2048] [node 1] ICS09 handshake done sender: [4:7577789019462699561:2113] self: [4:7577789019462699569:0] peer: [1:7577789019060214124:0] socket: 120 qp: -1
2025-11-28T14:51:34.119272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019462699573:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.119308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:34.120614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214136:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.120882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019462699572:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54069
2025-11-28T14:51:34.120880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789014765246823:2107] [node 4] ICP18 handshake (actor [1:7577789019060214136:2125]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.121147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019462699575:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54069
2025-11-28T14:51:34.121548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019462699575:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.122212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977035:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.122397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789019462699559:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.134481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789018975912151:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.144353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789018975912156:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.144437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214138:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.144705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789014765246752:2049] [node 3] ICP18 handshake (actor [1:7577789019060214138:2127]) is too old
2025-11-28T14:51:34.144867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789018975912163:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.144954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789014680944788:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.145012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:34.145054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789014680944788:2048] [node 1] ICP22 created new session: [3:7577789018975912171:2048]
2025-11-28T14:51:34.145102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789018975912171:2048] [node 1] ICS09 handshake done sender: [3:7577789018975912156:2113] self: [3:7577789018975912166:0] peer: [1:7577789019060214122:0] socket: 115 qp: -1
2025-11-28T14:51:34.145185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789018975912171:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.145242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.146020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789018975912169:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63036
2025-11-28T14:51:34.148145Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789018975912155:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.149432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019462699570:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.149580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789018975912170:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63036
2025-11-28T14:51:34.149579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789015167732196:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.149814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789018975912155:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.150328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789018975912155:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:34.158574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789015847337902:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15683
::ffff:127.1.0.0
2025-11-28T14:51:34.158797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019462699579:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54069
2025-11-28T14:51:34.159012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789018975912170:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.159108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789018975912147:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.159226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:34.159283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789018975912147:2107] [node 4] ICP22 created new session: [3:7577789018975912176:2107]
2025-11-28T14:51:34.159358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789018975912176:2107] [node 4] ICS09 handshake done sender: [3:7577789018975912155:2112] self: [3:7577789018975912154:0] peer: [4:7577789019462699578:0] socket: 137 qp: -1
2025-11-28T14:51:34.159415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789018975912176:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:34.159461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.159434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019462699570:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:34.159753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789015167732196:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.159815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:34.159865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789015167732196:2050] [node 3] ICP22 created new session: [4:7577789019462699580:2050]
2025-11-28T14:51:34.159933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019462699580:2050] [node 3] ICS09 handshake done sender: [4:7577789019462699570:2116] self: [4:7577789019462699578:0] peer: [3:7577789018975912154:0] socket: 139 qp: -1
2025-11-28T14:51:34.159983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019462699580:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:34.160016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.162182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789015847337903:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15683
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.164356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977036:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.172463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789018975912153:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.194227Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789019462699560:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.215077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789016935992678:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.223889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789016935992688:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.223996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789016935992687:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.224449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789019060214150:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.224637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789014765246751:2048] [node 2] ICP18 handshake (actor [1:7577789019060214150:2128]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.228835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789015847337904:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15683
2025-11-28T14:51:34.229850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789016935992680:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.239593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019462699572:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.239712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789015167732195:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.239968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789016935992680:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.240404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789016935992680:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.240731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019462699584:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54069
2025-11-28T14:51:34.241137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789012641025376:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.241244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019462699572:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.241218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:34.241278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789012641025376:2107] [node 4] ICP22 created new session: [2:7577789016935992703:2107]
2025-11-28T14:51:34.241322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789016935992703:2107] [node 4] ICS09 handshake done sender: [2:7577789016935992680:2112] self: [2:7577789016935992679:0] peer: [4:7577789019462699583:0] socket: 151 qp: -1
2025-11-28T14:51:34.241454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789016935992703:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:34.241502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.241575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789015167732195:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.241623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:34.241663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789015167732195:2049] [node 2] ICP22 created new session: [4:7577789019462699585:2049]
2025-11-28T14:51:34.241709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019462699585:2049] [node 2] ICS09 handshake done sender: [4:7577789019462699572:2118] self: [4:7577789019462699583:0] peer: [2:7577789016935992679:0] socket: 152 qp: -1
2025-11-28T14:51:34.241742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019462699585:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:34.241771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.243378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789019462699566:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.250661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789018532977035:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.250893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789018532977021:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.251448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019462699566:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.251640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977038:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.251797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019462699566:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.251951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789019462699553:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.252002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.252022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977039:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.252111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789019462699553:2109] [node 6] ICP22 created new session: [4:7577789019462699588:2109]
2025-11-28T14:51:34.252405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019462699588:2109] [node 6] ICS09 handshake done sender: [4:7577789019462699566:2114] self: [4:7577789019462699565:0] peer: [6:7577789018532977037:0] socket: 121 qp: -1
2025-11-28T14:51:34.252435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019462699588:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.252636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789018532977035:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:34.252674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.252912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789018532977021:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.253021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:34.253043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789018532977021:2109] [node 4] ICP22 created new session: [6:7577789018532977040:2109]
2025-11-28T14:51:34.253068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789018532977040:2109] [node 4] ICS09 handshake done sender: [6:7577789018532977035:2117] self: [6:7577789018532977037:0] peer: [4:7577789019462699565:0] socket: 126 qp: -1
2025-11-28T14:51:34.253088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789018532977040:2109] [node 4] ICS10 traffic start
2025-11-28T14:51:34.253168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.255770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789016935992677:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.267691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789018975912169:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.267865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789014680944789:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.269309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789016935992677:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.268072Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789018975912159:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.276263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789016935992677:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.276613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789015847337902:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.276768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789018975912181:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63036
2025-11-28T14:51:34.276802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789012641025314:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.276863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789011552370522:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.276942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:34.277004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789012641025314:2049] [node 3] ICP22 created new session: [2:7577789016935992707:2049]
2025-11-28T14:51:34.277052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789016935992707:2049] [node 3] ICS09 handshake done sender: [2:7577789016935992677:2110] self: [2:7577789016935992675:0] peer: [3:7577789018975912180:0] socket: 150 qp: -1
2025-11-28T14:51:34.277086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789016935992707:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:34.277176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:34.277055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789018975912169:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.277423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789014680944789:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.277447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789018975912159:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.277520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:34.277566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789014680944789:2049] [node 2] ICP22 created new session: [3:7577789018975912182:2049]
2025-11-28T14:51:34.277608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789018975912182:2049] [node 2] ICS09 handshake done sender: [3:7577789018975912169:2119] self: [3:7577789018975912180:0] peer: [2:7577789016935992675:0] socket: 154 qp: -1
2025-11-28T14:51:34.277643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789018975912182:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:34.277679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.278457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789018975912159:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:34.278622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789018975912148:2108] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:51:34.278680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.278733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789018975912148:2108] [node 5] ICP22 created new session: [3:7577789018975912185:2108]
2025-11-28T14:51:34.278752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789015847337906:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15683
2025-11-28T14:51:34.278800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789018975912185:2108] [node 5] ICS09 handshake done sender: [3:7577789018975912159:2114] self: [3:7577789018975912157:0] peer: [5:7577789015847337905:0] socket: 133 qp: -1
2025-11-28T14:51:34.278840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789018975912185:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:34.278866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:34.278968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789015847337902:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:34.280428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789011552370522:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789015847337902:2118] held: no
2025-11-28T14:51:34.280565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789011552370522:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.280692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:34.280882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789011552370522:2050] [node 3] ICP22 created new session: [5:7577789015847337907:2050]
2025-11-28T14:51:34.280952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789015847337907:2050] [node 3] ICS09 handshake done sender: [5:7577789015847337902:2118] self: [5:7577789015847337905:0] peer: [3:7577789018975912157:0] socket: 164 qp: -1
2025-11-28T14:51:34.280986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789015847337907:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:34.281027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.281491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977042:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.283167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789019462699568:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.291167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789018532977042:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.291406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789014238009715:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.291420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789015847337903:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.291649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789015847337879:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.291674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789015847337890:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.291698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789015847337890:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.291997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019462699568:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.292701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019462699568:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:34.292739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789015847337912:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15683
2025-11-28T14:51:34.292753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789015847337890:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.292998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977044:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.293046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789015847337903:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:34.293557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789015847337880:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.294232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789018532977042:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:34.293421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789019462699554:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.293496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.294429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789014238009715:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.294490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.295767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789014238009715:2106] [node 5] ICP22 created new session: [6:7577789018532977045:2106]
2025-11-28T14:51:34.293547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789019462699554:2108] [node 5] ICP22 created new session: [4:7577789019462699591:2108]
2025-11-28T14:51:34.293614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.297481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789018532977045:2106] [node 5] ICS09 handshake done sender: [6:7577789018532977042:2121] self: [6:7577789018532977043:0] peer: [5:7577789015847337889:0] socket: 125 qp: -1
2025-11-28T14:51:34.294051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019462699591:2108] [node 5] ICS09 handshake done sender: [4:7577789019462699568:2115] self: [4:7577789019462699567:0] peer: [5:7577789015847337910:0] socket: 123 qp: -1
2025-11-28T14:51:34.293643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789015847337880:2109] [node 6] ICP22 created new session: [5:7577789015847337913:2109]
2025-11-28T14:51:34.297519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789018532977045:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:34.297612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:34.294213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019462699591:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:34.293673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789015847337913:2109] [node 6] ICS09 handshake done sender: [5:7577789015847337890:2114] self: [5:7577789015847337889:0] peer: [6:7577789018532977043:0] socket: 117 qp: -1
2025-11-28T14:51:34.293693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789015847337913:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.294255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:34.293718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.294220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789015847337879:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.294266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:34.294570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789015847337879:2108] [node 4] ICP22 created new session: [5:7577789015847337915:2108]
2025-11-28T14:51:34.294613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789015847337915:2108] [node 4] ICS09 handshake done sender: [5:7577789015847337903:2119] self: [5:7577789015847337910:0] peer: [4:7577789019462699567:0] socket: 166 qp: -1
2025-11-28T14:51:34.294640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789015847337915:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:34.294667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.297696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789015847337879:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:34.300025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789018975912160:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.308432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789018532977036:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.308642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789014238009657:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.308891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789018975912160:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.309509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789018975912160:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.309897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977048:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.309752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789018975912149:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.309863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.310349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789018532977036:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:34.309931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789018975912149:2109] [node 6] ICP22 created new session: [3:7577789018975912188:2109]
2025-11-28T14:51:34.309982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789018975912188:2109] [node 6] ICS09 handshake done sender: [3:7577789018975912160:2115] self: [3:7577789018975912158:0] peer: [6:7577789018532977047:0] socket: 128 qp: -1
2025-11-28T14:51:34.310017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789018975912188:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.311341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789014238009657:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789018532977036:2118] held: no
2025-11-28T14:51:34.310071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.311456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789014238009657:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.311504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:34.311655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789014238009657:2050] [node 3] ICP22 created new session: [6:7577789018532977049:2050]
2025-11-28T14:51:34.311692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789018532977049:2050] [node 3] ICS09 handshake done sender: [6:7577789018532977036:2118] self: [6:7577789018532977047:0] peer: [3:7577789018975912158:0] socket: 167 qp: -1
2025-11-28T14:51:34.311728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789018532977049:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:34.311775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:34.313232Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789016935992682:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.321043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789015847337904:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.321269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789011552370521:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.321549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789016935992682:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.321942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789016935992682:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.322393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789015847337918:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15683
2025-11-28T14:51:34.322617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789015847337904:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.322218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789016935992673:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.322283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.322337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789016935992673:2108] [node 5] ICP22 created new session: [2:7577789016935992710:2108]
2025-11-28T14:51:34.322476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789016935992710:2108] [node 5] ICS09 handshake done sender: [2:7577789016935992682:2113] self: [2:7577789016935992681:0] peer: [5:7577789015847337917:0] socket: 149 qp: -1
2025-11-28T14:51:34.323239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789011552370521:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.323314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:34.323353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789011552370521:2049] [node 2] ICP22 created new session: [5:7577789015847337919:2049]
2025-11-28T14:51:34.322512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789016935992710:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:34.323928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789015847337919:2049] [node 2] ICS09 handshake done sender: [5:7577789015847337904:2120] self: [5:7577789015847337917:0] peer: [2:7577789016935992681:0] socket: 114 qp: -1
2025-11-28T14:51:34.322569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:34.324646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789015847337919:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:34.324736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.324952Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789011552370521:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:34.327804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789016935992685:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.328484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789018532977038:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.328691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789014238009656:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.328896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789016935992685:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.329459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789018532977052:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-28T14:51:34.329548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789016935992685:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:34.329702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789016935992674:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.329776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.329879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789016935992674:2109] [node 6] ICP22 created new session: [2:7577789016935992713:2109]
2025-11-28T14:51:34.329973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789016935992713:2109] [node 6] ICS09 handshake done sender: [2:7577789016935992685:2115] self: [2:7577789016935992684:0] peer: [6:7577789018532977051:0] socket: 127 qp: -1
2025-11-28T14:51:34.330024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789016935992713:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.330012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789018532977038:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.330122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.330312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789014238009656:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11167) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789018532977038:2119] held: no
2025-11-28T14:51:34.330423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789014238009656:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.330488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:34.330540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789014238009656:2049] [node 2] ICP22 created new session: [6:7577789018532977053:2049]
2025-11-28T14:51:34.330593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789018532977053:2049] [node 2] ICS09 handshake done sender: [6:7577789018532977038:2119] self: [6:7577789018532977051:0] peer: [2:7577789016935992684:0] socket: 140 qp: -1
2025-11-28T14:51:34.330624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789018532977053:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:34.330666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.359793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:34.360052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789016935992697:2048] [node 1] ICS07 socket disconnect 157 reason# EndOfStream
2025-11-28T14:51:34.360094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789016935992697:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.360212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:34.360291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789016935992697:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:34.360969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:34.361188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789018975912171:2048] [node 1] ICS07 socket disconnect 115 reason# EndOfStream
2025-11-28T14:51:34.361217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789018975912171:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.361301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:34.361385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789018975912171:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:34.362614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789012641025313:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:34.362666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789016935992697:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:34.363981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789014680944788:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:34.364450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789018975912171:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:34.364492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789014680944788:2048] [node 1] ICP30 unregister session Session# [3:7577789018975912171:2048] VirtualId# [3:7577789018975912166:0]
2025-11-28T14:51:34.364553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:34.364586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789014680944788:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.364604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.366005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:34.366142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789012641025313:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:34.366221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789016935992697:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:34.366232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789019462699573:2048] [node 1] ICS01 socket: 120 reason# EndOfStream
2025-11-28T14:51:34.366253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789012641025313:2048] [node 1] ICP30 unregister session Session# [2:7577789016935992697:2048] VirtualId# [2:7577789016935992695:0]
2025-11-28T14:51:34.366274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789015167732194:2048] [node 1] ICP30 unregister session Session# [4:7577789019462699573:2048] VirtualId# [4:7577789019462699569:0]
2025-11-28T14:51:34.366297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:34.366301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789019462699573:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.366329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789012641025313:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.366341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.366375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:34.366424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:34.367066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:34.367248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789015847337899:2048] [node 1] ICS07 socket disconnect 112 reason# EndOfStream
2025-11-28T14:51:34.367286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789015847337899:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.367359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:34.367413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789015847337899:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:34.367760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:34.367991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789018532977033:2048] [node 1] ICS01 socket: 155 reason# EndOfStream
2025-11-28T14:51:34.368043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789014238009655:2048] [node 1] ICP30 unregister session Session# [6:7577789018532977033:2048] VirtualId# [6:7577789018532977030:0]
2025-11-28T14:51:34.368024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789011552370520:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:34.368079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789018532977033:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.368088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789015847337899:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:34.368162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:34.368224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:34.368168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789011552370520:2048] [node 1] ICP30 unregister session Session# [5:7577789015847337899:2048] VirtualId# [5:7577789015847337896:0]
2025-11-28T14:51:34.368222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:34.368249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789011552370520:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.368259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.373923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:34.374071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789014680944788:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:34.374095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789018975912182:2049] [node 2] ICS01 socket: 154 reason# EndOfStream
2025-11-28T14:51:34.374122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789014680944789:2049] [node 2] ICP30 unregister session Session# [3:7577789018975912182:2049] VirtualId# [3:7577789018975912180:0]
2025-11-28T14:51:34.374146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789018975912182:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.374224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:34.374275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:34.376334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:34.376566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789019462699585:2049] [node 2] ICS01 socket: 152 reason# EndOfStream
2025-11-28T14:51:34.376605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789015167732195:2049] [node 2] ICP30 unregister session Session# [4:7577789019462699585:2049] VirtualId# [4:7577789019462699583:0]
2025-11-28T14:51:34.376633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789019462699585:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.376706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:34.376756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:34.377390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:34.377604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789015847337919:2049] [node 2] ICS07 socket disconnect 114 reason# EndOfStream
2025-11-28T14:51:34.377640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789015847337919:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.377711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:34.377756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789015847337919:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:34.377796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:34.378021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789018532977053:2049] [node 2] ICS01 socket: 140 reason# EndOfStream
2025-11-28T14:51:34.378082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789014238009656:2049] [node 2] ICP30 unregister session Session# [6:7577789018532977053:2049] VirtualId# [6:7577789018532977051:0]
2025-11-28T14:51:34.378112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789018532977053:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.378130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789011552370520:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:34.378176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:34.378220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:34.378429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789011552370521:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11167) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:34.378476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789015847337919:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:34.378567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789011552370521:2049] [node 2] ICP30 unregister session Session# [5:7577789015847337919:2049] VirtualId# [5:7577789015847337917:0]
2025-11-28T14:51:34.378608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:34.378636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789011552370521:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.378649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.388265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:34.388515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789019462699580:2050] [node 3] ICS01 socket: 139 reason# EndOfStream
2025-11-28T14:51:34.388566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789015167732196:2050] [node 3] ICP30 unregister session Session# [4:7577789019462699580:2050] VirtualId# [4:7577789019462699578:0]
2025-11-28T14:51:34.388532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789011552370521:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:34.388597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789019462699580:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.388693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:34.388759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:34.389180Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:34.389370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789015847337907:2050] [node 3] ICS01 socket: 164 reason# EndOfStream
2025-11-28T14:51:34.389409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789011552370522:2050] [node 3] ICP30 unregister session Session# [5:7577789015847337907:2050] VirtualId# [5:7577789015847337905:0]
2025-11-28T14:51:34.389430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789015847337907:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.389507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:34.389549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:34.389856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:34.390046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789018532977049:2050] [node 3] ICS01 socket: 167 reason# EndOfStream
2025-11-28T14:51:34.390098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789014238009657:2050] [node 3] ICP30 unregister session Session# [6:7577789018532977049:2050] VirtualId# [6:7577789018532977047:0]
2025-11-28T14:51:34.390146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789018532977049:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.390212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:34.390262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:34.397369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:34.397588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789015847337915:2108] [node 4] ICS01 socket: 166 reason# EndOfStream
2025-11-28T14:51:34.397638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789015847337879:2108] [node 4] ICP30 unregister session Session# [5:7577789015847337915:2108] VirtualId# [5:7577789015847337910:0]
2025-11-28T14:51:34.398064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:34.397673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789015847337915:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.397736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:34.397773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:34.398263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789018532977040:2109] [node 4] ICS07 socket disconnect 126 reason# EndOfStream
2025-11-28T14:51:34.398307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789018532977040:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.398388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:34.398434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789018532977040:2109] [node 4] ICS15 start handshake
2025-11-28T14:51:34.399242Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789018532977021:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13779) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:34.399293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789018532977040:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:34.399377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789018532977021:2109] [node 4] ICP30 unregister session Session# [6:7577789018532977040:2109] VirtualId# [6:7577789018532977037:0]
2025-11-28T14:51:34.399443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:34.399495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789018532977021:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.399527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.408891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789018532977021:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:51:34.434240Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:34.434481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789018532977045:2106] [node 5] ICS01 socket: 125 reason# EndOfStream
2025-11-28T14:51:34.434525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789014238009715:2106] [node 5] ICP30 unregister session Session# [6:7577789018532977045:2106] VirtualId# [6:7577789018532977043:0]
2025-11-28T14:51:34.434558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789018532977045:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.434676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:34.434740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:34.496292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789018889724130:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:34.496419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789018889724131:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:34.515026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789018180747637:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:34.515072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789018180747638:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:34.534900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789017997876402:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:34.534932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789017997876403:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:34.554539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789019113319012:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:34.554542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789019113319014:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:34.554544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789019113319013:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:34.573000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789017741943878:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:34.573037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789017741943879:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:34.573098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789017741943880:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:34.592092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789016475398245:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:34.592117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789016475398247:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:34.592221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789016475398246:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:34.657228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789018889724193:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:34.677227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789018889724195:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.697899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789017997876466:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.697220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789018889724194:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:34.698099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789016475398306:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:34.698161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789018180747701:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.698355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789017741943941:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:34.698384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789019113319076:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:34.698978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789017997876465:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:34.698107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789018180747700:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:34.699108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789018180747702:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:34.699377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789017741943942:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.699414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789016475398309:2106] [node 4] ICP01 ready to work
2025-11-28T14:51:34.699452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789019113319077:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:34.699668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789017997876467:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:34.710444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789017997876402:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40288294ns) {Configure:104(32001281ns) {ProcessConfigured:135(5113959ns) {StartInitialHandshake:157(5112100ns) {DropHandshakes:470(1182ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(482ns) PrepareNewSessionHandshake:477(2091ns) {SwitchToState:179(919ns)}}}}}
2025-11-28T14:51:34.710615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789019113319013:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(59230959ns) {Configure:104(59226446ns) {ProcessConfigured:135(484708ns) {StartInitialHandshake:157(482331ns) {DropHandshakes:470(1221ns) {DropIncomingHandshake:442(387ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(562ns) PrepareNewSessionHandshake:477(1121ns) {SwitchToState:179(627ns)}}}}}
2025-11-28T14:51:34.712277Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789019113319014:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(57385669ns) {Configure:104(52738601ns) {ProcessConfigured:135(855804ns) {StartInitialHandshake:157(853740ns) {DropHandshakes:470(875ns) {DropIncomingHandshake:442(227ns) DropOutgoingHandshake:457(167ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:477(3662ns) {SwitchToState:179(2855ns)}}}}}
2025-11-28T14:51:34.713541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789019113319012:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53063566ns) {Configure:104(53058135ns) {ProcessConfigured:135(953055ns) {StartInitialHandshake:157(951197ns) {DropHandshakes:470(1140ns) {DropIncomingHandshake:442(340ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(412ns) PrepareNewSessionHandshake:477(1654ns) {SwitchToState:179(616ns)}}}}}
2025-11-28T14:51:34.713270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789018889724130:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(70957700ns) {Configure:104(70953328ns) {ProcessConfigured:135(2176856ns) {StartInitialHandshake:157(2175210ns) {DropHandshakes:470(1092ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(420ns) PrepareNewSessionHandshake:477(1242ns) {SwitchToState:179(541ns)}}}}}
2025-11-28T14:51:34.713424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789016475398246:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21533632ns) {Configure:104(16745272ns) {ProcessConfigured:135(1482298ns) {StartInitialHandshake:157(1480002ns) {DropHandshakes:470(1564ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:477(2031ns) {SwitchToState:179(993ns)}}}}}
2025-11-28T14:51:34.713422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789018889724131:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21649901ns) {Configure:104(21642051ns) {ProcessConfigured:135(2267420ns) {StartInitialHandshake:157(2119674ns) {DropHandshakes:470(11854ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(791ns) PrepareNewSessionHandshake:477(1339ns) {SwitchToState:179(620ns)}}}}}
2025-11-28T14:51:34.714309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789017741943880:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(55554568ns) {Configure:104(55543667ns) {ProcessConfigured:135(2147354ns) {StartInitialHandshake:157(2145304ns) {DropHandshakes:470(1532ns) {DropIncomingHandshake:442(737ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(611ns) PrepareNewSessionHandshake:477(3803ns) {SwitchToState:179(2040ns)}}}}}
2025-11-28T14:51:34.714349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789018180747637:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17326974ns) {Configure:104(17320205ns) {ProcessConfigured:135(1923324ns) {StartInitialHandshake:157(1921501ns) {DropHandshakes:470(1239ns) {DropIncomingHandshake:442(418ns) DropOutgoingHandshake:457(185ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(12964ns) {SwitchToState:179(684ns)}}}}}
2025-11-28T14:51:34.714971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789016475398247:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17683517ns) {Configure:104(17675926ns) {ProcessConfigured:135(697229ns) {StartInitialHandshake:157(695839ns) {DropHandshakes:470(993ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(351ns) PrepareNewSessionHandshake:477(1861ns) {SwitchToState:179(1047ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.718059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789017741943955:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13898
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.718516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789017741943954:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13898
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.720939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789018889724207:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62272
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.723678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789018889724208:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62272
2025-11-28T14:51:34.724544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398319:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.725449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019113319091:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35135
2025-11-28T14:51:34.724553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398318:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.725688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019113319092:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35135
2025-11-28T14:51:34.728007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789016475398318:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.727461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019113319092:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.728384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789017741943954:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.728930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789018889724211:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62272
2025-11-28T14:51:34.728345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019113319091:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.729486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789017741943958:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13898
2025-11-28T14:51:34.729698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789017741943941:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.729514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724208:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.729064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789019113319014:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.728425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789018180747711:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6986
2025-11-28T14:51:34.729869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789018180747712:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6986
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.729795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789016475398247:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.729891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789017741943955:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.730219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019113319083:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.730433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789018180747717:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6986
2025-11-28T14:51:34.730280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789017741943878:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.733487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398325:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
2025-11-28T14:51:34.730258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789019113319083:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.730228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789017997876475:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.734323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724200:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.732523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789018180747719:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6986
2025-11-28T14:51:34.730263Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789017997876475:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.743618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019113319083:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:34.734360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789018889724200:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.734234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789017997876472:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.733487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789018180747716:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6986
2025-11-28T14:51:34.734388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789018889724212:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62272
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.742603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789018180747716:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.734266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789017997876472:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.745006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789018889724200:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:34.746265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789018889724193:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.746337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.741439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789017997876481:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55659
2025-11-28T14:51:34.745864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019113319096:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35135
2025-11-28T14:51:34.742613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789017997876475:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:34.746830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789018889724193:2108] [node 5] ICP22 created new session: [1:7577789018889724215:2108]
2025-11-28T14:51:34.745531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789017997876465:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.745573Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789017997876472:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:34.745867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-28T14:51:34.746250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789017997876465:2107] [node 4] ICP22 created new session: [3:7577789017997876484:2107]
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.746811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789019113319076:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.746697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789017997876484:2107] [node 4] ICS09 handshake done sender: [3:7577789017997876475:2113] self: [3:7577789017997876469:0] peer: [4:7577789019113319094:0] socket: 117 qp: -1
2025-11-28T14:51:34.747191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789018889724215:2108] [node 5] ICS09 handshake done sender: [1:7577789018889724200:2110] self: [1:7577789018889724196:0] peer: [5:7577789017741943960:0] socket: 124 qp: -1
2025-11-28T14:51:34.747850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789017741943961:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13898
2025-11-28T14:51:34.746735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789017997876484:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:34.746854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.747247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789018889724215:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:34.747118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789019113319076:2108] [node 5] ICP22 created new session: [4:7577789019113319097:2108]
2025-11-28T14:51:34.747281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:34.748048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789017741943962:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13898
2025-11-28T14:51:34.746770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.746839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789017997876466:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:34.747311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019113319097:2108] [node 5] ICS09 handshake done sender: [4:7577789019113319083:2112] self: [4:7577789019113319080:0] peer: [5:7577789017741943959:0] socket: 116 qp: -1
::ffff:127.1.0.0
2025-11-28T14:51:34.747308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.748138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019113319097:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:34.748278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789017741943878:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789017741943955:2115] held: no
2025-11-28T14:51:34.747908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789017997876466:2109] [node 6] ICP22 created new session: [3:7577789017997876486:2109]
2025-11-28T14:51:34.749344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398327:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
2025-11-28T14:51:34.748301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:34.748479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789017997876486:2109] [node 6] ICS09 handshake done sender: [3:7577789017997876472:2110] self: [3:7577789017997876468:0] peer: [6:7577789016475398326:0] socket: 115 qp: -1
2025-11-28T14:51:34.749509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019113319091:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:34.749448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789017741943954:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:34.748742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789017997876486:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.749599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789017741943955:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.748857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.750011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789017741943941:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16265) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789017741943954:2114] held: no
2025-11-28T14:51:34.750893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789017741943941:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.751254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789019113319014:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.751465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:34.751531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789017741943878:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.751600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.751324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:34.751667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789017741943941:2108] [node 4] ICP22 created new session: [5:7577789017741943964:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.751774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789017741943878:2048] [node 1] ICP22 created new session: [5:7577789017741943966:2048]
2025-11-28T14:51:34.751534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789019113319014:2050] [node 3] ICP22 created new session: [4:7577789019113319100:2050]
2025-11-28T14:51:34.753604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789018889724218:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62272
2025-11-28T14:51:34.753695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789016475398318:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.751590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019113319100:2050] [node 3] ICS09 handshake done sender: [4:7577789019113319091:2116] self: [4:7577789019113319094:0] peer: [3:7577789017997876469:0] socket: 136 qp: -1
2025-11-28T14:51:34.751830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789017741943966:2048] [node 1] ICS09 handshake done sender: [5:7577789017741943955:2115] self: [5:7577789017741943960:0] peer: [1:7577789018889724196:0] socket: 125 qp: -1
2025-11-28T14:51:34.751894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789017741943964:2108] [node 4] ICS09 handshake done sender: [5:7577789017741943954:2114] self: [5:7577789017741943959:0] peer: [4:7577789019113319080:0] socket: 123 qp: -1
2025-11-28T14:51:34.752181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789017741943964:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:34.752212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.752316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789017741943966:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.753645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:34.753976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789017741943965:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13898
2025-11-28T14:51:34.755067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789017997876490:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55659
2025-11-28T14:51:34.751620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019113319100:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:34.751761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:34.755613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789017741943965:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.755221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724218:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.755738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789017741943880:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.755936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789017997876491:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55659
2025-11-28T14:51:34.756012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789017997876480:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.756046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789017997876480:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.756406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789017997876491:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.756395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789016475398247:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.756554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:34.756632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789016475398247:2050] [node 3] ICP22 created new session: [6:7577789016475398329:2050]
2025-11-28T14:51:34.756920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789016475398329:2050] [node 3] ICS09 handshake done sender: [6:7577789016475398318:2114] self: [6:7577789016475398326:0] peer: [3:7577789017997876468:0] socket: 131 qp: -1
2025-11-28T14:51:34.757101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789016475398329:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:34.757145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:34.758625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789017997876480:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.758878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789017997876467:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.758887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789017741943971:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13898
2025-11-28T14:51:34.759147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.759249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789017997876467:2108] [node 5] ICP22 created new session: [3:7577789017997876494:2108]
2025-11-28T14:51:34.759293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789017741943965:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:34.759313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789017997876494:2108] [node 5] ICS09 handshake done sender: [3:7577789017997876480:2114] self: [3:7577789017997876479:0] peer: [5:7577789017741943970:0] socket: 164 qp: -1
2025-11-28T14:51:34.759400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789017997876494:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:34.759609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:34.759661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789017741943880:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.759720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.760376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789017741943880:2050] [node 3] ICP22 created new session: [5:7577789017741943972:2050]
2025-11-28T14:51:34.760423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789017741943972:2050] [node 3] ICS09 handshake done sender: [5:7577789017741943965:2120] self: [5:7577789017741943970:0] peer: [3:7577789017997876479:0] socket: 166 qp: -1
2025-11-28T14:51:34.760462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789017741943972:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:34.760473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398328:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
2025-11-28T14:51:34.760624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.760829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398331:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
2025-11-28T14:51:34.760892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789017741943880:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:34.760960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789016475398328:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.761229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789016475398331:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.761284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789016475398309:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.761987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789016475398306:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.762019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019113319087:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.762074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789019113319087:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.762360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019113319102:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35135
2025-11-28T14:51:34.763877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019113319087:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:34.764506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789019113319085:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.764755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789019113319077:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.764667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789017741943957:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.771774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.764699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789017741943957:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:34.771960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789019113319077:2109] [node 6] ICP22 created new session: [4:7577789019113319104:2109]
2025-11-28T14:51:34.772970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019113319104:2109] [node 6] ICS09 handshake done sender: [4:7577789019113319087:2115] self: [4:7577789019113319086:0] peer: [6:7577789016475398333:0] socket: 168 qp: -1
2025-11-28T14:51:34.773007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019113319104:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.773046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.773170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724207:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.774381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789017741943957:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:34.774584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789017741943942:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.774710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.774766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789017741943942:2109] [node 6] ICP22 created new session: [5:7577789017741943975:2109]
2025-11-28T14:51:34.774835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789017741943975:2109] [node 6] ICS09 handshake done sender: [5:7577789017741943957:2116] self: [5:7577789017741943956:0] peer: [6:7577789016475398334:0] socket: 170 qp: -1
2025-11-28T14:51:34.774870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789017741943975:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.774914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.776729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789017997876473:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.784887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789018180747711:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.789040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789018889724203:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.798714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789016475398316:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.808457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724211:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.808690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789016475398319:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.808929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398335:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.809490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398336:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
2025-11-28T14:51:34.809894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789016475398245:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.809958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789016475398328:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:34.810317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789016475398309:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.810366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:34.810407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789016475398309:2106] [node 4] ICP22 created new session: [6:7577789016475398338:2106]
2025-11-28T14:51:34.810381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724203:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.810450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789016475398338:2106] [node 4] ICS09 handshake done sender: [6:7577789016475398328:2119] self: [6:7577789016475398333:0] peer: [4:7577789019113319086:0] socket: 121 qp: -1
2025-11-28T14:51:34.810485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789016475398338:2106] [node 4] ICS10 traffic start
2025-11-28T14:51:34.810533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.810834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789018889724203:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:34.811048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789018889724195:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.811102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.811158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789018889724195:2109] [node 6] ICP22 created new session: [1:7577789018889724223:2109]
2025-11-28T14:51:34.811203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789018889724223:2109] [node 6] ICS09 handshake done sender: [1:7577789018889724203:2113] self: [1:7577789018889724199:0] peer: [6:7577789016475398337:0] socket: 130 qp: -1
2025-11-28T14:51:34.811233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789018889724223:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.811260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.811247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789016475398331:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:34.811675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789016475398306:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.811710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.811738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789016475398306:2107] [node 5] ICP22 created new session: [6:7577789016475398341:2107]
2025-11-28T14:51:34.811782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789016475398341:2107] [node 5] ICS09 handshake done sender: [6:7577789016475398331:2120] self: [6:7577789016475398334:0] peer: [5:7577789017741943956:0] socket: 173 qp: -1
2025-11-28T14:51:34.811809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789016475398341:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:34.811836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.812133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019113319106:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35135
2025-11-28T14:51:34.813219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789016475398245:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789016475398319:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.813393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398340:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
2025-11-28T14:51:34.813696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789016475398319:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.813854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789016475398245:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.813886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:34.813920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789016475398245:2048] [node 1] ICP22 created new session: [6:7577789016475398343:2048]
2025-11-28T14:51:34.813951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789016475398343:2048] [node 1] ICS09 handshake done sender: [6:7577789016475398319:2115] self: [6:7577789016475398337:0] peer: [1:7577789018889724199:0] socket: 135 qp: -1
2025-11-28T14:51:34.813972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789016475398343:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.814008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:34.823913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789019113319084:2113] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.832627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789018180747712:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.842562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789016475398315:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.849841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789017741943958:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.850029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789017741943942:2109] [node 6] ICP18 handshake (actor [5:7577789017741943958:2117]) is too old
2025-11-28T14:51:34.867155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789018889724201:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.875006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789018180747717:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.875181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789018180747637:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.875518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724201:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.876079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789018889724201:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.876201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789018180747725:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6986
2025-11-28T14:51:34.876327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789018889724130:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.876416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:34.876475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789018889724130:2048] [node 2] ICP22 created new session: [1:7577789018889724227:2048]
2025-11-28T14:51:34.876547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789018889724227:2048] [node 2] ICS09 handshake done sender: [1:7577789018889724201:2111] self: [1:7577789018889724197:0] peer: [2:7577789018180747724:0] socket: 128 qp: -1
2025-11-28T14:51:34.876587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789018889724227:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:34.876630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.876860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789018180747717:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.877147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789018180747637:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.877189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:34.877223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789018180747637:2048] [node 1] ICP22 created new session: [2:7577789018180747726:2048]
2025-11-28T14:51:34.877261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789018180747726:2048] [node 1] ICS09 handshake done sender: [2:7577789018180747717:2117] self: [2:7577789018180747724:0] peer: [1:7577789018889724197:0] socket: 143 qp: -1
2025-11-28T14:51:34.877297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789018180747726:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.877330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:34.879532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789016475398313:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.887948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789018180747719:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.891319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789018180747708:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.899084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724212:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.899345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789018889724130:2048] [node 2] ICP18 handshake (actor [1:7577789018889724212:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.900025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789017741943977:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13898
2025-11-28T14:51:34.903368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789018180747707:2110] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.910759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789016475398325:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.910939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789016475398246:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.911162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789018180747707:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.911765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789018180747707:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.912061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789018180747701:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.912142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:34.912195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789016475398346:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64878
2025-11-28T14:51:34.912200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789018180747701:2109] [node 6] ICP22 created new session: [2:7577789018180747731:2109]
2025-11-28T14:51:34.912255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789018180747731:2109] [node 6] ICS09 handshake done sender: [2:7577789018180747707:2110] self: [2:7577789018180747703:0] peer: [6:7577789016475398345:0] socket: 148 qp: -1
2025-11-28T14:51:34.912287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789018180747731:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:34.912325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:34.912463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789016475398325:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.912638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789016475398246:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.912708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:34.912838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789016475398246:2049] [node 2] ICP22 created new session: [6:7577789016475398347:2049]
2025-11-28T14:51:34.912884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789016475398347:2049] [node 2] ICS09 handshake done sender: [6:7577789016475398325:2117] self: [6:7577789016475398345:0] peer: [2:7577789018180747703:0] socket: 149 qp: -1
2025-11-28T14:51:34.912913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789016475398347:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:34.913038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.913321Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789016475398246:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.913875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019113319107:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35135
2025-11-28T14:51:34.914000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789018180747709:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.922073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789017997876481:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.922298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789017997876403:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.922526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789018180747709:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.923021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789018180747709:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.923176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789017997876497:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55659
2025-11-28T14:51:34.923331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789018180747638:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.923404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:34.923402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789017997876481:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.923474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789018180747638:2049] [node 3] ICP22 created new session: [2:7577789018180747735:2049]
2025-11-28T14:51:34.923571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789017997876403:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.923652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789018180747735:2049] [node 3] ICS09 handshake done sender: [2:7577789018180747709:2112] self: [2:7577789018180747705:0] peer: [3:7577789017997876496:0] socket: 150 qp: -1
2025-11-28T14:51:34.923764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:34.923692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789018180747735:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:34.923738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:34.923826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789017997876403:2049] [node 2] ICP22 created new session: [3:7577789017997876498:2049]
2025-11-28T14:51:34.923932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789017997876498:2049] [node 2] ICS09 handshake done sender: [3:7577789017997876481:2115] self: [3:7577789017997876496:0] peer: [2:7577789018180747705:0] socket: 153 qp: -1
2025-11-28T14:51:34.923963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789017997876498:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:34.924011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.926146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789018889724202:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.934167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789017997876490:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.934393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789017997876402:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.934589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724202:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.935108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789018889724202:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.935315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789018889724131:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.935376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789017997876501:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55659
2025-11-28T14:51:34.935436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:34.935495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789018889724131:2049] [node 3] ICP22 created new session: [1:7577789018889724230:2049]
2025-11-28T14:51:34.935552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789018889724230:2049] [node 3] ICS09 handshake done sender: [1:7577789018889724202:2112] self: [1:7577789018889724198:0] peer: [3:7577789017997876500:0] socket: 120 qp: -1
2025-11-28T14:51:34.935588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789018889724230:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:34.935649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:34.935629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789017997876490:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.936004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789017997876402:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789017997876490:2116] held: no
2025-11-28T14:51:34.936278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789017997876402:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.936327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:34.936385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789017997876402:2048] [node 1] ICP22 created new session: [3:7577789017997876502:2048]
2025-11-28T14:51:34.936484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789017997876502:2048] [node 1] ICS09 handshake done sender: [3:7577789017997876490:2116] self: [3:7577789017997876500:0] peer: [1:7577789018889724198:0] socket: 165 qp: -1
2025-11-28T14:51:34.936517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789017997876502:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.936558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:34.939023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789016475398324:2116] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.946579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019113319102:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.946878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789019113319077:2109] [node 6] ICP18 handshake (actor [4:7577789019113319102:2119]) is too old
2025-11-28T14:51:34.949349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789018889724210:2116] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.957521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019113319106:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.957767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789019113319012:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.957990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789018889724210:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.958418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789018889724210:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.958715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789018889724194:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.958705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019113319109:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35135
2025-11-28T14:51:34.958783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:34.958827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789018889724194:2107] [node 4] ICP22 created new session: [1:7577789018889724233:2107]
2025-11-28T14:51:34.958906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789018889724233:2107] [node 4] ICS09 handshake done sender: [1:7577789018889724210:2116] self: [1:7577789018889724209:0] peer: [4:7577789019113319108:0] socket: 179 qp: -1
2025-11-28T14:51:34.958943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789018889724233:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:34.958987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.959164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019113319106:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:34.959385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789019113319012:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.959448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:34.959562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789019113319012:2048] [node 1] ICP22 created new session: [4:7577789019113319110:2048]
2025-11-28T14:51:34.959627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019113319110:2048] [node 1] ICS09 handshake done sender: [4:7577789019113319106:2120] self: [4:7577789019113319108:0] peer: [1:7577789018889724209:0] socket: 180 qp: -1
2025-11-28T14:51:34.959680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019113319110:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:34.959729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:34.960104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789019113319012:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:34.962111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789018180747710:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.969942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789017741943977:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.970154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789017741943879:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.970378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789018180747710:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.970882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789018180747710:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.971126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789018180747700:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.971179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789017741943979:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13898
2025-11-28T14:51:34.971209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:34.971262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789018180747700:2108] [node 5] ICP22 created new session: [2:7577789018180747738:2108]
2025-11-28T14:51:34.971378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789018180747738:2108] [node 5] ICS09 handshake done sender: [2:7577789018180747710:2113] self: [2:7577789018180747706:0] peer: [5:7577789017741943978:0] socket: 147 qp: -1
2025-11-28T14:51:34.971417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789018180747738:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:34.971430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789017741943977:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.971497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:34.971804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789017741943879:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.971857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:34.971904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789017741943879:2049] [node 2] ICP22 created new session: [5:7577789017741943980:2049]
2025-11-28T14:51:34.971972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789017741943980:2049] [node 2] ICS09 handshake done sender: [5:7577789017741943977:2122] self: [5:7577789017741943978:0] peer: [2:7577789018180747706:0] socket: 151 qp: -1
2025-11-28T14:51:34.972033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789017741943980:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:34.972073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.973757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789017741943879:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:34.973924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789018180747720:2119] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:34.975333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789019113319107:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.975505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789019113319013:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:34.975729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789018180747720:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:34.976173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789018180747720:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:34.976357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789019113319113:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35135
2025-11-28T14:51:34.976430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789018180747702:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:34.976531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:34.976587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789018180747702:2107] [node 4] ICP22 created new session: [2:7577789018180747741:2107]
2025-11-28T14:51:34.976601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789019113319107:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:34.976644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789018180747741:2107] [node 4] ICS09 handshake done sender: [2:7577789018180747720:2119] self: [2:7577789018180747718:0] peer: [4:7577789019113319112:0] socket: 141 qp: -1
2025-11-28T14:51:34.976689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789018180747741:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:34.976731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789019113319013:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:34.976778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:34.977124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:34.977270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789019113319013:2049] [node 2] ICP22 created new session: [4:7577789019113319114:2049]
2025-11-28T14:51:34.977365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789019113319114:2049] [node 2] ICS09 handshake done sender: [4:7577789019113319107:2121] self: [4:7577789019113319112:0] peer: [2:7577789018180747718:0] socket: 146 qp: -1
2025-11-28T14:51:34.977407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789019113319114:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:34.977449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:34.994005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:34.994227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789018180747726:2048] [node 1] ICS07 socket disconnect 143 reason# EndOfStream
2025-11-28T14:51:34.994267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789018180747726:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.994382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:34.994433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789018180747726:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:34.994970Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:34.995212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789017997876502:2048] [node 1] ICS01 socket: 165 reason# EndOfStream
2025-11-28T14:51:34.995259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789017997876402:2048] [node 1] ICP30 unregister session Session# [3:7577789017997876502:2048] VirtualId# [3:7577789017997876500:0]
2025-11-28T14:51:34.995294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789017997876502:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.995375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:34.995428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:34.997334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:34.997496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789018180747637:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:34.997515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789019113319110:2048] [node 1] ICS07 socket disconnect 180 reason# EndOfStream
2025-11-28T14:51:34.997538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789019113319110:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.997598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:34.997537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789018180747726:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:34.997638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789019113319110:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:34.998531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789018180747637:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:34.998556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:34.998661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789019113319012:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:34.998729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789019113319110:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:34.998715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789018180747726:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:34.998823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789017741943966:2048] [node 1] ICS01 socket: 125 reason# EndOfStream
2025-11-28T14:51:34.998821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789018180747637:2048] [node 1] ICP30 unregister session Session# [2:7577789018180747726:2048] VirtualId# [2:7577789018180747724:0]
2025-11-28T14:51:34.998811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789019113319012:2048] [node 1] ICP30 unregister session Session# [4:7577789019113319110:2048] VirtualId# [4:7577789019113319108:0]
2025-11-28T14:51:34.998882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789017741943878:2048] [node 1] ICP30 unregister session Session# [5:7577789017741943966:2048] VirtualId# [5:7577789017741943960:0]
2025-11-28T14:51:34.998868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:34.998916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789017741943966:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.998908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:34.998901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789019113319012:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.998914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.998944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789018180747637:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.998958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16627) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:34.998996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:34.999055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:34.999556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:34.999809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789016475398343:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-11-28T14:51:34.999863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789016475398245:2048] [node 1] ICP30 unregister session Session# [6:7577789016475398343:2048] VirtualId# [6:7577789016475398337:0]
2025-11-28T14:51:34.999910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789016475398343:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:34.999999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:35.000054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:35.006170Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:35.006418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789017997876498:2049] [node 2] ICS07 socket disconnect 153 reason# EndOfStream
2025-11-28T14:51:35.006465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789017997876498:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.006563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:35.006610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789017997876498:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:35.007285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789017997876403:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.007541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789017997876498:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.007606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789017997876403:2049] [node 2] ICP30 unregister session Session# [3:7577789017997876498:2049] VirtualId# [3:7577789017997876496:0]
2025-11-28T14:51:35.007653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:35.007688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789017997876403:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.007702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.008604Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:35.008793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789019113319114:2049] [node 2] ICS07 socket disconnect 146 reason# EndOfStream
2025-11-28T14:51:35.008822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789019113319114:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.008913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:35.008939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789019113319012:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:35.008969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789019113319114:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:35.009339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:35.009524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789017741943980:2049] [node 2] ICS07 socket disconnect 151 reason# EndOfStream
2025-11-28T14:51:35.009566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789017741943980:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.009564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789019113319013:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.009643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:35.009694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789017741943980:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:35.009685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789019113319114:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.009731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789019113319013:2049] [node 2] ICP30 unregister session Session# [4:7577789019113319114:2049] VirtualId# [4:7577789019113319112:0]
2025-11-28T14:51:35.009774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:35.009802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789019113319013:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.009817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.009958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:35.010227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789016475398347:2049] [node 2] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-28T14:51:35.010265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789016475398347:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.010338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:35.010325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789017741943879:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.010391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789016475398347:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:35.010370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789017741943980:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.010441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789017741943879:2049] [node 2] ICP30 unregister session Session# [5:7577789017741943980:2049] VirtualId# [5:7577789017741943978:0]
2025-11-28T14:51:35.010499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:35.010522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789017741943879:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.010540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.011113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789016475398246:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.011162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789016475398347:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.011220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789016475398246:2049] [node 2] ICP30 unregister session Session# [6:7577789016475398347:2049] VirtualId# [6:7577789016475398345:0]
2025-11-28T14:51:35.011269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:35.011312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789016475398246:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.011352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.019169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789019113319013:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:35.020193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789017741943879:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:35.021234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789016475398246:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:35.042840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:35.043067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789019113319100:2050] [node 3] ICS07 socket disconnect 136 reason# EndOfStream
2025-11-28T14:51:35.043113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789019113319100:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.043217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:35.043268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789019113319100:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:35.043523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:35.043736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789017741943972:2050] [node 3] ICS07 socket disconnect 166 reason# EndOfStream
2025-11-28T14:51:35.043777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789017741943972:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.043852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:35.043896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789017741943972:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:35.044045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789019113319014:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27609) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.044136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:35.044173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789019113319100:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.044214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789019113319014:2050] [node 3] ICP30 unregister session Session# [4:7577789019113319100:2050] VirtualId# [4:7577789019113319094:0]
2025-11-28T14:51:35.044265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:35.044298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789019113319014:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.044323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.044410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789016475398329:2050] [node 3] ICS07 socket disconnect 131 reason# EndOfStream
2025-11-28T14:51:35.044444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789016475398329:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.044528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:35.044576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789017741943880:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27609) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.044583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789016475398329:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:35.044863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789017741943972:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.044896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789017741943880:2050] [node 3] ICP30 unregister session Session# [5:7577789017741943972:2050] VirtualId# [5:7577789017741943970:0]
2025-11-28T14:51:35.044991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:35.045025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789017741943880:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.045037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.045571Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789016475398247:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27609) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.045666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789016475398329:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.045707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789016475398247:2050] [node 3] ICP30 unregister session Session# [6:7577789016475398329:2050] VirtualId# [6:7577789016475398326:0]
2025-11-28T14:51:35.045748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:35.045776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789016475398247:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.045789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.055028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789017741943880:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:35.056061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789016475398247:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:35.068298Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:35.068529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789017741943964:2108] [node 4] ICS01 socket: 123 reason# EndOfStream
2025-11-28T14:51:35.068580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789017741943941:2108] [node 4] ICP30 unregister session Session# [5:7577789017741943964:2108] VirtualId# [5:7577789017741943959:0]
2025-11-28T14:51:35.068610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789017741943964:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.068706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:35.068771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:35.069285Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:35.069518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789016475398338:2106] [node 4] ICS07 socket disconnect 121 reason# EndOfStream
2025-11-28T14:51:35.069555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789016475398338:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.069669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:35.069728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789016475398338:2106] [node 4] ICS15 start handshake
2025-11-28T14:51:35.070463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789016475398309:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16265) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.070503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789016475398338:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.070619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789016475398309:2106] [node 4] ICP30 unregister session Session# [6:7577789016475398338:2106] VirtualId# [6:7577789016475398333:0]
2025-11-28T14:51:35.070701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:35.070735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789016475398309:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16265) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.070758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16265) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.080608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789016475398309:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:51:35.111689Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:35.111905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789016475398341:2107] [node 5] ICS07 socket disconnect 173 reason# EndOfStream
2025-11-28T14:51:35.111938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789016475398341:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.112041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:35.112123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789016475398341:2107] [node 5] ICS15 start handshake
2025-11-28T14:51:35.112921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789016475398306:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18998) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.112959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789016475398341:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.113021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789016475398306:2107] [node 5] ICP30 unregister session Session# [6:7577789016475398341:2107] VirtualId# [6:7577789016475398334:0]
2025-11-28T14:51:35.113098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:35.113123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789016475398306:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18998) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.113134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18998) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:35.178814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789023134856469:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:35.178855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789023134856468:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:35.197616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789022254885394:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:35.197658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789022254885395:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:35.220261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789022302289495:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:35.220402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789022302289494:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:35.241865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789020713406238:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:35.241916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789020713406240:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:35.242190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789020713406239:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:35.264412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789023001862143:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:35.264538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789023001862144:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:35.264543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789023001862145:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:35.287318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789023074983446:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:35.287321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789023074983445:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:35.287367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789023074983447:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:35.342838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789023074983507:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:35.343826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789023134856532:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:35.343874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789023134856533:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:35.344011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789023134856534:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:35.345612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789022254885457:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:35.420792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789023074983507:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(77586324ns) {Configure:104(77580635ns) {ProcessConfigured:135(40077101ns) {StartInitialHandshake:157(40072721ns) {DropHandshakes:470(16525453ns) {DropIncomingHandshake:442(856ns) DropOutgoingHandshake:457(1913ns)} GenerateSessionVirtualId:988(2517ns) PrepareNewSessionHandshake:477(7801ns) {SwitchToState:179(4205ns)}}}}}
2025-11-28T14:51:35.359616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789022254885458:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:35.396452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789020713406301:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.436424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789022302289557:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:35.380332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789022254885459:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:35.448463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789023001862207:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:35.437729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789022302289558:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:35.458162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789022254885460:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19491
2025-11-28T14:51:35.446580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789020713406302:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:35.465355Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789023134856534:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(121105926ns) {Configure:104(121103496ns) {ProcessConfigured:135(20757548ns) {StartInitialHandshake:157(20755801ns) {DropHandshakes:470(3359ns) {DropIncomingHandshake:442(920ns) DropOutgoingHandshake:457(661ns)} GenerateSessionVirtualId:988(1805ns) PrepareNewSessionHandshake:477(14448ns) {SwitchToState:179(503ns)}}}}}
2025-11-28T14:51:35.465374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789023134856533:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(121243502ns) {Configure:104(121239618ns) {ProcessConfigured:135(27302162ns) {StartInitialHandshake:157(27298663ns) {DropHandshakes:470(3618ns) {DropIncomingHandshake:442(905ns) DropOutgoingHandshake:457(739ns)} GenerateSessionVirtualId:988(999ns) PrepareNewSessionHandshake:477(3377ns) {SwitchToState:179(1408ns)}}}}}
2025-11-28T14:51:35.467787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789023134856469:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(101949215ns) {Configure:104(101940123ns) {ProcessConfigured:135(40466615ns) {StartInitialHandshake:157(40463821ns) {DropHandshakes:470(3156ns) {DropIncomingHandshake:442(910ns) DropOutgoingHandshake:457(386ns)} GenerateSessionVirtualId:988(1596ns) PrepareNewSessionHandshake:477(3083ns) {SwitchToState:179(1189ns)}}}}}
2025-11-28T14:51:35.467331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789022302289559:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:35.469423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789023074983512:2109] [node 4] ICP01 ready to work
2025-11-28T14:51:35.467740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789023001862209:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:35.471936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789023134856532:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(123417013ns) {Configure:104(123412878ns) {ProcessConfigured:135(25911354ns) {StartInitialHandshake:157(18964815ns) {DropHandshakes:470(7065ns) {DropIncomingHandshake:442(1239ns) DropOutgoingHandshake:457(701ns)} GenerateSessionVirtualId:988(2094ns) PrepareNewSessionHandshake:477(1816ns) {SwitchToState:179(581ns)}}}}}
2025-11-28T14:51:35.475807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789022254885394:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(127345487ns) {Configure:104(127337557ns) {ProcessConfigured:135(3524969ns) {StartInitialHandshake:157(3522352ns) {DropHandshakes:470(3501ns) {DropIncomingHandshake:442(914ns) DropOutgoingHandshake:457(517ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(2496ns) {SwitchToState:179(1002ns)}}}}}
2025-11-28T14:51:35.477800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789022254885395:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(129321988ns) {Configure:104(129309694ns) {ProcessConfigured:135(640000ns) {StartInitialHandshake:157(637706ns) {DropHandshakes:470(3034ns) {DropIncomingHandshake:442(870ns) DropOutgoingHandshake:457(424ns)} GenerateSessionVirtualId:988(393ns) PrepareNewSessionHandshake:477(1721ns) {SwitchToState:179(745ns)}}}}}
2025-11-28T14:51:35.478555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789022302289494:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(129013792ns) {Configure:104(129010151ns) {ProcessConfigured:135(575511ns) {StartInitialHandshake:157(573716ns) {DropHandshakes:470(2387ns) {DropIncomingHandshake:442(398ns) DropOutgoingHandshake:457(564ns)} GenerateSessionVirtualId:988(727ns) PrepareNewSessionHandshake:477(1217ns) {SwitchToState:179(539ns)}}}}}
2025-11-28T14:51:35.477859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789022254885457:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(129046757ns) {Configure:104(129043685ns) {ProcessConfigured:135(845359ns) {StartInitialHandshake:157(843332ns) {DropHandshakes:470(1274ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(372ns) PrepareNewSessionHandshake:477(1820ns) {SwitchToState:179(733ns)}}}}}
2025-11-28T14:51:35.478909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789022302289495:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(130622960ns) {Configure:104(130619057ns) {ProcessConfigured:135(1570347ns) {StartInitialHandshake:157(1567516ns) {DropHandshakes:470(1413ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(376ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:477(2720ns) {SwitchToState:179(1460ns)}}}}}
2025-11-28T14:51:35.479810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789023001862145:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(126398640ns) {Configure:104(126394937ns) {ProcessConfigured:135(1159149ns) {StartInitialHandshake:157(1153512ns) {DropHandshakes:470(3157ns) {DropIncomingHandshake:442(411ns) DropOutgoingHandshake:457(1948ns)} GenerateSessionVirtualId:988(822ns) PrepareNewSessionHandshake:477(1422ns) {SwitchToState:179(619ns)}}}}}
2025-11-28T14:51:35.481527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789022254885460:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.483064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789022302289557:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28110572ns) {Configure:104(28101149ns) {ProcessConfigured:135(1507737ns) {StartInitialHandshake:157(1506155ns) {DropHandshakes:470(1047ns) {DropIncomingHandshake:442(267ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(298ns) PrepareNewSessionHandshake:477(1458ns) {SwitchToState:179(531ns)}}}}}
2025-11-28T14:51:35.481939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789023074983447:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(46693354ns) {Configure:104(46682089ns) {ProcessConfigured:135(1298850ns) {StartInitialHandshake:157(1297509ns) {DropHandshakes:470(1782ns) {DropIncomingHandshake:442(489ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(305ns) PrepareNewSessionHandshake:477(1271ns) {SwitchToState:179(593ns)}}}}}
2025-11-28T14:51:35.482720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789022254885394:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.483572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789023074983446:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41938340ns) {Configure:104(41928963ns) {ProcessConfigured:135(822962ns) {StartInitialHandshake:157(821200ns) {DropHandshakes:470(1460ns) {DropIncomingHandshake:442(616ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(1226ns) PrepareNewSessionHandshake:477(1642ns) {SwitchToState:179(1032ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.483719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789023001862143:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(100064602ns) {Configure:104(100057607ns) {ProcessConfigured:135(362100ns) {StartInitialHandshake:157(360870ns) {DropHandshakes:470(722ns) {DropIncomingHandshake:442(150ns) DropOutgoingHandshake:457(133ns)} GenerateSessionVirtualId:988(292ns) PrepareNewSessionHandshake:477(1765ns) {SwitchToState:179(969ns)}}}}}
2025-11-28T14:51:35.485133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789022254885459:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34643027ns) {Configure:104(34636602ns) {ProcessConfigured:135(1477222ns) {StartInitialHandshake:157(1475500ns) {DropHandshakes:470(4101ns) {DropIncomingHandshake:442(393ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(1024ns) {SwitchToState:179(128ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.485353Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789020713406240:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(121988923ns) {Configure:104(121980267ns) {ProcessConfigured:135(444693ns) {StartInitialHandshake:157(443131ns) {DropHandshakes:470(1162ns) {DropIncomingHandshake:442(294ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:477(1874ns) {SwitchToState:179(889ns)}}}}}
2025-11-28T14:51:35.485875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789023001862144:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(131635814ns) {Configure:104(131631236ns) {ProcessConfigured:135(2634215ns) {StartInitialHandshake:157(2632912ns) {DropHandshakes:470(1081ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:477(1206ns) {SwitchToState:179(100ns)}}}}}
2025-11-28T14:51:35.487985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789023074983445:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(112398811ns) {Configure:104(112391009ns) {ProcessConfigured:135(1376922ns) {StartInitialHandshake:157(1375592ns) {DropHandshakes:470(929ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(154ns)} GenerateSessionVirtualId:988(920ns) PrepareNewSessionHandshake:477(193507ns) {SwitchToState:179(2404ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.490489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789022302289558:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36896022ns) {Configure:104(36887511ns) {ProcessConfigured:135(2475136ns) {StartInitialHandshake:157(2167724ns) {DropHandshakes:470(918ns) {DropIncomingHandshake:442(249ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(718ns) PrepareNewSessionHandshake:477(1547ns) {SwitchToState:179(600ns)}}}}}
2025-11-28T14:51:35.485668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856536:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.487281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789022254885475:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19491
2025-11-28T14:51:35.485720Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789023134856536:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:35.490674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789022302289560:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38944
2025-11-28T14:51:35.488810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789020713406301:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(46063229ns) {Configure:104(46056878ns) {ProcessConfigured:135(1925711ns) {StartInitialHandshake:157(1132362ns) {DropHandshakes:470(1461ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(181ns)} GenerateSessionVirtualId:988(884ns) PrepareNewSessionHandshake:477(1157ns) {SwitchToState:179(500ns)}}}}}
2025-11-28T14:51:35.492750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789023134856536:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:35.489553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789022254885462:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.485998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789023001862208:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-11-28T14:51:35.489688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789020713406302:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16198573ns) {Configure:104(16191979ns) {ProcessConfigured:135(2276287ns) {StartInitialHandshake:157(2274546ns) {DropHandshakes:470(1305ns) {DropIncomingHandshake:442(491ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(387ns) PrepareNewSessionHandshake:477(1474ns) {SwitchToState:179(605ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.499744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789022302289560:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.486193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789023001862210:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-11-28T14:51:35.493189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789023134856468:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.489577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789022254885462:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:51:35.490403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789020713406238:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(135484093ns) {Configure:104(135481750ns) {ProcessConfigured:135(1664290ns) {StartInitialHandshake:157(1662914ns) {DropHandshakes:470(1055ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:477(1321ns) {SwitchToState:179(549ns)}}}}}
::ffff:127.1.0.0
2025-11-28T14:51:35.497259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789023134856548:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64554
2025-11-28T14:51:35.491558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789022254885462:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:35.503088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789022302289494:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.486696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789023001862210:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.497454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789020713406239:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(79506722ns) {Configure:104(79497078ns) {ProcessConfigured:135(2261187ns) {StartInitialHandshake:157(2259672ns) {DropHandshakes:470(768ns) {DropIncomingHandshake:442(436ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(918ns) {SwitchToState:179(154ns)}}}}}
2025-11-28T14:51:35.487109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789023001862144:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.505809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789022302289559:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23813148ns) {Configure:104(23807021ns) {ProcessConfigured:135(5026057ns) {StartInitialHandshake:157(5024159ns) {DropHandshakes:470(3118ns) {DropIncomingHandshake:442(869ns) DropOutgoingHandshake:457(361ns)} GenerateSessionVirtualId:988(719ns) PrepareNewSessionHandshake:477(1316ns) {SwitchToState:179(464ns)}}}}}
2025-11-28T14:51:35.506811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789022254885458:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.497467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.506875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:35.500552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789023134856468:2048] [node 2] ICP22 created new session: [1:7577789023134856552:2048]
2025-11-28T14:51:35.514020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789023074983512:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36021248ns) {Configure:104(36014332ns) {ProcessConfigured:135(6887228ns) {StartInitialHandshake:157(6880151ns) {DropHandshakes:470(3597ns) {DropIncomingHandshake:442(799ns) DropOutgoingHandshake:457(904ns)} GenerateSessionVirtualId:988(858ns) PrepareNewSessionHandshake:477(3181ns) {SwitchToState:179(1786ns)}}}}}
2025-11-28T14:51:35.490789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789023001862208:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:51:35.495473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789023001862223:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.501677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789023134856552:2048] [node 2] ICS09 handshake done sender: [1:7577789023134856536:2111] self: [1:7577789023134856535:0] peer: [2:7577789022254885474:0] socket: 120 qp: -1
2025-11-28T14:51:35.511987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789022254885458:2108] [node 5] ICP22 created new session: [2:7577789022254885477:2108]
2025-11-28T14:51:35.502682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789023001862207:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23111144ns) {Configure:104(22718647ns) {ProcessConfigured:135(1372257ns) {StartInitialHandshake:157(1369324ns) {DropHandshakes:470(2275ns) {DropIncomingHandshake:442(601ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(904ns) PrepareNewSessionHandshake:477(1785ns) {SwitchToState:179(690ns)}}}}}
2025-11-28T14:51:35.540755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789020713406313:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53355
2025-11-28T14:51:35.510632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789023001862210:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.514817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789022254885477:2108] [node 5] ICS09 handshake done sender: [2:7577789022254885462:2111] self: [2:7577789022254885461:0] peer: [5:7577789023001862220:0] socket: 125 qp: -1
2025-11-28T14:51:35.542609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789022302289573:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38944
2025-11-28T14:51:35.513983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789023001862216:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-11-28T14:51:35.502560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856548:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.545607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789020713406313:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.503700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789023134856552:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:35.514924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789023001862209:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30283648ns) {Configure:104(30275151ns) {ProcessConfigured:135(2635512ns) {StartInitialHandshake:157(2631468ns) {DropHandshakes:470(1527ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:477(3010ns) {SwitchToState:179(1181ns)}}}}}
2025-11-28T14:51:35.546809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789020713406314:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53355
2025-11-28T14:51:35.503750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:35.520459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789023001862144:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.546888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789020713406239:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.524475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:35.507504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856541:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.547662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789020713406316:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53355
2025-11-28T14:51:35.507554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789023134856541:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:35.524535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789023001862144:2049] [node 2] ICP22 created new session: [5:7577789023001862227:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.509208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789023134856554:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64554
2025-11-28T14:51:35.546819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789022302289576:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38944
2025-11-28T14:51:35.529211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789023001862227:2049] [node 2] ICS09 handshake done sender: [5:7577789023001862210:2112] self: [5:7577789023001862220:0] peer: [2:7577789022254885461:0] socket: 128 qp: -1
2025-11-28T14:51:35.522617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789022254885477:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.548400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789020713406314:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.533158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:35.511979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789023134856551:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64554
2025-11-28T14:51:35.547656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789022302289577:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38944
2025-11-28T14:51:35.533117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789023001862227:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:35.521067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789023134856541:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:35.548415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789022302289576:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.533154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:35.542738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789023134856555:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64554
2025-11-28T14:51:35.548756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789022302289578:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38944
2025-11-28T14:51:35.545502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856554:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.546244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789023134856469:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.550147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789020713406238:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.541301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789022254885475:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.550787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983523:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
2025-11-28T14:51:35.551032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789023001862144:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(32721931ns) {HandleHandshakeStatus:329(32719453ns) {DropIncomingHandshake:442(1504ns) DropOutgoingHandshake:457(4392ns) SwitchToState:179(3135ns) ProcessPendingSessionEvents:495(3662164ns) {ForwardSessionEventToSession:612(2338927ns) ForwardSessionEventToSession:612(3446ns) ForwardSessionEventToSession:612(118845ns) ForwardSessionEventToSession:612(2656ns) ForwardSessionEventToSession:612(2248ns) ForwardSessionEventToSession:612(140867ns) ForwardSessionEventToSession:612(2950ns) ForwardSessionEventToSession:612(2201ns) ForwardSessionEventToSession:612(1035174ns) ForwardSessionEventToSession:612(2275ns)}}}
2025-11-28T14:51:35.546292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:35.545351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789022254885478:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19491
2025-11-28T14:51:35.546334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789023134856469:2049] [node 3] ICP22 created new session: [1:7577789023134856558:2049]
2025-11-28T14:51:35.553491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789022302289560:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.547895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789023134856558:2049] [node 3] ICS09 handshake done sender: [1:7577789023134856541:2112] self: [1:7577789023134856538:0] peer: [3:7577789022302289575:0] socket: 127 qp: -1
2025-11-28T14:51:35.547951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789023134856558:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:35.554785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789022302289494:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.546845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789022254885479:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19491
2025-11-28T14:51:35.555179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789020713406320:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53355
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.555088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983524:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
2025-11-28T14:51:35.548000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:35.547263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789022254885481:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19491
2025-11-28T14:51:35.555182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:35.555229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789022302289494:2048] [node 1] ICP22 created new session: [3:7577789022302289580:2048]
2025-11-28T14:51:35.547684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789022254885460:2110] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:35.548247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789022254885469:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.555263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789022302289580:2048] [node 1] ICS09 handshake done sender: [3:7577789022302289560:2110] self: [3:7577789022302289575:0] peer: [1:7577789023134856538:0] socket: 129 qp: -1
2025-11-28T14:51:35.548271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789022254885469:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:35.555307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789022302289580:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:35.549595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789022254885394:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.555362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:35.550823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:35.550872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789022254885394:2048] [node 1] ICP22 created new session: [2:7577789022254885484:2048]
2025-11-28T14:51:35.555774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789020713406322:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53355
2025-11-28T14:51:35.551019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789022254885469:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:35.556217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789020713406320:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.548117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789023134856468:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(51394842ns) {HandleHandshakeStatus:329(51391663ns) {DropIncomingHandshake:442(771ns) DropOutgoingHandshake:457(752ns) SwitchToState:179(592ns) ProcessPendingSessionEvents:495(31658608ns) {ForwardSessionEventToSession:612(20136750ns) ForwardSessionEventToSession:612(8170ns) ForwardSessionEventToSession:612(3303ns) ForwardSessionEventToSession:612(7650764ns) ForwardSessionEventToSession:612(414079ns) ForwardSessionEventToSession:612(391090ns) ForwardSessionEventToSession:612(6210ns) ForwardSessionEventToSession:612(3696ns) ForwardSessionEventToSession:612(3013767ns) ForwardSessionEventToSession:612(4286ns)}}}
2025-11-28T14:51:35.551047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789022254885458:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(41207817ns) {HandleHandshakeStatus:329(41203916ns) {DropIncomingHandshake:442(868ns) DropOutgoingHandshake:457(1111ns) SwitchToState:179(910ns) ProcessPendingSessionEvents:495(3409276ns) {ForwardSessionEventToSession:612(1006937ns) ForwardSessionEventToSession:612(21756ns) ForwardSessionEventToSession:612(395643ns) ForwardSessionEventToSession:612(2302ns) ForwardSessionEventToSession:612(422137ns) ForwardSessionEventToSession:612(3666ns) ForwardSessionEventToSession:612(311006ns) ForwardSessionEventToSession:612(14557ns) ForwardSessionEventToSession:612(2813ns) ForwardSessionEventToSession:612(7857ns)}}}
2025-11-28T14:51:35.548270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789023134856557:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64554
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.553772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789022254885484:2048] [node 1] ICS09 handshake done sender: [2:7577789022254885460:2110] self: [2:7577789022254885474:0] peer: [1:7577789023134856535:0] socket: 123 qp: -1
2025-11-28T14:51:35.551281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856546:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.557004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789020713406313:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:35.551311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789023134856546:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:35.553638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789023134856468:2048] [node 2] ICP18 handshake (actor [1:7577789023134856548:2116]) is too old
2025-11-28T14:51:35.557792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789023074983524:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.555603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789023134856546:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:35.558162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789020713406239:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.558042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789023074983445:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.558233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:35.557127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789023134856532:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.558260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789020713406323:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53355
No RDMA context found for GID: 2025-11-28T14:51:35.557442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
::ffff:127.1.0.0
2025-11-28T14:51:35.553808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789022254885484:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:35.557630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789023134856532:2107] [node 4] ICP22 created new session: [1:7577789023134856563:2107]
2025-11-28T14:51:35.554107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789022254885458:2108] [node 5] ICP18 handshake (actor [2:7577789022254885475:2116]) is too old
2025-11-28T14:51:35.558280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789020713406239:2049] [node 2] ICP22 created new session: [4:7577789020713406324:2049]
2025-11-28T14:51:35.558248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856544:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.554182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789022254885457:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.558325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789020713406324:2049] [node 2] ICS09 handshake done sender: [4:7577789020713406313:2115] self: [4:7577789020713406319:0] peer: [2:7577789022254885468:0] socket: 136 qp: -1
2025-11-28T14:51:35.554306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:35.558505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983525:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
2025-11-28T14:51:35.554375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789022254885457:2107] [node 4] ICP22 created new session: [2:7577789022254885485:2107]
2025-11-28T14:51:35.558266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789023134856544:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:35.559674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789023074983525:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.558495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789020713406324:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:35.554491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:35.560713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789023074983446:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.559164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:35.558524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789023134856563:2107] [node 4] ICS09 handshake done sender: [1:7577789023134856546:2115] self: [1:7577789023134856542:0] peer: [4:7577789020713406321:0] socket: 145 qp: -1
2025-11-28T14:51:35.554750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789022254885485:2107] [node 4] ICS09 handshake done sender: [2:7577789022254885469:2114] self: [2:7577789022254885468:0] peer: [4:7577789020713406319:0] socket: 133 qp: -1
2025-11-28T14:51:35.558556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789023134856563:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:35.554788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789022254885485:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:35.559239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:35.554947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:35.561099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789020713406314:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:35.561342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789022254885473:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.561369Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789022254885473:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:35.561315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789023134856544:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:35.561495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789023134856534:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.562292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789020713406238:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.561551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.562259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789023134856534:2110] [node 6] ICP22 created new session: [1:7577789023134856566:2110]
2025-11-28T14:51:35.562403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789023134856566:2110] [node 6] ICS09 handshake done sender: [1:7577789023134856544:2113] self: [1:7577789023134856540:0] peer: [6:7577789023074983526:0] socket: 173 qp: -1
2025-11-28T14:51:35.562597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789023074983445:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789023074983524:2116] held: no
2025-11-28T14:51:35.562726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983528:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
2025-11-28T14:51:35.562430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789023134856566:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:35.562658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:35.562484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:35.563355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789020713406238:2048] [node 1] ICP22 created new session: [4:7577789020713406327:2048]
2025-11-28T14:51:35.563559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789022254885473:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:35.563460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789020713406327:2048] [node 1] ICS09 handshake done sender: [4:7577789020713406314:2116] self: [4:7577789020713406321:0] peer: [1:7577789023134856542:0] socket: 155 qp: -1
2025-11-28T14:51:35.563552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789020713406327:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:35.563602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.563979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789022254885459:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.563979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789023074983524:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:35.564080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:35.564148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789023074983445:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.564346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983529:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
2025-11-28T14:51:35.564123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789022254885459:2109] [node 6] ICP22 created new session: [2:7577789022254885490:2109]
2025-11-28T14:51:35.564575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:35.564571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789022254885490:2109] [node 6] ICS09 handshake done sender: [2:7577789022254885473:2115] self: [2:7577789022254885472:0] peer: [6:7577789023074983527:0] socket: 138 qp: -1
2025-11-28T14:51:35.564681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789022254885490:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:35.564888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789023074983445:2048] [node 1] ICP22 created new session: [6:7577789023074983531:2048]
2025-11-28T14:51:35.565040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789023074983525:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:35.564912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:35.565095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789023074983531:2048] [node 1] ICS09 handshake done sender: [6:7577789023074983524:2116] self: [6:7577789023074983526:0] peer: [1:7577789023134856540:0] socket: 174 qp: -1
No RDMA context found for GID: 2025-11-28T14:51:35.565124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789023074983531:2048] [node 1] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:51:35.565308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:35.565427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789023074983446:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.566370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789022254885491:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19491
2025-11-28T14:51:35.565458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:35.566119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789023074983446:2049] [node 2] ICP22 created new session: [6:7577789023074983533:2049]
2025-11-28T14:51:35.566583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789023074983533:2049] [node 2] ICS09 handshake done sender: [6:7577789023074983525:2117] self: [6:7577789023074983527:0] peer: [2:7577789022254885472:0] socket: 177 qp: -1
2025-11-28T14:51:35.566720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789023074983533:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:35.566756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.569421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789022254885459:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:35.569410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789020713406329:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53355
2025-11-28T14:51:35.571759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789023001862214:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.571775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789020713406329:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.579414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856551:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.581378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789023134856545:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.589924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789023001862216:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.590160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789023001862143:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.590430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856545:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.590917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789023134856545:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.591179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789023134856533:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.591254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:35.591321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789023134856533:2108] [node 5] ICP22 created new session: [1:7577789023134856570:2108]
2025-11-28T14:51:35.591375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789023134856570:2108] [node 5] ICS09 handshake done sender: [1:7577789023134856545:2114] self: [1:7577789023134856539:0] peer: [5:7577789023001862231:0] socket: 131 qp: -1
2025-11-28T14:51:35.591402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789023134856570:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:35.591260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789023001862232:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-11-28T14:51:35.591460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:35.591516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789023001862216:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:35.591751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789023001862143:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.591845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:35.591999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789023001862143:2048] [node 1] ICP22 created new session: [5:7577789023001862233:2048]
2025-11-28T14:51:35.592044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789023001862233:2048] [node 1] ICS09 handshake done sender: [5:7577789023001862216:2115] self: [5:7577789023001862231:0] peer: [1:7577789023134856539:0] socket: 134 qp: -1
2025-11-28T14:51:35.592080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789023001862233:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:35.592155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:35.592485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789023001862143:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.593643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983536:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
2025-11-28T14:51:35.594907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789022302289562:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.602544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789023074983536:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.602714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789023074983507:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.602865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856555:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.602933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789023001862226:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.603025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789023134856469:2049] [node 3] ICP18 handshake (actor [1:7577789023134856555:2119]) is too old
2025-11-28T14:51:35.602972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789023001862226:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.604359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983538:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.604580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983539:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
2025-11-28T14:51:35.604459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789023001862226:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:35.604704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789023001862209:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.604769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:35.604871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789023001862209:2109] [node 6] ICP22 created new session: [5:7577789023001862237:2109]
2025-11-28T14:51:35.604922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789023001862237:2109] [node 6] ICS09 handshake done sender: [5:7577789023001862226:2119] self: [5:7577789023001862225:0] peer: [6:7577789023074983537:0] socket: 139 qp: -1
2025-11-28T14:51:35.604944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789023074983536:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:35.604987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789023001862237:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:35.605083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:35.605072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789023074983507:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.605438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:35.606125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789023074983507:2106] [node 5] ICP22 created new session: [6:7577789023074983540:2106]
2025-11-28T14:51:35.606339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789023074983540:2106] [node 5] ICS09 handshake done sender: [6:7577789023074983536:2120] self: [6:7577789023074983537:0] peer: [5:7577789023001862225:0] socket: 141 qp: -1
2025-11-28T14:51:35.606378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789023074983540:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:35.606420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:35.606721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789023074983507:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:35.607878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789023001862212:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.608817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789022302289573:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.633521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789022302289564:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.642402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789022254885479:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.655402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789022302289566:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.665210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789020713406316:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.665395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789020713406240:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.665670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789022302289566:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.666344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789020713406332:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53355
2025-11-28T14:51:35.666569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789020713406316:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:35.666626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789022302289566:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:35.666890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789020713406240:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.666991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789022302289557:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.667112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:35.666946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:35.666992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789020713406240:2050] [node 3] ICP22 created new session: [4:7577789020713406333:2050]
2025-11-28T14:51:35.667137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789020713406333:2050] [node 3] ICS09 handshake done sender: [4:7577789020713406316:2117] self: [4:7577789020713406331:0] peer: [3:7577789022302289565:0] socket: 160 qp: -1
2025-11-28T14:51:35.667262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789022302289557:2107] [node 4] ICP22 created new session: [3:7577789022302289585:2107]
2025-11-28T14:51:35.667230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789020713406333:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:35.667390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789022302289585:2107] [node 4] ICS09 handshake done sender: [3:7577789022302289566:2113] self: [3:7577789022302289565:0] peer: [4:7577789020713406331:0] socket: 156 qp: -1
2025-11-28T14:51:35.667431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789022302289585:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:35.667397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:35.667547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.667836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789023001862239:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-11-28T14:51:35.667910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789023001862240:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-11-28T14:51:35.692948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789020713406312:2114] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.693704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789022254885481:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.698066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789022254885457:2107] [node 4] ICP18 handshake (actor [2:7577789022254885481:2119]) is too old
2025-11-28T14:51:35.706904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789020713406310:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.715055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789023134856557:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.715277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789023134856532:2107] [node 4] ICP18 handshake (actor [1:7577789023134856557:2120]) is too old
2025-11-28T14:51:35.718500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789022254885467:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.727204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789022254885467:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.727934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789022254885467:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:35.726528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789022302289577:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.726763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789022302289495:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.727937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789022302289589:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38944
2025-11-28T14:51:35.728142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789022302289577:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:35.728303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789022302289495:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.730057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:35.730333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789022302289495:2049] [node 2] ICP22 created new session: [3:7577789022302289590:2049]
2025-11-28T14:51:35.730389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789022302289590:2049] [node 2] ICS09 handshake done sender: [3:7577789022302289577:2118] self: [3:7577789022302289588:0] peer: [2:7577789022254885466:0] socket: 164 qp: -1
2025-11-28T14:51:35.730428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789022302289590:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:35.730479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:35.728312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789022254885395:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.730316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:35.730370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789022254885395:2049] [node 3] ICP22 created new session: [2:7577789022254885495:2049]
2025-11-28T14:51:35.730425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789022254885495:2049] [node 3] ICS09 handshake done sender: [2:7577789022254885467:2113] self: [2:7577789022254885466:0] peer: [3:7577789022302289588:0] socket: 163 qp: -1
2025-11-28T14:51:35.730461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789022254885495:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:35.730496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:35.732813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789020713406309:2111] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.733556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789023074983523:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.733745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789023074983512:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.734021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789020713406309:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.740874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789020713406309:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.741112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789020713406302:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.741192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:35.741635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789020713406302:2109] [node 6] ICP22 created new session: [4:7577789020713406337:2109]
2025-11-28T14:51:35.741677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789020713406337:2109] [node 6] ICS09 handshake done sender: [4:7577789020713406309:2111] self: [4:7577789020713406304:0] peer: [6:7577789023074983542:0] socket: 167 qp: -1
2025-11-28T14:51:35.741710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789020713406337:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:35.741747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:35.741102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983543:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
2025-11-28T14:51:35.748485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789023074983523:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:35.760628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789023074983512:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27600) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789023074983523:2115] held: no
2025-11-28T14:51:35.768761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789023074983512:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.768816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:35.768871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789023074983512:2109] [node 4] ICP22 created new session: [6:7577789023074983544:2109]
2025-11-28T14:51:35.768914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789023074983544:2109] [node 4] ICS09 handshake done sender: [6:7577789023074983523:2115] self: [6:7577789023074983542:0] peer: [4:7577789020713406304:0] socket: 170 qp: -1
2025-11-28T14:51:35.768967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789023074983544:2109] [node 4] ICS10 traffic start
2025-11-28T14:51:35.769009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:35.781191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789022302289570:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.788899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789023074983538:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.789139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789023074983447:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.789454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789022302289570:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.790059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789022302289570:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:35.790421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789022302289559:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.790393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789023074983547:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63527
2025-11-28T14:51:35.790486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:35.790528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789022302289559:2109] [node 6] ICP22 created new session: [3:7577789022302289593:2109]
2025-11-28T14:51:35.790601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789022302289593:2109] [node 6] ICS09 handshake done sender: [3:7577789022302289570:2115] self: [3:7577789022302289569:0] peer: [6:7577789023074983546:0] socket: 157 qp: -1
2025-11-28T14:51:35.790631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789022302289593:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:35.790670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:35.790666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789023074983538:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:35.790785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789023074983447:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.790994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:35.792393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789023074983447:2050] [node 3] ICP22 created new session: [6:7577789023074983548:2050]
2025-11-28T14:51:35.792472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789023074983548:2050] [node 3] ICS09 handshake done sender: [6:7577789023074983538:2121] self: [6:7577789023074983546:0] peer: [3:7577789022302289569:0] socket: 184 qp: -1
2025-11-28T14:51:35.792587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789023074983548:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:35.792670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:35.794432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789020713406311:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.802491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789023001862240:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.802689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789023001862207:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.802887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789020713406311:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.803251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789020713406311:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.803469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789020713406301:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.803593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:35.803517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789023001862242:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-11-28T14:51:35.803644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789020713406301:2108] [node 5] ICP22 created new session: [4:7577789020713406340:2108]
2025-11-28T14:51:35.803754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789023001862240:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:35.803898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789023001862207:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.803705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789020713406340:2108] [node 5] ICS09 handshake done sender: [4:7577789020713406311:2113] self: [4:7577789020713406308:0] peer: [5:7577789023001862241:0] socket: 188 qp: -1
2025-11-28T14:51:35.803756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789020713406340:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:35.804421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:35.804496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789023001862207:2108] [node 4] ICP22 created new session: [5:7577789023001862243:2108]
2025-11-28T14:51:35.803795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:35.804963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789023001862243:2108] [node 4] ICS09 handshake done sender: [5:7577789023001862240:2122] self: [5:7577789023001862241:0] peer: [4:7577789020713406308:0] socket: 189 qp: -1
2025-11-28T14:51:35.805005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789023001862243:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:35.805045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:35.805343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789023001862207:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:35.807078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789022302289568:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:35.808140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789023001862239:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.808303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789023001862145:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:35.808578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789022302289568:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:35.808941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789022302289568:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:35.809099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789022302289558:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:35.809234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:35.809271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789022302289558:2108] [node 5] ICP22 created new session: [3:7577789022302289596:2108]
2025-11-28T14:51:35.809324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789022302289596:2108] [node 5] ICS09 handshake done sender: [3:7577789022302289568:2114] self: [3:7577789022302289567:0] peer: [5:7577789023001862245:0] socket: 186 qp: -1
2025-11-28T14:51:35.809354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789022302289596:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:35.809392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:35.809208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789023001862246:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-11-28T14:51:35.810042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789023001862145:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8344) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789023001862239:2121] held: no
2025-11-28T14:51:35.810069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789023001862239:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:35.810350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789023001862145:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:35.810586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:35.810630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789023001862145:2050] [node 3] ICP22 created new session: [5:7577789023001862247:2050]
2025-11-28T14:51:35.810684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789023001862247:2050] [node 3] ICS09 handshake done sender: [5:7577789023001862239:2121] self: [5:7577789023001862245:0] peer: [3:7577789022302289567:0] socket: 187 qp: -1
2025-11-28T14:51:35.810716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789023001862247:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:35.810769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:35.823808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:35.824036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789022254885484:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-11-28T14:51:35.824066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789022254885394:2048] [node 1] ICP30 unregister session Session# [2:7577789022254885484:2048] VirtualId# [2:7577789022254885474:0]
2025-11-28T14:51:35.824091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789022254885484:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.824173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:35.824230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:35.824913Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:35.825112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789022302289580:2048] [node 1] ICS07 socket disconnect 129 reason# EndOfStream
2025-11-28T14:51:35.825142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789022302289580:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.825222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:35.825268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789022302289580:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:35.827106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:35.827262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789022302289494:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:35.827312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789022302289580:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:35.827323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789020713406327:2048] [node 1] ICS07 socket disconnect 155 reason# EndOfStream
2025-11-28T14:51:35.827350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789020713406327:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.827405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:35.827435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789020713406327:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:35.828248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789022302289494:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.828304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789022302289580:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.828337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789022302289494:2048] [node 1] ICP30 unregister session Session# [3:7577789022302289580:2048] VirtualId# [3:7577789022302289575:0]
2025-11-28T14:51:35.828396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:35.828435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789022302289494:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.828449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.828497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789020713406238:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.828546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789020713406327:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.828609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789020713406238:2048] [node 1] ICP30 unregister session Session# [4:7577789020713406327:2048] VirtualId# [4:7577789020713406321:0]
2025-11-28T14:51:35.828658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:35.828694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789020713406238:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.828710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.828741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:35.829042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789023001862233:2048] [node 1] ICS07 socket disconnect 134 reason# EndOfStream
2025-11-28T14:51:35.829086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789023001862233:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.829204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:35.829268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789023001862233:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:35.829610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:35.829794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789023074983531:2048] [node 1] ICS01 socket: 174 reason# EndOfStream
2025-11-28T14:51:35.829832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789023074983445:2048] [node 1] ICP30 unregister session Session# [6:7577789023074983531:2048] VirtualId# [6:7577789023074983526:0]
2025-11-28T14:51:35.829857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789023074983531:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.829921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789023001862143:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.829922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:35.829982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:35.830053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789023001862233:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.830102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789023001862143:2048] [node 1] ICP30 unregister session Session# [5:7577789023001862233:2048] VirtualId# [5:7577789023001862231:0]
2025-11-28T14:51:35.830141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:35.830167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789023001862143:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.830196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.838353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789022302289494:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:35.838354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789020713406238:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:35.840384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789023001862143:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:35.865035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:35.865224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789020713406324:2049] [node 2] ICS07 socket disconnect 136 reason# EndOfStream
2025-11-28T14:51:35.865250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789020713406324:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.865315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:35.865368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789020713406324:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:35.866230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789020713406239:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.866323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789020713406324:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.866354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789020713406239:2049] [node 2] ICP30 unregister session Session# [4:7577789020713406324:2049] VirtualId# [4:7577789020713406319:0]
2025-11-28T14:51:35.866352Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:35.866570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:35.863357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:35.863654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789022302289590:2049] [node 2] ICS07 socket disconnect 164 reason# EndOfStream
2025-11-28T14:51:35.866603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789020713406239:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.863695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789022302289590:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.866615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.863795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:35.866541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789023001862227:2049] [node 2] ICS07 socket disconnect 128 reason# EndOfStream
2025-11-28T14:51:35.866564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789023001862227:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.863836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789022302289590:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:35.866622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:35.866656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789023001862227:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:35.864823Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789022302289495:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.864931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789022302289590:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.864985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789022302289495:2049] [node 2] ICP30 unregister session Session# [3:7577789022302289590:2049] VirtualId# [3:7577789022302289588:0]
2025-11-28T14:51:35.865037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:35.865067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789022302289495:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.865080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.867555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789023001862144:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.867641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789023001862227:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.867738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789023001862144:2049] [node 2] ICP30 unregister session Session# [5:7577789023001862227:2049] VirtualId# [5:7577789023001862220:0]
2025-11-28T14:51:35.867787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:35.867818Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789023001862144:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.867830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.867872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:35.868081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789023074983533:2049] [node 2] ICS07 socket disconnect 177 reason# EndOfStream
2025-11-28T14:51:35.868108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789023074983533:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.868200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:35.868236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789023074983533:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:35.869054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789023074983446:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.869126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789023074983533:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.869162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789023074983446:2049] [node 2] ICP30 unregister session Session# [6:7577789023074983533:2049] VirtualId# [6:7577789023074983527:0]
2025-11-28T14:51:35.869206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:35.869261Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789023074983446:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.869282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.876259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789020713406239:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:35.877260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789023001862144:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:35.878362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:35.878632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789020713406333:2050] [node 3] ICS07 socket disconnect 160 reason# EndOfStream
2025-11-28T14:51:35.878667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789020713406333:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.878749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:35.878790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789020713406333:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:35.879288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789023074983446:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:35.879328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:35.879543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789023001862247:2050] [node 3] ICS01 socket: 187 reason# EndOfStream
2025-11-28T14:51:35.879558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789020713406240:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8344) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.879611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789020713406333:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.879581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789023001862145:2050] [node 3] ICP30 unregister session Session# [5:7577789023001862247:2050] VirtualId# [5:7577789023001862245:0]
2025-11-28T14:51:35.879641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789020713406240:2050] [node 3] ICP30 unregister session Session# [4:7577789020713406333:2050] VirtualId# [4:7577789020713406331:0]
2025-11-28T14:51:35.879612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789023001862247:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.879680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:35.879715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789020713406240:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.879731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:35.879727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.879844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:35.880332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:35.880554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789023074983548:2050] [node 3] ICS07 socket disconnect 184 reason# EndOfStream
2025-11-28T14:51:35.880579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789023074983548:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.880640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:35.880694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789023074983548:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:35.881362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789023074983447:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8344) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.881411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789023074983548:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.881483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789023074983447:2050] [node 3] ICP30 unregister session Session# [6:7577789023074983548:2050] VirtualId# [6:7577789023074983546:0]
2025-11-28T14:51:35.881529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:35.881554Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789023074983447:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.881572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.889357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:35.889592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789023001862243:2108] [node 4] ICS07 socket disconnect 189 reason# EndOfStream
2025-11-28T14:51:35.889629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789023001862243:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.889713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:35.889775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789023001862243:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:35.890252Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:35.890495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789023074983544:2109] [node 4] ICS01 socket: 170 reason# EndOfStream
2025-11-28T14:51:35.890520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789023001862207:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27600) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.890585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789023074983512:2109] [node 4] ICP30 unregister session Session# [6:7577789023074983544:2109] VirtualId# [6:7577789023074983542:0]
2025-11-28T14:51:35.890635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789023001862243:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.890620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789023074983544:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.890674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789023001862207:2108] [node 4] ICP30 unregister session Session# [5:7577789023001862243:2108] VirtualId# [5:7577789023001862241:0]
2025-11-28T14:51:35.890737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:35.890693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:35.890773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789023001862207:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.890742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:35.890795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.891557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789023074983447:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:35.914895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:35.915131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789023074983540:2106] [node 5] ICS07 socket disconnect 141 reason# EndOfStream
2025-11-28T14:51:35.915162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789023074983540:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:35.915241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:35.915284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789023074983540:2106] [node 5] ICS15 start handshake
2025-11-28T14:51:35.915976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789023074983507:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11971) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:35.916079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789023074983540:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:35.916119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789023074983507:2106] [node 5] ICP30 unregister session Session# [6:7577789023074983540:2106] VirtualId# [6:7577789023074983537:0]
2025-11-28T14:51:35.916168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:35.916196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789023074983507:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:35.916209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:35.997978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789020135810454:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:35.998015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789020135810455:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:36.018514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789028453498776:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.018545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789028453498777:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:36.039340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789027236674115:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.039387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789027236674116:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:36.060006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789027760823354:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.060054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789027760823355:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:36.060088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789027760823356:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:36.081170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789027711272583:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.081211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789027711272584:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:36.081234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789027711272585:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:36.101003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789028023019460:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.101040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789028023019461:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:36.101066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789028023019462:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:36.200085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789028453498857:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:36.200030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789024430777813:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:36.200674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789028453498858:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:36.200689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789024430777814:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:36.200922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789028023019523:2106] [node 4] ICP01 ready to work
2025-11-28T14:51:36.201245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789027711272646:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:36.201347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789027760823417:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:36.201335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789027236674178:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:36.201661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789028023019524:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:36.201714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789027711272647:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:36.201641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789024430777815:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:36.202064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789027236674179:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:36.202120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789027760823418:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:36.202369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789027236674180:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:36.206930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789028453498863:2117] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.222391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789024430777826:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15148
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.222380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789027236674191:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59241
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.222452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789027236674192:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59241
2025-11-28T14:51:36.222509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789027236674193:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59241
2025-11-28T14:51:36.222587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789024430777827:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15148
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.222591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789028453498868:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14613
2025-11-28T14:51:36.222711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789024430777828:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15148
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.223669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789028453498871:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14613
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.224334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777826:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.224430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789024430777832:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15148
2025-11-28T14:51:36.223836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789028453498872:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14613
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.224411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789028453498868:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.224562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789024430777833:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15148
2025-11-28T14:51:36.224407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789027236674197:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59241
2025-11-28T14:51:36.224571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789028453498876:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14613
2025-11-28T14:51:36.224471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789027236674198:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59241
2025-11-28T14:51:36.224645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789028453498877:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14613
2025-11-28T14:51:36.226486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789028453498872:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.226461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789027236674191:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.226981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777828:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.227092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777827:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.227537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789027760823432:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41557
2025-11-28T14:51:36.227598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789027760823433:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41557
2025-11-28T14:51:36.227648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789027760823434:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41557
2025-11-28T14:51:36.230872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789028453498871:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.231443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789027236674192:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.231545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789027236674193:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.231525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789027760823433:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.231722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789027760823355:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.231881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789028453498865:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.231909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789028453498865:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.232018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789027760823434:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789027760823432:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789027236674198:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777832:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789027760823356:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.232306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789027760823354:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.232324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789027236674197:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789028453498877:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777833:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789028453498876:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777821:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789027236674186:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.232576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789024430777821:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.232563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789027236674186:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.233182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789028453498776:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.233387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789027236674116:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.233461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789027236674115:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.233422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789027760823438:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41557
2025-11-28T14:51:36.233494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789028453498865:2130] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:36.233507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777817:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.233533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789024430777817:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.233819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789027236674186:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:36.234256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789028453498862:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.234286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789028453498862:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.234382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777819:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.234406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789024430777819:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.234937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789027236674178:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.235030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:36.235010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789028453498857:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.235142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789027236674178:2107] [node 4] ICP22 created new session: [3:7577789027236674205:2107]
2025-11-28T14:51:36.235068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:36.235204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789027236674205:2107] [node 4] ICS09 handshake done sender: [3:7577789027236674186:2112] self: [3:7577789027236674185:0] peer: [4:7577789027760823436:0] socket: 160 qp: -1
2025-11-28T14:51:36.235117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789028453498857:2107] [node 4] ICP22 created new session: [2:7577789028453498884:2107]
2025-11-28T14:51:36.235346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789027236674205:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:36.235158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789028453498884:2107] [node 4] ICS09 handshake done sender: [2:7577789028453498865:2130] self: [2:7577789028453498864:0] peer: [4:7577789027760823435:0] socket: 152 qp: -1
2025-11-28T14:51:36.235383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:36.235195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789028453498884:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:36.235333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:36.235466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789027760823433:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:36.235799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789024430777821:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:36.235790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789027760823355:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.235981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.236034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789027760823355:2049] [node 2] ICP22 created new session: [4:7577789027760823441:2049]
2025-11-28T14:51:36.236075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789027760823441:2049] [node 2] ICS09 handshake done sender: [4:7577789027760823433:2116] self: [4:7577789027760823435:0] peer: [2:7577789028453498864:0] socket: 166 qp: -1
2025-11-28T14:51:36.236151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789027760823441:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:36.236179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:36.236531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789024430777813:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.236585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:36.236624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789024430777813:2107] [node 4] ICP22 created new session: [1:7577789024430777842:2107]
2025-11-28T14:51:36.236676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789024430777842:2107] [node 4] ICS09 handshake done sender: [1:7577789024430777821:2112] self: [1:7577789024430777820:0] peer: [4:7577789027760823437:0] socket: 149 qp: -1
2025-11-28T14:51:36.236730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789024430777842:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:36.236785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.236923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789027760823439:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41557
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.237237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789027236674207:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59241
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.237439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789027236674208:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59241
2025-11-28T14:51:36.237500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789027760823440:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41557
2025-11-28T14:51:36.237495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789028453498862:2129] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.238031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789028453498777:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.238055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789024430777817:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.238088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.238126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789028453498777:2049] [node 3] ICP22 created new session: [2:7577789028453498889:2049]
2025-11-28T14:51:36.238078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789027711272661:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33043
2025-11-28T14:51:36.238158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789028453498889:2049] [node 3] ICS09 handshake done sender: [2:7577789028453498862:2129] self: [2:7577789028453498861:0] peer: [3:7577789027236674202:0] socket: 151 qp: -1
2025-11-28T14:51:36.238150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789027711272662:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33043
2025-11-28T14:51:36.238191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789028453498889:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:36.238169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789027236674198:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:36.238220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:36.238786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789024430777819:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.239076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789020135810454:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.239045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789028453498887:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14613
2025-11-28T14:51:36.239109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.239134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789020135810454:2048] [node 2] ICP22 created new session: [1:7577789024430777845:2048]
2025-11-28T14:51:36.239177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789024430777845:2048] [node 2] ICS09 handshake done sender: [1:7577789024430777817:2110] self: [1:7577789024430777816:0] peer: [2:7577789028453498883:0] socket: 147 qp: -1
2025-11-28T14:51:36.239197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789024430777845:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:36.239240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:36.240225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789027236674116:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.240241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789020135810455:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.240278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.240283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.240323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789027236674116:2049] [node 2] ICP22 created new session: [3:7577789027236674210:2049]
2025-11-28T14:51:36.240313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789020135810455:2049] [node 3] ICP22 created new session: [1:7577789024430777847:2049]
2025-11-28T14:51:36.240350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789024430777847:2049] [node 3] ICS09 handshake done sender: [1:7577789024430777819:2111] self: [1:7577789024430777818:0] peer: [3:7577789027236674204:0] socket: 148 qp: -1
2025-11-28T14:51:36.240384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789024430777847:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:36.240399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789027236674210:2049] [node 2] ICS09 handshake done sender: [3:7577789027236674198:2119] self: [3:7577789027236674202:0] peer: [2:7577789028453498861:0] socket: 154 qp: -1
2025-11-28T14:51:36.240411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:36.241544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789027236674210:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:36.241581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:36.241846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789027760823356:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27111) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789027760823434:2117] held: no
2025-11-28T14:51:36.241969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789028453498876:2136] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:36.242064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789027760823432:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:36.242534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789027760823354:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789027760823432:2115] held: no
::ffff:127.1.0.0
2025-11-28T14:51:36.242609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789027760823354:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.242649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.242629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789027711272663:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33043
2025-11-28T14:51:36.242689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789027760823354:2048] [node 1] ICP22 created new session: [4:7577789027760823443:2048]
2025-11-28T14:51:36.242725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789027760823443:2048] [node 1] ICS09 handshake done sender: [4:7577789027760823432:2115] self: [4:7577789027760823437:0] peer: [1:7577789024430777820:0] socket: 165 qp: -1
2025-11-28T14:51:36.242754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789027760823443:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.242879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:36.243383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789027760823434:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:36.244108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789028453498776:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.244178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.244179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789027760823356:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.244226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789028453498776:2048] [node 1] ICP22 created new session: [2:7577789028453498892:2048]
2025-11-28T14:51:36.244227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.244264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789028453498892:2048] [node 1] ICS09 handshake done sender: [2:7577789028453498876:2136] self: [2:7577789028453498883:0] peer: [1:7577789024430777816:0] socket: 156 qp: -1
2025-11-28T14:51:36.244292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789028453498892:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.244258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789027760823356:2050] [node 3] ICP22 created new session: [4:7577789027760823445:2050]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:36.244314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:36.244301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789027760823445:2050] [node 3] ICS09 handshake done sender: [4:7577789027760823434:2117] self: [4:7577789027760823436:0] peer: [3:7577789027236674185:0] socket: 167 qp: -1
2025-11-28T14:51:36.244323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789027760823445:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:36.244349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:36.244396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789027711272661:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.244405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019538:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
2025-11-28T14:51:36.244518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789027711272585:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.244518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019539:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.246142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789027236674197:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:36.246151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019540:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
2025-11-28T14:51:36.246237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789027711272665:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33043
2025-11-28T14:51:36.246455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789027711272662:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.246465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789027236674115:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789027236674197:2118] held: no
2025-11-28T14:51:36.246540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789027236674115:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.246574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.246580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789027711272584:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.246616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789027236674115:2048] [node 1] ICP22 created new session: [3:7577789027236674212:2048]
2025-11-28T14:51:36.246467Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789028453498776:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:36.246657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789027236674212:2048] [node 1] ICS09 handshake done sender: [3:7577789027236674197:2118] self: [3:7577789027236674204:0] peer: [1:7577789024430777818:0] socket: 153 qp: -1
2025-11-28T14:51:36.246686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789027236674212:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.246767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789027711272663:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.246782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:36.246856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789027711272583:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.247779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789027236674188:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.247810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789027236674188:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.248485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789028453498867:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.248515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789028453498867:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.249205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777823:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.249240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789024430777823:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.249283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789027711272665:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.249288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789028023019538:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.249342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789028453498867:2131] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:36.249873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789027236674188:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:36.249521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789028023019540:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.250333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789028453498858:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.250467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.250472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789027236674179:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.250517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789028453498858:2108] [node 5] ICP22 created new session: [2:7577789028453498895:2108]
2025-11-28T14:51:36.250562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789028453498895:2108] [node 5] ICS09 handshake done sender: [2:7577789028453498867:2131] self: [2:7577789028453498866:0] peer: [5:7577789027711272666:0] socket: 180 qp: -1
2025-11-28T14:51:36.250553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.250550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789028023019461:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.250593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789028453498895:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:36.250592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789027236674179:2108] [node 5] ICP22 created new session: [3:7577789027236674216:2108]
2025-11-28T14:51:36.250616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789027236674216:2108] [node 5] ICS09 handshake done sender: [3:7577789027236674188:2113] self: [3:7577789027236674187:0] peer: [5:7577789027711272664:0] socket: 178 qp: -1
2025-11-28T14:51:36.250618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:36.250618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789028023019523:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.250639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789027236674216:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:36.250616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789024430777823:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:36.250666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:36.250720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789028023019539:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.250896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789027711272646:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.251021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789024430777814:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.250992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789027711272668:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33043
2025-11-28T14:51:36.251025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019541:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
2025-11-28T14:51:36.251072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.251106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789024430777814:2108] [node 5] ICP22 created new session: [1:7577789024430777851:2108]
2025-11-28T14:51:36.251102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789028023019460:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.251099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789027760823428:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.251144Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789027760823428:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.251145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789024430777851:2108] [node 5] ICS09 handshake done sender: [1:7577789024430777823:2113] self: [1:7577789024430777822:0] peer: [5:7577789027711272667:0] socket: 130 qp: -1
2025-11-28T14:51:36.251298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789024430777851:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:36.251330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.252059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789027711272669:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33043
2025-11-28T14:51:36.252140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789028453498870:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.252256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789027760823428:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:36.252167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789028453498870:2133] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.253442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789027760823418:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.253444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019545:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
2025-11-28T14:51:36.253493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:36.253539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789027760823418:2109] [node 6] ICP22 created new session: [4:7577789027760823450:2109]
2025-11-28T14:51:36.253576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789027760823450:2109] [node 6] ICS09 handshake done sender: [4:7577789027760823428:2114] self: [4:7577789027760823427:0] peer: [6:7577789028023019543:0] socket: 182 qp: -1
2025-11-28T14:51:36.253597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789027760823450:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:36.253623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789024430777825:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.253668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:36.253658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789024430777825:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.253974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789028453498870:2133] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:36.255162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789028453498863:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.255304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:36.255349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789028453498863:2117] [node 6] ICP22 created new session: [2:7577789028453498898:2117]
2025-11-28T14:51:36.255389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789028453498898:2117] [node 6] ICS09 handshake done sender: [2:7577789028453498870:2133] self: [2:7577789028453498869:0] peer: [6:7577789028023019542:0] socket: 136 qp: -1
2025-11-28T14:51:36.255294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789027760823426:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.255426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789028453498898:2117] [node 6] ICS10 traffic start
2025-11-28T14:51:36.255392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789024430777825:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:36.255313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789027760823426:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.255461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:36.255493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789028023019541:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.255941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789024430777815:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.255987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:36.256060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789024430777815:2109] [node 6] ICP22 created new session: [1:7577789024430777854:2109]
2025-11-28T14:51:36.256350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789024430777854:2109] [node 6] ICS09 handshake done sender: [1:7577789024430777825:2114] self: [1:7577789024430777824:0] peer: [6:7577789028023019544:0] socket: 158 qp: -1
2025-11-28T14:51:36.256393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789024430777854:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:36.256428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:36.256440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789028023019540:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:36.256557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789027760823426:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:36.257096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789027760823417:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.257159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789028023019523:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.257129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.257119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789027711272585:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27111) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789027711272661:2115] held: no
2025-11-28T14:51:36.257158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789027760823417:2108] [node 5] ICP22 created new session: [4:7577789027760823453:2108]
2025-11-28T14:51:36.257214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:36.257258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789028023019523:2106] [node 4] ICP22 created new session: [6:7577789028023019548:2106]
2025-11-28T14:51:36.257240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789027760823453:2108] [node 5] ICS09 handshake done sender: [4:7577789027760823426:2113] self: [4:7577789027760823425:0] peer: [5:7577789027711272671:0] socket: 157 qp: -1
2025-11-28T14:51:36.257256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789027760823453:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:36.257272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:36.257324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789028023019548:2106] [node 4] ICS09 handshake done sender: [6:7577789028023019540:2117] self: [6:7577789028023019543:0] peer: [4:7577789027760823427:0] socket: 183 qp: -1
2025-11-28T14:51:36.257389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789028023019548:2106] [node 4] ICS10 traffic start
2025-11-28T14:51:36.257427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:36.258578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789027711272662:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:36.259236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789027711272584:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789027711272662:2116] held: no
2025-11-28T14:51:36.259321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789027711272584:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.259376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.259420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789027711272584:2049] [node 2] ICP22 created new session: [5:7577789027711272672:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.259463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789027711272672:2049] [node 2] ICS09 handshake done sender: [5:7577789027711272662:2116] self: [5:7577789027711272666:0] peer: [2:7577789028453498866:0] socket: 181 qp: -1
2025-11-28T14:51:36.259506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019546:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
2025-11-28T14:51:36.259523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789027711272672:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:36.259558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.259929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019547:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
2025-11-28T14:51:36.260120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789027711272661:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:36.260477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789027711272583:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789027711272663:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.260609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789027711272670:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33043
2025-11-28T14:51:36.261666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789028023019461:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789028023019538:2115] held: no
2025-11-28T14:51:36.261737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789027711272585:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.261811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.261828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789028023019462:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.261866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789027711272585:2050] [node 3] ICP22 created new session: [5:7577789027711272675:2050]
2025-11-28T14:51:36.261905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789028023019460:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11323) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789028023019539:2116] held: no
2025-11-28T14:51:36.261914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789027711272675:2050] [node 3] ICS09 handshake done sender: [5:7577789027711272661:2115] self: [5:7577789027711272664:0] peer: [3:7577789027236674187:0] socket: 179 qp: -1
2025-11-28T14:51:36.261947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789027711272675:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:36.261973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:36.262100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789028023019538:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:36.262474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789028023019539:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:36.262702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789027711272663:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:36.262681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789027236674190:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.262723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789027236674190:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.262801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789028023019461:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.263060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.263265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789028023019461:2049] [node 2] ICP22 created new session: [6:7577789028023019551:2049]
2025-11-28T14:51:36.263338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789028023019551:2049] [node 2] ICS09 handshake done sender: [6:7577789028023019538:2115] self: [6:7577789028023019542:0] peer: [2:7577789028453498869:0] socket: 140 qp: -1
2025-11-28T14:51:36.263380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789028023019551:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:36.263404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.263638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789027236674190:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:36.263668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789027711272674:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33043
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.263873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789028023019460:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.263920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789027760823455:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41557
2025-11-28T14:51:36.264052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.264139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789027236674180:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.264134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789028023019460:2048] [node 1] ICP22 created new session: [6:7577789028023019554:2048]
2025-11-28T14:51:36.264177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789028023019554:2048] [node 1] ICS09 handshake done sender: [6:7577789028023019539:2116] self: [6:7577789028023019544:0] peer: [1:7577789024430777824:0] socket: 159 qp: -1
2025-11-28T14:51:36.264210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789028023019554:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.264201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:36.264250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789027236674180:2109] [node 6] ICP22 created new session: [3:7577789027236674219:2109]
2025-11-28T14:51:36.264236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:36.264293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789027236674219:2109] [node 6] ICS09 handshake done sender: [3:7577789027236674190:2114] self: [3:7577789027236674189:0] peer: [6:7577789028023019550:0] socket: 185 qp: -1
2025-11-28T14:51:36.264276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789027711272583:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.264338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789027236674219:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:36.264377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.264385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:36.264449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789027711272583:2048] [node 1] ICP22 created new session: [5:7577789027711272679:2048]
2025-11-28T14:51:36.264495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789027711272679:2048] [node 1] ICS09 handshake done sender: [5:7577789027711272663:2117] self: [5:7577789027711272667:0] peer: [1:7577789024430777822:0] socket: 133 qp: -1
2025-11-28T14:51:36.264523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789027711272679:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.264552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:36.265319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789027711272665:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:36.265661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789027711272646:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.265745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:36.265779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789027711272646:2108] [node 4] ICP22 created new session: [5:7577789027711272682:2108]
2025-11-28T14:51:36.265846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789027711272682:2108] [node 4] ICS09 handshake done sender: [5:7577789027711272665:2118] self: [5:7577789027711272671:0] peer: [4:7577789027760823425:0] socket: 161 qp: -1
2025-11-28T14:51:36.265880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789027711272682:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:36.265910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.266704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019556:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.267057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019557:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
2025-11-28T14:51:36.267123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789028023019462:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27111) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789028023019541:2118] held: no
2025-11-28T14:51:36.267287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789028023019541:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:36.269049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789028023019462:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.269121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.269069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789027760823417:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:36.269157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789028023019462:2050] [node 3] ICP22 created new session: [6:7577789028023019558:2050]
2025-11-28T14:51:36.269187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789028023019558:2050] [node 3] ICS09 handshake done sender: [6:7577789028023019541:2118] self: [6:7577789028023019550:0] peer: [3:7577789027236674189:0] socket: 187 qp: -1
2025-11-28T14:51:36.269372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789028023019558:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:36.269540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.269777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789027711272681:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33043
2025-11-28T14:51:36.270211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789027711272681:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.270239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789028023019557:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.270345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789028023019524:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.270532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789027711272657:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.270557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789027711272657:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.271274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789027711272657:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.271425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789028023019561:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57437
2025-11-28T14:51:36.271594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789027711272647:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.271676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:36.271724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789027711272647:2109] [node 6] ICP22 created new session: [5:7577789027711272686:2109]
2025-11-28T14:51:36.271760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789027711272686:2109] [node 6] ICS09 handshake done sender: [5:7577789027711272657:2114] self: [5:7577789027711272656:0] peer: [6:7577789028023019560:0] socket: 134 qp: -1
2025-11-28T14:51:36.271856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789027711272686:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:36.271887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:36.271914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789028023019557:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:36.272115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789028023019524:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.272172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.272207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789028023019524:2107] [node 5] ICP22 created new session: [6:7577789028023019562:2107]
2025-11-28T14:51:36.272237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789028023019562:2107] [node 5] ICS09 handshake done sender: [6:7577789028023019557:2123] self: [6:7577789028023019560:0] peer: [5:7577789027711272656:0] socket: 193 qp: -1
2025-11-28T14:51:36.272262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789028023019562:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:36.272309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:36.289228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:36.289465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789028453498892:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-28T14:51:36.289516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789028453498776:2048] [node 1] ICP30 unregister session Session# [2:7577789028453498892:2048] VirtualId# [2:7577789028453498883:0]
2025-11-28T14:51:36.289555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789028453498892:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.289627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.289691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.290372Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:36.290574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789027236674212:2048] [node 1] ICS01 socket: 153 reason# EndOfStream
2025-11-28T14:51:36.290618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789027236674115:2048] [node 1] ICP30 unregister session Session# [3:7577789027236674212:2048] VirtualId# [3:7577789027236674204:0]
2025-11-28T14:51:36.290643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789027236674212:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.290709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.290764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.292807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:36.293036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789027760823443:2048] [node 1] ICS01 socket: 165 reason# EndOfStream
2025-11-28T14:51:36.293100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789027760823354:2048] [node 1] ICP30 unregister session Session# [4:7577789027760823443:2048] VirtualId# [4:7577789027760823437:0]
2025-11-28T14:51:36.293135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789027760823443:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.293204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.293258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.293520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:36.293737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789027711272679:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-11-28T14:51:36.293790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789027711272583:2048] [node 1] ICP30 unregister session Session# [5:7577789027711272679:2048] VirtualId# [5:7577789027711272667:0]
2025-11-28T14:51:36.293836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789027711272679:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.293902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.293946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.294325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:36.294530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789028023019554:2048] [node 1] ICS01 socket: 159 reason# EndOfStream
2025-11-28T14:51:36.294581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789028023019460:2048] [node 1] ICP30 unregister session Session# [6:7577789028023019554:2048] VirtualId# [6:7577789028023019544:0]
2025-11-28T14:51:36.294608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789028023019554:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.294680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.294743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.313958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:36.314181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789027236674210:2049] [node 2] ICS07 socket disconnect 154 reason# EndOfStream
2025-11-28T14:51:36.314211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789027236674210:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.314301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:36.314346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789027236674210:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:36.315196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789027236674116:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5433) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:36.315277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789027236674210:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:36.315309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789027236674116:2049] [node 2] ICP30 unregister session Session# [3:7577789027236674210:2049] VirtualId# [3:7577789027236674202:0]
2025-11-28T14:51:36.315351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:36.315386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789027236674116:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.315404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.316038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:36.316264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789027760823441:2049] [node 2] ICS01 socket: 166 reason# EndOfStream
2025-11-28T14:51:36.316318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789027760823355:2049] [node 2] ICP30 unregister session Session# [4:7577789027760823441:2049] VirtualId# [4:7577789027760823435:0]
2025-11-28T14:51:36.316353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789027760823441:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.316432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:36.316484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:36.316641Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:36.316866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789027711272672:2049] [node 2] ICS01 socket: 181 reason# EndOfStream
2025-11-28T14:51:36.316926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789027711272584:2049] [node 2] ICP30 unregister session Session# [5:7577789027711272672:2049] VirtualId# [5:7577789027711272666:0]
2025-11-28T14:51:36.316950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789027711272672:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.317021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:36.317073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:36.317421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:36.317622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789028023019551:2049] [node 2] ICS01 socket: 140 reason# EndOfStream
2025-11-28T14:51:36.317673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789028023019461:2049] [node 2] ICP30 unregister session Session# [6:7577789028023019551:2049] VirtualId# [6:7577789028023019542:0]
2025-11-28T14:51:36.317700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789028023019551:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.317764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:36.317817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:36.337527Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:36.337770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789027760823445:2050] [node 3] ICS01 socket: 167 reason# EndOfStream
2025-11-28T14:51:36.337825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789027760823356:2050] [node 3] ICP30 unregister session Session# [4:7577789027760823445:2050] VirtualId# [4:7577789027760823436:0]
2025-11-28T14:51:36.337870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789027760823445:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.337947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:36.337994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:36.338232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:36.338447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789027711272675:2050] [node 3] ICS01 socket: 179 reason# EndOfStream
2025-11-28T14:51:36.338499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789027711272585:2050] [node 3] ICP30 unregister session Session# [5:7577789027711272675:2050] VirtualId# [5:7577789027711272664:0]
2025-11-28T14:51:36.338534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789027711272675:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.338597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:36.338632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:36.338962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:36.339158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789028023019558:2050] [node 3] ICS01 socket: 187 reason# EndOfStream
2025-11-28T14:51:36.339209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789028023019462:2050] [node 3] ICP30 unregister session Session# [6:7577789028023019558:2050] VirtualId# [6:7577789028023019550:0]
2025-11-28T14:51:36.339231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789028023019558:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.339295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:36.339346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:36.364090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:36.364330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789027711272682:2108] [node 4] ICS07 socket disconnect 161 reason# EndOfStream
2025-11-28T14:51:36.364377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789027711272682:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.364447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:36.364495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789027711272682:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:36.365237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:36.365277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789027711272646:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21922) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:36.365326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789027711272682:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:36.365360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789027711272646:2108] [node 4] ICP30 unregister session Session# [5:7577789027711272682:2108] VirtualId# [5:7577789027711272671:0]
2025-11-28T14:51:36.365404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:36.365434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789027711272646:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21922) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.365447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21922) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.365480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789028023019548:2106] [node 4] ICS07 socket disconnect 183 reason# EndOfStream
2025-11-28T14:51:36.365515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789028023019548:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.365607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:36.365653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789028023019548:2106] [node 4] ICS15 start handshake
2025-11-28T14:51:36.366371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789028023019523:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21922) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:36.366419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789028023019548:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:36.366453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789028023019523:2106] [node 4] ICP30 unregister session Session# [6:7577789028023019548:2106] VirtualId# [6:7577789028023019543:0]
2025-11-28T14:51:36.366486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:36.366508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789028023019523:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21922) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.366522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21922) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.375984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789028023019523:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:51:36.380917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:36.381103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789028023019562:2107] [node 5] ICS01 socket: 193 reason# EndOfStream
2025-11-28T14:51:36.381201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789028023019524:2107] [node 5] ICP30 unregister session Session# [6:7577789028023019562:2107] VirtualId# [6:7577789028023019560:0]
2025-11-28T14:51:36.381250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789028023019562:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.381328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:36.381374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:36.440950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789026794560915:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:36.440953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789026794560914:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:36.459090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789025777231093:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.459094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789025777231094:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:36.476060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789026055183044:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.476108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789026055183045:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:36.494417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789026793350266:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:36.494422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789026793350265:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.494455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789026793350267:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:36.512173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789024908582112:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:36.512173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789024908582111:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.512200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789024908582113:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:36.532848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789027262370391:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:36.532854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789027262370392:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:36.532907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789027262370393:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:36.634118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789026793350329:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:36.634414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789027262370453:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:36.634818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789026793350330:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:36.635035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789027262370455:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:36.635071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789024908582174:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:36.635103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789024908582175:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:36.635115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789025777231157:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:36.635569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789025777231158:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:36.635931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789025777231159:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:36.637106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789025777231093:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51801455ns) {Configure:104(51795651ns) {ProcessConfigured:135(142850ns) {StartInitialHandshake:157(140066ns) {DropHandshakes:470(2091ns) {DropIncomingHandshake:442(669ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(795ns) PrepareNewSessionHandshake:477(963ns) {SwitchToState:179(220ns)}}}}}
2025-11-28T14:51:36.637354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789026793350265:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52183460ns) {Configure:104(52175419ns) {ProcessConfigured:135(162798ns) {StartInitialHandshake:157(160705ns) {DropHandshakes:470(7531ns) {DropIncomingHandshake:442(3308ns) DropOutgoingHandshake:457(329ns)} GenerateSessionVirtualId:988(860ns) PrepareNewSessionHandshake:477(1013ns) {SwitchToState:179(289ns)}}}}}
2025-11-28T14:51:36.638903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789026794560915:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(58918662ns) {Configure:104(58915566ns) {ProcessConfigured:135(335407ns) {StartInitialHandshake:157(330981ns) {DropHandshakes:470(1204ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:477(1012ns) {SwitchToState:179(531ns)}}}}}
2025-11-28T14:51:36.638905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789026794560914:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(59006992ns) {Configure:104(59002515ns) {ProcessConfigured:135(345223ns) {StartInitialHandshake:157(342643ns) {DropHandshakes:470(1175ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(485ns) PrepareNewSessionHandshake:477(1908ns) {SwitchToState:179(271ns)}}}}}
2025-11-28T14:51:36.639088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789026055183045:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55510897ns) {Configure:104(55507370ns) {ProcessConfigured:135(181235ns) {StartInitialHandshake:157(179874ns) {DropHandshakes:470(53130ns) {DropIncomingHandshake:442(259ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(907ns) {SwitchToState:179(451ns)}}}}}
2025-11-28T14:51:36.639097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789026055183044:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(55693979ns) {Configure:104(55689937ns) {ProcessConfigured:135(274691ns) {StartInitialHandshake:157(269697ns) {DropHandshakes:470(1112ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(670ns) PrepareNewSessionHandshake:477(771ns) {SwitchToState:179(406ns)}}}}}
2025-11-28T14:51:36.639468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789026794560982:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:36.639469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789026794560983:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:36.640275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789026055183138:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:36.641001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789026794560984:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:36.640948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789026055183139:2113] [node 5] ICP01 ready to work
2025-11-28T14:51:36.642835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789026055183140:2125] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.647821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789025777231173:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48178
2025-11-28T14:51:36.648157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789026793350344:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20726
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.657508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789025777231173:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.658291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789025777231174:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48178
2025-11-28T14:51:36.657511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789026793350344:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.658844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789025777231165:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.658880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789025777231165:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.658589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789026793350266:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.659724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789025777231165:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:36.659636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789024908582175:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17730129ns) {Configure:104(17726068ns) {ProcessConfigured:135(1643841ns) {StartInitialHandshake:157(1642433ns) {DropHandshakes:470(537069ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(1286ns)} GenerateSessionVirtualId:988(1651ns) PrepareNewSessionHandshake:477(2009ns) {SwitchToState:179(1194ns)}}}}}
2025-11-28T14:51:36.664966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789025777231157:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:36.665382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
::ffff:127.1.0.0
2025-11-28T14:51:36.665436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789025777231157:2107] [node 4] ICP22 created new session: [2:7577789025777231177:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.665777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789025777231177:2107] [node 4] ICS09 handshake done sender: [2:7577789025777231165:2113] self: [2:7577789025777231164:0] peer: [4:7577789026793350345:0] socket: 134 qp: -1
2025-11-28T14:51:36.666014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789025777231177:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:36.666040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789026055183146:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48492
2025-11-28T14:51:36.666215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789026055183145:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48492
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.666235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789026793350346:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20726
2025-11-28T14:51:36.666302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.666635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789024908582186:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22304
2025-11-28T14:51:36.668393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789026794560990:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9507
2025-11-28T14:51:36.669278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789026794560989:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9507
2025-11-28T14:51:36.668660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789026793350266:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12988) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789026793350344:2116] held: no
2025-11-28T14:51:36.669463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789026793350344:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.669984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789024908582186:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.670460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789026793350266:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.669734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789025777231174:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.669783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789026055183145:2141] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.670485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789026794560991:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9507
2025-11-28T14:51:36.670609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.670677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789026793350266:2049] [node 2] ICP22 created new session: [4:7577789026793350348:2049]
2025-11-28T14:51:36.670543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789026055183146:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.670739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560990:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.671369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789026793350348:2049] [node 2] ICS09 handshake done sender: [4:7577789026793350344:2116] self: [4:7577789026793350345:0] peer: [2:7577789025777231164:0] socket: 138 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.672050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560991:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.671590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789026793350348:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:36.672032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.672431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789026794560997:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9507
2025-11-28T14:51:36.672436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789026794560996:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9507
2025-11-28T14:51:36.673019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789026055183152:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48492
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.672894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560989:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.673362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789026055183045:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.673298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789025777231179:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48178
2025-11-28T14:51:36.673310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789026793350347:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20726
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:51:36.673458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789025777231181:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48178
2025-11-28T14:51:36.673488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789026793350351:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20726
2025-11-28T14:51:36.673492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789025777231182:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48178
2025-11-28T14:51:36.673551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560997:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.686016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789024908582190:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22304
2025-11-28T14:51:36.686617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789024908582191:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22304
2025-11-28T14:51:36.686770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789026055183153:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48492
2025-11-28T14:51:36.687259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789024908582190:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.687801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789024908582112:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.687739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789025777231163:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.688027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789024908582191:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.687993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789025777231163:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.688549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789025777231167:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.688577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789024908582175:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.688573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789025777231167:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.689613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789026793350338:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.689654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789026793350338:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.689998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789025777231167:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:36.690079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789025777231163:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.690867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789025777231158:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.690889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789024908582195:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22304
2025-11-28T14:51:36.690933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.690986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789025777231094:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.690988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789025777231158:2108] [node 5] ICP22 created new session: [2:7577789025777231186:2108]
2025-11-28T14:51:36.691031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.691042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789025777231186:2108] [node 5] ICS09 handshake done sender: [2:7577789025777231167:2114] self: [2:7577789025777231166:0] peer: [5:7577789024908582193:0] socket: 171 qp: -1
2025-11-28T14:51:36.691096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789025777231186:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:36.691103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789025777231094:2049] [node 3] ICP22 created new session: [2:7577789025777231187:2049]
2025-11-28T14:51:36.691140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:36.691181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789025777231187:2049] [node 3] ICS09 handshake done sender: [2:7577789025777231163:2112] self: [2:7577789025777231162:0] peer: [3:7577789026055183154:0] socket: 133 qp: -1
2025-11-28T14:51:36.691220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789026793350338:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:36.691219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789025777231187:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:36.691254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:36.691519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789026793350329:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.691571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.691631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789026793350329:2108] [node 5] ICP22 created new session: [4:7577789026793350353:2108]
2025-11-28T14:51:36.691696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789026793350353:2108] [node 5] ICS09 handshake done sender: [4:7577789026793350338:2114] self: [4:7577789026793350337:0] peer: [5:7577789024908582194:0] socket: 175 qp: -1
2025-11-28T14:51:36.691730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789026793350353:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:36.691762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:36.691971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789024908582190:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.692377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370469:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.692460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370470:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.692810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789024908582112:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.692855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.692904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789024908582112:2049] [node 2] ICP22 created new session: [5:7577789024908582197:2049]
2025-11-28T14:51:36.693117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789024908582197:2049] [node 2] ICS09 handshake done sender: [5:7577789024908582190:2116] self: [5:7577789024908582193:0] peer: [2:7577789025777231166:0] socket: 174 qp: -1
2025-11-28T14:51:36.693165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789024908582197:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:36.693194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:36.693822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789027262370469:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.693954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789027262370455:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.694207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789024908582196:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22304
2025-11-28T14:51:36.694224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789027262370470:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.694180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789026793350343:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.694217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789026793350343:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.694388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789027262370392:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.694507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789025777231169:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.694535Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789025777231169:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.696256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789026793350343:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:36.696284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789025777231169:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:36.696283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789024908582191:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:36.697353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789026793350330:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.697580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:36.698100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789026793350330:2109] [node 6] ICP22 created new session: [4:7577789026793350357:2109]
::ffff:127.1.0.0
2025-11-28T14:51:36.698065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789024908582175:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.698075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789025777231159:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.698156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789026793350357:2109] [node 6] ICS09 handshake done sender: [4:7577789026793350343:2115] self: [4:7577789026793350340:0] peer: [6:7577789027262370471:0] socket: 183 qp: -1
2025-11-28T14:51:36.698177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:36.698165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370474:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.698202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789026793350357:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:36.698180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:36.698226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789024908582175:2108] [node 4] ICP22 created new session: [5:7577789024908582199:2108]
2025-11-28T14:51:36.698264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:36.698269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789024908582199:2108] [node 4] ICS09 handshake done sender: [5:7577789024908582191:2117] self: [5:7577789024908582194:0] peer: [4:7577789026793350337:0] socket: 177 qp: -1
2025-11-28T14:51:36.698195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370473:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.698245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789025777231159:2109] [node 6] ICP22 created new session: [2:7577789025777231192:2109]
2025-11-28T14:51:36.698297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789024908582199:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:36.698336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:36.698308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789025777231192:2109] [node 6] ICS09 handshake done sender: [2:7577789025777231169:2115] self: [2:7577789025777231168:0] peer: [6:7577789027262370472:0] socket: 185 qp: -1
2025-11-28T14:51:36.698350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789025777231192:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:36.698390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:36.699026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789027262370469:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:36.699759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789027262370470:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:36.700104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789027262370455:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.700111Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789027262370392:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12988) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789027262370470:2116] held: no
2025-11-28T14:51:36.700152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:36.700197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789027262370455:2108] [node 4] ICP22 created new session: [6:7577789027262370475:2108]
2025-11-28T14:51:36.700213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789027262370392:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.700237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789027262370475:2108] [node 4] ICS09 handshake done sender: [6:7577789027262370469:2115] self: [6:7577789027262370471:0] peer: [4:7577789026793350340:0] socket: 184 qp: -1
2025-11-28T14:51:36.700241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.700260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789027262370475:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:36.700298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:36.700304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789027262370392:2049] [node 2] ICP22 created new session: [6:7577789027262370476:2049]
2025-11-28T14:51:36.700351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789027262370476:2049] [node 2] ICS09 handshake done sender: [6:7577789027262370470:2116] self: [6:7577789027262370472:0] peer: [2:7577789025777231168:0] socket: 186 qp: -1
2025-11-28T14:51:36.700397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789027262370476:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:36.700437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.701368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370479:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.702225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789027262370479:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.702320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789027262370453:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.702493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789024908582183:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.702520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789024908582183:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:36.703608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789024908582183:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:36.704867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789026055183136:2137] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.705653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789024908582174:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.705716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:36.705753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789024908582174:2109] [node 6] ICP22 created new session: [5:7577789024908582203:2109]
2025-11-28T14:51:36.705795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789024908582203:2109] [node 6] ICS09 handshake done sender: [5:7577789024908582183:2113] self: [5:7577789024908582182:0] peer: [6:7577789027262370481:0] socket: 143 qp: -1
2025-11-28T14:51:36.705828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789024908582203:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:36.705874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.706024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370482:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.706893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789027262370479:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:36.713550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789027262370453:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.713663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.713779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789027262370453:2106] [node 5] ICP22 created new session: [6:7577789027262370483:2106]
2025-11-28T14:51:36.713862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789027262370483:2106] [node 5] ICS09 handshake done sender: [6:7577789027262370479:2119] self: [6:7577789027262370481:0] peer: [5:7577789024908582182:0] socket: 144 qp: -1
2025-11-28T14:51:36.713905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789027262370483:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:36.713952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:36.728932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789024908582179:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.729775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789025777231181:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.729938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789025777231158:2108] [node 5] ICP18 handshake (actor [2:7577789025777231181:2119]) is too old
2025-11-28T14:51:36.748581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789026055183137:2138] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.762720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789026055183153:2145] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.763443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789026055183156:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48492
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.763814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789025777231179:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.764005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789026055183157:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48492
2025-11-28T14:51:36.764095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789025777231094:2049] [node 3] ICP18 handshake (actor [2:7577789025777231179:2118]) is too old
2025-11-28T14:51:36.764150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789026055183145:2141] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:36.764424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789026055183045:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.764484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.764593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789026055183045:2049] [node 2] ICP22 created new session: [3:7577789026055183159:2049]
2025-11-28T14:51:36.764668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789026055183159:2049] [node 2] ICS09 handshake done sender: [3:7577789026055183145:2141] self: [3:7577789026055183154:0] peer: [2:7577789025777231162:0] socket: 149 qp: -1
2025-11-28T14:51:36.764706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789026055183159:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:36.764751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:36.765911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789026055183157:2147] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.766738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789024908582205:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22304
2025-11-28T14:51:36.771515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789026794560981:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.779525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560996:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.779593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789025777231182:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.779852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789025777231093:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.792689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789026794560986:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.801959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560981:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.802109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789026793350351:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.802338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789026793350265:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.802457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789026794560981:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.802833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789025777231195:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48178
2025-11-28T14:51:36.802921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560986:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.802966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789026794560914:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.803018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:36.803051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789026794560914:2048] [node 2] ICP22 created new session: [1:7577789026794561005:2048]
2025-11-28T14:51:36.803081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789026794561005:2048] [node 2] ICS09 handshake done sender: [1:7577789026794560981:2112] self: [1:7577789026794560979:0] peer: [2:7577789025777231194:0] socket: 166 qp: -1
2025-11-28T14:51:36.803110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789026794561005:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:36.803139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:36.803138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789025777231182:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:36.803491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789025777231093:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.803549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.803622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789025777231093:2048] [node 1] ICP22 created new session: [2:7577789025777231196:2048]
2025-11-28T14:51:36.803669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789025777231196:2048] [node 1] ICS09 handshake done sender: [2:7577789025777231182:2120] self: [2:7577789025777231194:0] peer: [1:7577789026794560979:0] socket: 170 qp: -1
2025-11-28T14:51:36.803704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789025777231196:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.803740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.805273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789026793350360:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20726
2025-11-28T14:51:36.805415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789026794560986:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:36.805698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789026793350351:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:36.805829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789026794560983:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.805893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:36.805962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789026793350265:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.805981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789026794560983:2107] [node 4] ICP22 created new session: [1:7577789026794561009:2107]
2025-11-28T14:51:36.806000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.806026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789026794561009:2107] [node 4] ICS09 handshake done sender: [1:7577789026794560986:2113] self: [1:7577789026794560985:0] peer: [4:7577789026793350359:0] socket: 172 qp: -1
2025-11-28T14:51:36.806025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789026793350265:2048] [node 1] ICP22 created new session: [4:7577789026793350361:2048]
2025-11-28T14:51:36.806061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789026794561009:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:36.806048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789026793350361:2048] [node 1] ICS09 handshake done sender: [4:7577789026793350351:2119] self: [4:7577789026793350359:0] peer: [1:7577789026794560985:0] socket: 173 qp: -1
2025-11-28T14:51:36.806091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:36.806032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789024908582206:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22304
2025-11-28T14:51:36.806079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789026793350361:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.806097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.808763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370485:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.821825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789026055183142:2139] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.829808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789026793350347:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.830038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789026793350267:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.830309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789026055183142:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.830758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789026055183142:2139] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.831036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789026793350364:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20726
2025-11-28T14:51:36.831075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789026055183138:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.831150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:36.831204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789026055183138:2107] [node 4] ICP22 created new session: [3:7577789026055183164:2107]
2025-11-28T14:51:36.831254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789026055183164:2107] [node 4] ICS09 handshake done sender: [3:7577789026055183142:2139] self: [3:7577789026055183141:0] peer: [4:7577789026793350363:0] socket: 160 qp: -1
2025-11-28T14:51:36.831285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789026793350347:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:36.831321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789026055183164:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:36.831423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:36.831610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789026793350267:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.831667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.831743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789026793350267:2050] [node 3] ICP22 created new session: [4:7577789026793350365:2050]
2025-11-28T14:51:36.831793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789026793350365:2050] [node 3] ICS09 handshake done sender: [4:7577789026793350347:2118] self: [4:7577789026793350363:0] peer: [3:7577789026055183141:0] socket: 167 qp: -1
2025-11-28T14:51:36.831831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789026793350365:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:36.831873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.834339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370486:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.844924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789026055183144:2140] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.854086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789024908582205:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.854303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789024908582113:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.854539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789026055183144:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.855412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789026055183144:2140] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.855776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789024908582208:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22304
2025-11-28T14:51:36.859227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789026055183139:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.859557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789024908582205:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:36.859299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.859352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789026055183139:2113] [node 5] ICP22 created new session: [3:7577789026055183168:2113]
2025-11-28T14:51:36.859395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789026055183168:2113] [node 5] ICS09 handshake done sender: [3:7577789026055183144:2140] self: [3:7577789026055183143:0] peer: [5:7577789024908582207:0] socket: 156 qp: -1
2025-11-28T14:51:36.859428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789026055183168:2113] [node 5] ICS10 traffic start
2025-11-28T14:51:36.859465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:36.861437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789024908582113:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.861965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789026794560980:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.861517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.862206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789024908582113:2050] [node 3] ICP22 created new session: [5:7577789024908582209:2050]
2025-11-28T14:51:36.886772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789026055183152:2144] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.862373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789024908582209:2050] [node 3] ICS09 handshake done sender: [5:7577789024908582205:2120] self: [5:7577789024908582207:0] peer: [3:7577789026055183143:0] socket: 161 qp: -1
2025-11-28T14:51:36.862411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789024908582209:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:36.864682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:36.887774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789026055183044:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.892793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789026794560988:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.893685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560980:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.895445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789026794560980:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:36.901105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789024908582206:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.904609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789024908582111:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.906517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789026794560915:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.906617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789026055183171:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48492
2025-11-28T14:51:36.906610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.906654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789026794560915:2049] [node 3] ICP22 created new session: [1:7577789026794561013:2049]
2025-11-28T14:51:36.906712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789026794561013:2049] [node 3] ICS09 handshake done sender: [1:7577789026794560980:2111] self: [1:7577789026794560978:0] peer: [3:7577789026055183170:0] socket: 157 qp: -1
2025-11-28T14:51:36.906748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789026794561013:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:36.906785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:36.907172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789026055183152:2144] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:36.908293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789026055183044:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.908383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.908345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560988:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.908426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789026055183044:2048] [node 1] ICP22 created new session: [3:7577789026055183172:2048]
2025-11-28T14:51:36.908478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789026055183172:2048] [node 1] ICS09 handshake done sender: [3:7577789026055183152:2144] self: [3:7577789026055183170:0] peer: [1:7577789026794560978:0] socket: 139 qp: -1
2025-11-28T14:51:36.908503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789026055183172:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.908531Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789026055183148:2143] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.908542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:36.911982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789027262370486:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.917814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789024908582212:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22304
2025-11-28T14:51:36.917915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789026055183148:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.916847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789027262370393:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.917996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789026794560988:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:36.918440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789024908582206:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:36.918440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789026055183148:2143] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:36.919802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789024908582111:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.918913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789026055183140:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.919069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:36.919853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789026794560982:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.919910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:36.919119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789026055183140:2125] [node 6] ICP22 created new session: [3:7577789026055183175:2125]
2025-11-28T14:51:36.919163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789026055183175:2125] [node 6] ICS09 handshake done sender: [3:7577789026055183148:2143] self: [3:7577789026055183147:0] peer: [6:7577789027262370487:0] socket: 137 qp: -1
2025-11-28T14:51:36.919959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789026794560982:2108] [node 5] ICP22 created new session: [1:7577789026794561016:2108]
2025-11-28T14:51:36.919200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789026055183175:2125] [node 6] ICS10 traffic start
2025-11-28T14:51:36.919231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:36.920017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789026794561016:2108] [node 5] ICS09 handshake done sender: [1:7577789026794560988:2114] self: [1:7577789026794560987:0] peer: [5:7577789024908582211:0] socket: 194 qp: -1
No RDMA context found for GID: 2025-11-28T14:51:36.920054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789026794561016:2108] [node 5] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:51:36.920081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:36.919939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.920404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370488:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.919999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789024908582111:2048] [node 1] ICP22 created new session: [5:7577789024908582213:2048]
2025-11-28T14:51:36.920045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789024908582213:2048] [node 1] ICS09 handshake done sender: [5:7577789024908582206:2121] self: [5:7577789024908582211:0] peer: [1:7577789026794560987:0] socket: 195 qp: -1
2025-11-28T14:51:36.920077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789024908582213:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.922432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789027262370393:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27837) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789027262370486:2122] held: no
2025-11-28T14:51:36.920107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:36.922627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789027262370486:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:36.922946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789027262370393:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.922992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:36.923038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789027262370393:2050] [node 3] ICP22 created new session: [6:7577789027262370489:2050]
2025-11-28T14:51:36.923338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789027262370489:2050] [node 3] ICS09 handshake done sender: [6:7577789027262370486:2122] self: [6:7577789027262370487:0] peer: [3:7577789026055183147:0] socket: 150 qp: -1
2025-11-28T14:51:36.923385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789027262370489:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:36.923419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:36.926845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789026794560994:2118] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:36.927634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789027262370485:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.927829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789027262370391:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:36.928118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789026794560994:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:36.928544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789026794560994:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:36.928778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789026794560984:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:36.928836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:36.928883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789026794560984:2109] [node 6] ICP22 created new session: [1:7577789026794561019:2109]
2025-11-28T14:51:36.928778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789027262370492:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34888
2025-11-28T14:51:36.928923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789026794561019:2109] [node 6] ICS09 handshake done sender: [1:7577789026794560994:2118] self: [1:7577789026794560992:0] peer: [6:7577789027262370491:0] socket: 135 qp: -1
2025-11-28T14:51:36.928966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789026794561019:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:36.928995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:36.929284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789027262370485:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:36.929683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789027262370391:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:36.929724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:36.929762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789027262370391:2048] [node 1] ICP22 created new session: [6:7577789027262370493:2048]
2025-11-28T14:51:36.929795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789027262370493:2048] [node 1] ICS09 handshake done sender: [6:7577789027262370485:2121] self: [6:7577789027262370491:0] peer: [1:7577789026794560992:0] socket: 147 qp: -1
2025-11-28T14:51:36.929821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789027262370493:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:36.929865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:36.930248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789027262370391:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:36.937004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:36.937244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789025777231196:2048] [node 1] ICS01 socket: 170 reason# EndOfStream
2025-11-28T14:51:36.937292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789025777231093:2048] [node 1] ICP30 unregister session Session# [2:7577789025777231196:2048] VirtualId# [2:7577789025777231194:0]
2025-11-28T14:51:36.937323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789025777231196:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.937422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.937373Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:36.937496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.937614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789026055183172:2048] [node 1] ICS07 socket disconnect 139 reason# EndOfStream
2025-11-28T14:51:36.937645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789026055183172:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.937743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.937785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789026055183172:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:36.939431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789026055183044:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8997) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:36.939487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789026055183172:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:36.939513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789026055183044:2048] [node 1] ICP30 unregister session Session# [3:7577789026055183172:2048] VirtualId# [3:7577789026055183170:0]
2025-11-28T14:51:36.939552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.939586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789026055183044:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8997) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.939597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8997) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.946331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:36.946577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789026793350361:2048] [node 1] ICS01 socket: 173 reason# EndOfStream
2025-11-28T14:51:36.946616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789026793350265:2048] [node 1] ICP30 unregister session Session# [4:7577789026793350361:2048] VirtualId# [4:7577789026793350359:0]
2025-11-28T14:51:36.946651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789026793350361:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.946770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.946818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.946879Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:36.947118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789024908582213:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-11-28T14:51:36.947168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789024908582111:2048] [node 1] ICP30 unregister session Session# [5:7577789024908582213:2048] VirtualId# [5:7577789024908582211:0]
2025-11-28T14:51:36.947196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789024908582213:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.947313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.947362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.948498Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:36.948733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789027262370493:2048] [node 1] ICS07 socket disconnect 147 reason# EndOfStream
2025-11-28T14:51:36.948766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789027262370493:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.948847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:36.948896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789027262370493:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:36.949369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789026055183044:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:36.949589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789027262370391:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8997) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:36.949676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789027262370493:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:36.949735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789027262370391:2048] [node 1] ICP30 unregister session Session# [6:7577789027262370493:2048] VirtualId# [6:7577789027262370491:0]
2025-11-28T14:51:36.949780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:36.949805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789027262370391:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8997) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.949815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8997) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.959650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789027262370391:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:36.983873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:36.984052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789026055183159:2049] [node 2] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-28T14:51:36.984086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789026055183159:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.984192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:36.984232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789026055183159:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:36.985174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789026055183045:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12988) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:36.985417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789026055183159:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:36.986129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789026055183045:2049] [node 2] ICP30 unregister session Session# [3:7577789026055183159:2049] VirtualId# [3:7577789026055183154:0]
2025-11-28T14:51:36.986204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:36.986246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789026055183045:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12988) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.986261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12988) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.986667Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:36.987843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:36.987195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789026793350348:2049] [node 2] ICS01 socket: 138 reason# EndOfStream
2025-11-28T14:51:36.987894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789026793350266:2049] [node 2] ICP30 unregister session Session# [4:7577789026793350348:2049] VirtualId# [4:7577789026793350345:0]
2025-11-28T14:51:36.987927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789026793350348:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.987990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789024908582197:2049] [node 2] ICS07 socket disconnect 174 reason# EndOfStream
2025-11-28T14:51:36.988015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789024908582197:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.988027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:36.988084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:36.988080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:36.988117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789024908582197:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:36.988960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789024908582112:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12988) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:36.989026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789024908582197:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:36.989054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789024908582112:2049] [node 2] ICP30 unregister session Session# [5:7577789024908582197:2049] VirtualId# [5:7577789024908582193:0]
2025-11-28T14:51:36.989100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:36.989134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789024908582112:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12988) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.989150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12988) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:36.988926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:36.989134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789027262370476:2049] [node 2] ICS01 socket: 186 reason# EndOfStream
2025-11-28T14:51:36.989165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789027262370392:2049] [node 2] ICP30 unregister session Session# [6:7577789027262370476:2049] VirtualId# [6:7577789027262370472:0]
2025-11-28T14:51:36.989208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789027262370476:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:36.989273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:36.989325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:36.998566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789024908582112:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:37.000455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:37.000700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789026793350365:2050] [node 3] ICS01 socket: 167 reason# EndOfStream
2025-11-28T14:51:37.000749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789026793350267:2050] [node 3] ICP30 unregister session Session# [4:7577789026793350365:2050] VirtualId# [4:7577789026793350363:0]
2025-11-28T14:51:37.000779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789026793350365:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.000879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:37.000983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:37.001876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:37.002031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789024908582209:2050] [node 3] ICS01 socket: 161 reason# EndOfStream
2025-11-28T14:51:37.002064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789024908582113:2050] [node 3] ICP30 unregister session Session# [5:7577789024908582209:2050] VirtualId# [5:7577789024908582207:0]
2025-11-28T14:51:37.002091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789024908582209:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.002160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:37.002209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:37.002752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:37.002955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789027262370489:2050] [node 3] ICS01 socket: 150 reason# EndOfStream
2025-11-28T14:51:37.002993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789027262370393:2050] [node 3] ICP30 unregister session Session# [6:7577789027262370489:2050] VirtualId# [6:7577789027262370487:0]
2025-11-28T14:51:37.003015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789027262370489:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.003081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:37.003154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:37.025557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:37.025790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789024908582199:2108] [node 4] ICS07 socket disconnect 177 reason# EndOfStream
2025-11-28T14:51:37.025822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789024908582199:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.025915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:37.025965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789024908582199:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:37.026528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:37.026641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789024908582175:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63056) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.026718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789024908582199:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.026769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789024908582175:2108] [node 4] ICP30 unregister session Session# [5:7577789024908582199:2108] VirtualId# [5:7577789024908582194:0]
2025-11-28T14:51:37.026739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789027262370475:2108] [node 4] ICS07 socket disconnect 184 reason# EndOfStream
2025-11-28T14:51:37.026772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789027262370475:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.026813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:37.026846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:37.026856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789024908582175:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63056) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.026874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63056) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.026889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789027262370475:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:37.027437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789027262370455:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63056) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.027523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789027262370475:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.027573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789027262370455:2108] [node 4] ICP30 unregister session Session# [6:7577789027262370475:2108] VirtualId# [6:7577789027262370471:0]
2025-11-28T14:51:37.027600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:37.027618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789027262370455:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63056) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.027643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63056) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.037485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789027262370455:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:37.047841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:37.048088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789027262370483:2106] [node 5] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-28T14:51:37.048119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789027262370483:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.048253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:37.048305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789027262370483:2106] [node 5] ICS15 start handshake
2025-11-28T14:51:37.049034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789027262370453:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8279) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.049092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789027262370483:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.049133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789027262370453:2106] [node 5] ICP30 unregister session Session# [6:7577789027262370483:2106] VirtualId# [6:7577789027262370481:0]
2025-11-28T14:51:37.049194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:37.049220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789027262370453:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.049235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:37.145817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789032761622600:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:37.145853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789032761622601:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:37.163701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030818374646:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.163981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030818374647:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:37.180743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789029555022411:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.180786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789029555022412:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:37.200895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030934413742:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.200934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030934413743:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:37.201758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030934413744:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:37.222952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789029460303116:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.223001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789029460303118:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:37.223613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789029460303117:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:37.241482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789032303658099:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:37.241520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789032303658100:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:37.241847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789032303658098:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.332599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789032303658160:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:37.338738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030934413805:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:37.338583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030818374709:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:37.338709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789032761622672:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:37.338602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030818374710:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:37.339133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789032761622674:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:37.339145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030818374711:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:37.339203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789029555022483:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:37.339490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789029555022484:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:37.339503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789032303658162:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:37.339543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789029460303179:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:37.339470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789032761622673:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:37.339699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789029460303180:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:37.339665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030934413806:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:37.339821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789029555022485:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:37.342936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789030818374646:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50831898ns) {Configure:104(50825499ns) {ProcessConfigured:135(115496ns) {StartInitialHandshake:157(112258ns) {DropHandshakes:470(2104ns) {DropIncomingHandshake:442(725ns) DropOutgoingHandshake:457(357ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(1251ns) {SwitchToState:179(589ns)}}}}}
2025-11-28T14:51:37.344104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789030934413742:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46859968ns) {Configure:104(46853068ns) {ProcessConfigured:135(173324ns) {StartInitialHandshake:157(172510ns) {DropHandshakes:470(943ns) {DropIncomingHandshake:442(222ns) DropOutgoingHandshake:457(208ns)} GenerateSessionVirtualId:988(292ns) PrepareNewSessionHandshake:477(649ns) {SwitchToState:179(360ns)}}}}}
2025-11-28T14:51:37.344482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789029460303116:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47168591ns) {Configure:104(47159032ns) {ProcessConfigured:135(531025ns) {StartInitialHandshake:157(529766ns) {DropHandshakes:470(1150ns) {DropIncomingHandshake:442(214ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(789ns) {SwitchToState:179(307ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.350752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030818374725:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36081
2025-11-28T14:51:37.351307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030818374726:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36081
2025-11-28T14:51:37.356536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789029460303190:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35899
2025-11-28T14:51:37.356666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030934413813:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37389
2025-11-28T14:51:37.357209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789029460303190:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.357844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030818374725:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.358182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030934413813:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.358988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789030934413743:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:37.360004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030818374727:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36081
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.360420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789029555022492:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30282
2025-11-28T14:51:37.360416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030818374726:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.360421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789032761622682:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36687
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.360782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789029555022494:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30282
2025-11-28T14:51:37.360423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030818374717:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.360978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789029555022493:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30282
2025-11-28T14:51:37.360424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789032761622681:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36687
2025-11-28T14:51:37.362482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789029555022492:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.361939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622681:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.360449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789030818374717:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:37.362706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789029555022494:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.362266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622682:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.362951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789029555022493:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.362685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789032761622687:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36687
2025-11-28T14:51:37.363788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789029555022412:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.363953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789029555022501:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30282
2025-11-28T14:51:37.363970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789030818374717:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:37.363940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789029460303196:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35899
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:37.364145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622687:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:51:37.364156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789032761622691:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36687
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.372384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789029555022503:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30282
2025-11-28T14:51:37.364337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030818374730:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36081
2025-11-28T14:51:37.372387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030934413822:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37389
2025-11-28T14:51:37.372473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030934413823:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37389
2025-11-28T14:51:37.364395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030818374732:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36081
2025-11-28T14:51:37.372414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030818374716:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.375096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030934413813:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:37.372446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789030818374716:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:37.375936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789030934413743:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.372498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030818374727:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.375987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:37.372763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789030818374709:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.376691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030934413743:2049] [node 2] ICP22 created new session: [4:7577789030934413824:2049]
2025-11-28T14:51:37.372822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:37.377779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789030934413824:2049] [node 2] ICS09 handshake done sender: [4:7577789030934413813:2113] self: [4:7577789030934413821:0] peer: [2:7577789030818374715:0] socket: 144 qp: -1
2025-11-28T14:51:37.372869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030818374709:2107] [node 4] ICP22 created new session: [2:7577789030818374733:2107]
2025-11-28T14:51:37.378039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789030934413824:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:37.373193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789030818374733:2107] [node 4] ICS09 handshake done sender: [2:7577789030818374717:2112] self: [2:7577789030818374715:0] peer: [4:7577789030934413821:0] socket: 140 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.378093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:37.373258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789030818374733:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:37.373299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:37.379923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789029460303197:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35899
2025-11-28T14:51:37.380070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789029555022507:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30282
2025-11-28T14:51:37.377447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789030818374716:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:37.377822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789030818374647:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.377863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:37.378037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789030818374709:2107] [node 4] ICP18 handshake (actor [2:7577789030818374727:2117]) is too old
2025-11-28T14:51:37.378150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030818374647:2049] [node 3] ICP22 created new session: [2:7577789030818374737:2049]
2025-11-28T14:51:37.378674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789030818374737:2049] [node 3] ICS09 handshake done sender: [2:7577789030818374716:2111] self: [2:7577789030818374714:0] peer: [3:7577789029555022504:0] socket: 138 qp: -1
2025-11-28T14:51:37.378697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789030818374737:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:37.378947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:37.380864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789029460303197:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.381160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789029555022492:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:37.381315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789029460303117:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.381784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030818374722:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.382145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789029555022412:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.381824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789030818374722:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:37.382480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:37.383906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789029555022412:2049] [node 2] ICP22 created new session: [3:7577789029555022508:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.383987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789029555022508:2049] [node 2] ICS09 handshake done sender: [3:7577789029555022492:2122] self: [3:7577789029555022504:0] peer: [2:7577789030818374714:0] socket: 151 qp: -1
2025-11-28T14:51:37.384019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789029555022508:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:37.384052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:37.384047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658176:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.384075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658177:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.384377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789030818374722:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:37.384508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789032303658177:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.384762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789032303658099:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.384668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789030818374710:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.384934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789029460303199:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35899
2025-11-28T14:51:37.384743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:37.384891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030818374710:2108] [node 5] ICP22 created new session: [2:7577789030818374741:2108]
2025-11-28T14:51:37.385175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789030818374741:2108] [node 5] ICS09 handshake done sender: [2:7577789030818374722:2113] self: [2:7577789030818374721:0] peer: [5:7577789029460303198:0] socket: 183 qp: -1
2025-11-28T14:51:37.385224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789030818374741:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:37.385272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:37.385541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789029460303197:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:37.386761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789029460303117:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61836) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789029460303197:2117] held: no
2025-11-28T14:51:37.386960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030818374724:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.386962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789029460303117:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.387017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789030818374724:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:37.387022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:37.387069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789029460303117:2049] [node 2] ICP22 created new session: [5:7577789029460303200:2049]
2025-11-28T14:51:37.387128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789029460303200:2049] [node 2] ICS09 handshake done sender: [5:7577789029460303197:2117] self: [5:7577789029460303198:0] peer: [2:7577789030818374721:0] socket: 185 qp: -1
2025-11-28T14:51:37.387339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789029460303200:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:37.387384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:37.388465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789030818374724:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.388696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030934413827:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37389
2025-11-28T14:51:37.388799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789030818374711:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.388871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:37.388830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658179:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.388921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030818374711:2109] [node 6] ICP22 created new session: [2:7577789030818374744:2109]
2025-11-28T14:51:37.388980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789030818374744:2109] [node 6] ICS09 handshake done sender: [2:7577789030818374724:2114] self: [2:7577789030818374723:0] peer: [6:7577789032303658178:0] socket: 186 qp: -1
2025-11-28T14:51:37.389027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789030818374744:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:37.389082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:37.389243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789032303658177:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:37.389437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789032303658099:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61836) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789032303658177:2116] held: no
2025-11-28T14:51:37.389616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789032303658099:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.390037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:37.390412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789032303658099:2049] [node 2] ICP22 created new session: [6:7577789032303658180:2049]
2025-11-28T14:51:37.390634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789032303658180:2049] [node 2] ICS09 handshake done sender: [6:7577789032303658177:2116] self: [6:7577789032303658178:0] peer: [2:7577789030818374723:0] socket: 187 qp: -1
2025-11-28T14:51:37.390677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789032303658180:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:37.390712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:37.391111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030934413827:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.393213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789030934413812:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.402079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789029555022501:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.405494Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789032761622679:2120] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.414051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789029555022503:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.414361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789032761622695:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36687
2025-11-28T14:51:37.414819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622695:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.416833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789032761622678:2119] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.424324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789029460303196:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.424519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789029460303116:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.424811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622678:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.425323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789032761622678:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.425516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789029460303204:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35899
2025-11-28T14:51:37.425616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789032761622672:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.425699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:37.425736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789032761622672:2108] [node 5] ICP22 created new session: [1:7577789032761622698:2108]
2025-11-28T14:51:37.425808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789029460303196:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:37.426054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789029460303116:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.426016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789032761622698:2108] [node 5] ICS09 handshake done sender: [1:7577789032761622678:2119] self: [1:7577789032761622675:0] peer: [5:7577789029460303203:0] socket: 168 qp: -1
2025-11-28T14:51:37.426116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:37.426349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789029460303116:2048] [node 1] ICP22 created new session: [5:7577789029460303205:2048]
2025-11-28T14:51:37.426053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789032761622698:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:37.426083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:37.427101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789029460303205:2048] [node 1] ICS09 handshake done sender: [5:7577789029460303196:2116] self: [5:7577789029460303203:0] peer: [1:7577789032761622675:0] socket: 169 qp: -1
2025-11-28T14:51:37.427150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789029460303205:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:37.427193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.427914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658183:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.427982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030934413829:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37389
2025-11-28T14:51:37.430586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789029555022488:2119] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.437541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789032303658183:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.437819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789032303658160:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.437953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030818374732:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.438134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789030818374647:2049] [node 3] ICP18 handshake (actor [2:7577789030818374732:2119]) is too old
2025-11-28T14:51:37.438131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789029460303194:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.438179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789029460303194:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:37.438674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789029555022411:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.439040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622679:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.439389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789032761622679:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.439441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789029460303194:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:37.439592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658185:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.439712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658186:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.439659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789029460303180:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.439714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:37.439899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789032761622601:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.440126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789032303658183:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:37.439952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:37.439902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789029460303180:2109] [node 6] ICP22 created new session: [5:7577789029460303209:2109]
2025-11-28T14:51:37.439995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789032761622601:2049] [node 3] ICP22 created new session: [1:7577789032761622702:2049]
2025-11-28T14:51:37.439984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789029460303209:2109] [node 6] ICS09 handshake done sender: [5:7577789029460303194:2115] self: [5:7577789029460303193:0] peer: [6:7577789032303658184:0] socket: 147 qp: -1
2025-11-28T14:51:37.440592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789032761622702:2049] [node 3] ICS09 handshake done sender: [1:7577789032761622679:2120] self: [1:7577789032761622676:0] peer: [3:7577789029555022511:0] socket: 166 qp: -1
2025-11-28T14:51:37.441025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789032303658160:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15244) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789032303658183:2118] held: no
2025-11-28T14:51:37.440634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789032761622702:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:37.440129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789029460303209:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:37.440893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:37.440572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:37.441591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789032303658160:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.441633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:37.441682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789032303658160:2107] [node 5] ICP22 created new session: [6:7577789032303658187:2107]
2025-11-28T14:51:37.441780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789032303658187:2107] [node 5] ICS09 handshake done sender: [6:7577789032303658183:2118] self: [6:7577789032303658184:0] peer: [5:7577789029460303193:0] socket: 194 qp: -1
2025-11-28T14:51:37.441818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789032303658187:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:37.441865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.442517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658190:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.442529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030934413830:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37389
2025-11-28T14:51:37.444143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030934413830:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.455259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789032761622680:2121] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.463873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030818374730:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.464101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789030818374646:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.464346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622680:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.464853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789032761622680:2121] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.465177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030818374747:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36081
2025-11-28T14:51:37.465246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789032761622600:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.465316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:37.465375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789032761622600:2048] [node 2] ICP22 created new session: [1:7577789032761622706:2048]
2025-11-28T14:51:37.465398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789030818374730:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:37.465428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789032761622706:2048] [node 2] ICS09 handshake done sender: [1:7577789032761622680:2121] self: [1:7577789032761622677:0] peer: [2:7577789030818374746:0] socket: 167 qp: -1
2025-11-28T14:51:37.465469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789032761622706:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:37.465534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789030818374646:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.465589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:37.465673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030818374646:2048] [node 1] ICP22 created new session: [2:7577789030818374748:2048]
2025-11-28T14:51:37.465511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:37.465805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789030818374748:2048] [node 1] ICS09 handshake done sender: [2:7577789030818374730:2118] self: [2:7577789030818374746:0] peer: [1:7577789032761622677:0] socket: 170 qp: -1
2025-11-28T14:51:37.465863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789030818374748:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:37.465909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:37.477132Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789029555022491:2121] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.484709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789029555022513:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30282
2025-11-28T14:51:37.484739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030934413822:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.484967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789030934413744:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.485246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789029555022491:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.485341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789029555022503:2128] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:37.485610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789029555022411:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.485665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:37.485721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789029555022411:2048] [node 1] ICP22 created new session: [3:7577789029555022515:2048]
2025-11-28T14:51:37.485773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789029555022515:2048] [node 1] ICS09 handshake done sender: [3:7577789029555022503:2128] self: [3:7577789029555022511:0] peer: [1:7577789032761622676:0] socket: 174 qp: -1
2025-11-28T14:51:37.485821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789029555022515:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:37.485857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.486885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789029555022491:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:37.486987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030934413833:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37389
2025-11-28T14:51:37.487172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789029555022483:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.487224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030934413822:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:37.487227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:37.487278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789029555022483:2107] [node 4] ICP22 created new session: [3:7577789029555022517:2107]
2025-11-28T14:51:37.487318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789029555022517:2107] [node 4] ICS09 handshake done sender: [3:7577789029555022491:2121] self: [3:7577789029555022490:0] peer: [4:7577789030934413832:0] socket: 173 qp: -1
2025-11-28T14:51:37.487356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789029555022517:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:37.487470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:37.487567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789030934413744:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.487671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:37.487716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030934413744:2050] [node 3] ICP22 created new session: [4:7577789030934413834:2050]
2025-11-28T14:51:37.487788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789030934413834:2050] [node 3] ICS09 handshake done sender: [4:7577789030934413822:2116] self: [4:7577789030934413832:0] peer: [3:7577789029555022490:0] socket: 177 qp: -1
2025-11-28T14:51:37.487825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789030934413834:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:37.487852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.488805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789029460303211:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35899
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.489139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789029460303212:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35899
2025-11-28T14:51:37.497713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789030934413815:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.505122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789032303658176:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.505402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789032303658162:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.505702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030934413815:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.506275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030934413815:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.506548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789030934413806:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.506627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:37.506619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658192:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.506692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030934413806:2109] [node 6] ICP22 created new session: [4:7577789030934413838:2109]
2025-11-28T14:51:37.506746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789030934413838:2109] [node 6] ICS09 handshake done sender: [4:7577789030934413815:2114] self: [4:7577789030934413814:0] peer: [6:7577789032303658191:0] socket: 157 qp: -1
2025-11-28T14:51:37.506813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789030934413838:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:37.506855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:37.506919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789032303658176:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:37.507843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789032303658162:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.507902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:37.507946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789032303658162:2108] [node 4] ICP22 created new session: [6:7577789032303658193:2108]
2025-11-28T14:51:37.507992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789032303658193:2108] [node 4] ICS09 handshake done sender: [6:7577789032303658176:2115] self: [6:7577789032303658191:0] peer: [4:7577789030934413814:0] socket: 158 qp: -1
2025-11-28T14:51:37.508021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789032303658193:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:37.508080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:37.509502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789032303658162:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:37.509796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789029555022489:2120] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.516639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622691:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.516800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789032761622601:2049] [node 3] ICP18 handshake (actor [1:7577789032761622691:2127]) is too old
2025-11-28T14:51:37.521718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789032761622685:2124] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.529769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030934413829:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.530003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789030934413742:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.530267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622685:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.530763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789032761622685:2124] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.531043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030934413841:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37389
2025-11-28T14:51:37.531077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789032761622674:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.531178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:37.531234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789032761622674:2107] [node 4] ICP22 created new session: [1:7577789032761622709:2107]
2025-11-28T14:51:37.531293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789032761622709:2107] [node 4] ICS09 handshake done sender: [1:7577789032761622685:2124] self: [1:7577789032761622683:0] peer: [4:7577789030934413840:0] socket: 159 qp: -1
2025-11-28T14:51:37.531334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789032761622709:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:37.531303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030934413829:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:37.531373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:37.531540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789030934413742:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.531595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:37.531655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030934413742:2048] [node 1] ICP22 created new session: [4:7577789030934413842:2048]
2025-11-28T14:51:37.531782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789030934413842:2048] [node 1] ICS09 handshake done sender: [4:7577789030934413829:2119] self: [4:7577789030934413840:0] peer: [1:7577789032761622683:0] socket: 195 qp: -1
2025-11-28T14:51:37.531826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789030934413842:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:37.531853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:37.533390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789029555022496:2125] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.541369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789032303658185:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.541605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789032303658100:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.541851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789029555022496:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.542300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789029555022496:2125] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.542496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789029555022484:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.542599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:37.542578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658196:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.542643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789029555022484:2109] [node 6] ICP22 created new session: [3:7577789029555022521:2109]
2025-11-28T14:51:37.542683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789029555022521:2109] [node 6] ICS09 handshake done sender: [3:7577789029555022496:2125] self: [3:7577789029555022495:0] peer: [6:7577789032303658195:0] socket: 172 qp: -1
2025-11-28T14:51:37.542716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789029555022521:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:37.542759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:37.542863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789032303658185:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:37.543013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789032303658100:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.543141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:37.543196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789032303658100:2050] [node 3] ICP22 created new session: [6:7577789032303658197:2050]
2025-11-28T14:51:37.543326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789032303658197:2050] [node 3] ICS09 handshake done sender: [6:7577789032303658185:2119] self: [6:7577789032303658195:0] peer: [3:7577789029555022495:0] socket: 196 qp: -1
2025-11-28T14:51:37.543369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789032303658197:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:37.543423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:37.544301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789032303658100:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:37.546438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789032761622686:2125] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.553951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789032303658190:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.554123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789032303658098:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.554377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789032761622686:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.554821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789032761622686:2125] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.554976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789032761622673:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.555010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789032303658200:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62304
2025-11-28T14:51:37.555100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:37.555204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789032761622673:2109] [node 6] ICP22 created new session: [1:7577789032761622712:2109]
2025-11-28T14:51:37.555262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789032303658190:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:37.555259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789032761622712:2109] [node 6] ICS09 handshake done sender: [1:7577789032761622686:2125] self: [1:7577789032761622684:0] peer: [6:7577789032303658199:0] socket: 199 qp: -1
2025-11-28T14:51:37.555381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789032303658098:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.555621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:37.555296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789032761622712:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:37.555666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789032303658098:2048] [node 1] ICP22 created new session: [6:7577789032303658201:2048]
2025-11-28T14:51:37.555353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:37.555730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789032303658201:2048] [node 1] ICS09 handshake done sender: [6:7577789032303658190:2121] self: [6:7577789032303658199:0] peer: [1:7577789032761622684:0] socket: 201 qp: -1
2025-11-28T14:51:37.555792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789032303658201:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:37.555834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:37.556528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789032303658098:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:37.558178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789029555022498:2126] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.559065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789029460303212:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.559272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789029460303118:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.559452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789029555022498:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.559835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789029555022498:2126] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:37.566118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789029555022485:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.566213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:37.566297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789029555022485:2108] [node 5] ICP22 created new session: [3:7577789029555022524:2108]
2025-11-28T14:51:37.566351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789029555022524:2108] [node 5] ICS09 handshake done sender: [3:7577789029555022498:2126] self: [3:7577789029555022497:0] peer: [5:7577789029460303213:0] socket: 204 qp: -1
2025-11-28T14:51:37.566395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789029555022524:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:37.566440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.566558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789029460303214:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35899
2025-11-28T14:51:37.566753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789029460303118:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789029460303212:2121] held: no
2025-11-28T14:51:37.567355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789029460303212:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:37.567623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789029460303118:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.567849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:37.567893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789029460303118:2050] [node 3] ICP22 created new session: [5:7577789029460303215:2050]
2025-11-28T14:51:37.567931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789029460303215:2050] [node 3] ICS09 handshake done sender: [5:7577789029460303212:2121] self: [5:7577789029460303213:0] peer: [3:7577789029555022497:0] socket: 205 qp: -1
2025-11-28T14:51:37.567954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789029460303215:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:37.567997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:37.572067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789030934413820:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:37.573032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789029460303211:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.573211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789029460303179:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.573413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030934413820:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.573871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030934413820:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.574053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789030934413805:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:37.574128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789029460303218:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35899
2025-11-28T14:51:37.574117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:37.574157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030934413805:2108] [node 5] ICP22 created new session: [4:7577789030934413845:2108]
2025-11-28T14:51:37.574253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789030934413845:2108] [node 5] ICS09 handshake done sender: [4:7577789030934413820:2115] self: [4:7577789030934413817:0] peer: [5:7577789029460303217:0] socket: 161 qp: -1
2025-11-28T14:51:37.574299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789030934413845:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:37.574354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:37.574423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789029460303211:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:37.574732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789029460303179:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789029460303211:2120] held: no
2025-11-28T14:51:37.574986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789029460303179:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:37.575029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:37.575067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789029460303179:2108] [node 4] ICP22 created new session: [5:7577789029460303219:2108]
2025-11-28T14:51:37.575129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789029460303219:2108] [node 4] ICS09 handshake done sender: [5:7577789029460303211:2120] self: [5:7577789029460303217:0] peer: [4:7577789030934413817:0] socket: 165 qp: -1
2025-11-28T14:51:37.575174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789029460303219:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:37.575212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:37.578134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:37.578285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789030818374748:2048] [node 1] ICS07 socket disconnect 170 reason# EndOfStream
2025-11-28T14:51:37.578311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789030818374748:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.578383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:37.578424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789030818374748:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:37.579401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:37.579593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789029555022515:2048] [node 1] ICS07 socket disconnect 174 reason# EndOfStream
2025-11-28T14:51:37.579623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789029555022515:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.579710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:37.579760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789029555022515:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:37.581043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789030818374646:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:37.581102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789030818374748:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:37.582523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789029555022411:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:37.582566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789029555022515:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:37.584131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789030818374646:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.584064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:37.584218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789030818374748:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.584492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789030934413842:2048] [node 1] ICS07 socket disconnect 195 reason# EndOfStream
2025-11-28T14:51:37.584292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789030818374646:2048] [node 1] ICP30 unregister session Session# [2:7577789030818374748:2048] VirtualId# [2:7577789030818374746:0]
2025-11-28T14:51:37.584517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789030934413842:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.584341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:37.584433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789030818374646:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.584585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:37.584449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.584623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789030934413842:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:37.584769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789029555022411:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.584886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789029555022515:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.584925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789029555022411:2048] [node 1] ICP30 unregister session Session# [3:7577789029555022515:2048] VirtualId# [3:7577789029555022511:0]
2025-11-28T14:51:37.584972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:37.585000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789029555022411:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.585019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.585550Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789030934413742:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.585620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789030934413842:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.585656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789030934413742:2048] [node 1] ICP30 unregister session Session# [4:7577789030934413842:2048] VirtualId# [4:7577789030934413840:0]
2025-11-28T14:51:37.585609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:37.585704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:37.585730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789030934413742:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.585741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.585760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789029460303205:2048] [node 1] ICS07 socket disconnect 169 reason# EndOfStream
2025-11-28T14:51:37.585784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789029460303205:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.585842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:37.585871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789029460303205:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:37.586446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789029460303116:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.586480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789029460303205:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.586533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789029460303116:2048] [node 1] ICP30 unregister session Session# [5:7577789029460303205:2048] VirtualId# [5:7577789029460303203:0]
2025-11-28T14:51:37.586568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:37.586589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789029460303116:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.586603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.586687Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:37.586869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789032303658201:2048] [node 1] ICS07 socket disconnect 201 reason# EndOfStream
2025-11-28T14:51:37.586911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789032303658201:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.586981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:37.587032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789032303658201:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:37.587536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789032303658098:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.587599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789032303658201:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.587632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789032303658098:2048] [node 1] ICP30 unregister session Session# [6:7577789032303658201:2048] VirtualId# [6:7577789032303658199:0]
2025-11-28T14:51:37.587663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:37.587679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789032303658098:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.587687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20367) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.594543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789029555022411:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:37.595537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789030934413742:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:37.596543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789029460303116:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:37.597584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789032303658098:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:37.618724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:37.618941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789029555022508:2049] [node 2] ICS07 socket disconnect 151 reason# EndOfStream
2025-11-28T14:51:37.618980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789029555022508:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.619090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:37.619144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789029555022508:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:37.619988Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789029555022412:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61836) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.620120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789029555022508:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.620178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789029555022412:2049] [node 2] ICP30 unregister session Session# [3:7577789029555022508:2049] VirtualId# [3:7577789029555022504:0]
2025-11-28T14:51:37.620251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:37.620286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789029555022412:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.620308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.621202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:37.621394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789030934413824:2049] [node 2] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-28T14:51:37.621424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789030934413824:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.621509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:37.621561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789030934413824:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:37.621953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:37.622151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789029460303200:2049] [node 2] ICS01 socket: 185 reason# EndOfStream
2025-11-28T14:51:37.622200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789029460303117:2049] [node 2] ICP30 unregister session Session# [5:7577789029460303200:2049] VirtualId# [5:7577789029460303198:0]
2025-11-28T14:51:37.622232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789029460303200:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.622224Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789030934413743:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61836) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.622262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789030934413824:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.622397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:37.622296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789030934413743:2049] [node 2] ICP30 unregister session Session# [4:7577789030934413824:2049] VirtualId# [4:7577789030934413821:0]
2025-11-28T14:51:37.622492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:37.622358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:37.622385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789030934413743:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.622398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.623010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:37.623205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789032303658180:2049] [node 2] ICS01 socket: 187 reason# EndOfStream
2025-11-28T14:51:37.623254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789032303658099:2049] [node 2] ICP30 unregister session Session# [6:7577789032303658180:2049] VirtualId# [6:7577789032303658178:0]
2025-11-28T14:51:37.623280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789032303658180:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.623345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:37.623401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:37.632484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789030934413743:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:37.632888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:37.633103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789030934413834:2050] [node 3] ICS01 socket: 177 reason# EndOfStream
2025-11-28T14:51:37.633154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789030934413744:2050] [node 3] ICP30 unregister session Session# [4:7577789030934413834:2050] VirtualId# [4:7577789030934413832:0]
2025-11-28T14:51:37.633199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789030934413834:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.633300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:37.633354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:37.633861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:37.634044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789029460303215:2050] [node 3] ICS01 socket: 205 reason# EndOfStream
2025-11-28T14:51:37.634087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789029460303118:2050] [node 3] ICP30 unregister session Session# [5:7577789029460303215:2050] VirtualId# [5:7577789029460303213:0]
2025-11-28T14:51:37.634115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789029460303215:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.634191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:37.634229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:37.634699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:37.634968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789032303658197:2050] [node 3] ICS07 socket disconnect 196 reason# EndOfStream
2025-11-28T14:51:37.635001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789032303658197:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.635070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:37.635120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789032303658197:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:37.635875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789032303658100:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19062) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.635932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789032303658197:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.635964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789032303658100:2050] [node 3] ICP30 unregister session Session# [6:7577789032303658197:2050] VirtualId# [6:7577789032303658195:0]
2025-11-28T14:51:37.636006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:37.636034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789032303658100:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19062) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.636047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19062) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.645749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789032303658100:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:37.652734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:37.652976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789029460303219:2108] [node 4] ICS01 socket: 165 reason# EndOfStream
2025-11-28T14:51:37.653042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789029460303179:2108] [node 4] ICP30 unregister session Session# [5:7577789029460303219:2108] VirtualId# [5:7577789029460303217:0]
2025-11-28T14:51:37.653082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789029460303219:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.653178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:37.653233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:37.653609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:37.653820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789032303658193:2108] [node 4] ICS07 socket disconnect 158 reason# EndOfStream
2025-11-28T14:51:37.653848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789032303658193:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.653914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:37.653958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789032303658193:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:37.654608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789032303658162:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3474) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:37.654671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789032303658193:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:37.654727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789032303658162:2108] [node 4] ICP30 unregister session Session# [6:7577789032303658193:2108] VirtualId# [6:7577789032303658191:0]
2025-11-28T14:51:37.654771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:37.654796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789032303658162:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.654808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:37.664194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789032303658162:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:37.675414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:37.675641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789032303658187:2107] [node 5] ICS01 socket: 194 reason# EndOfStream
2025-11-28T14:51:37.675688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789032303658160:2107] [node 5] ICP30 unregister session Session# [6:7577789032303658187:2107] VirtualId# [6:7577789032303658184:0]
2025-11-28T14:51:37.675728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789032303658187:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:37.675836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:37.675892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:37.750749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789029968738652:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:37.750754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789029968738651:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:37.772442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030481132277:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.772522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030481132278:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:37.793208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789030756524213:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.793248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789030756524214:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:37.812963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030241326124:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.812992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030241326126:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:37.813300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030241326125:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:37.833261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789032323462283:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.833280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789032323462284:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:37.833313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789032323462285:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:37.852071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789029035092887:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:37.852081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789029035092888:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:37.852172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789029035092889:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:37.955859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789029035092949:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:37.961571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789032323462347:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:37.961446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030481132340:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:37.964605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789029968738725:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:37.969165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789029035092951:2109] [node 4] ICP01 ready to work
2025-11-28T14:51:37.968751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789032323462348:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:37.968737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789030756524276:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:37.968776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789029968738724:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:37.969172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789030756524278:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:37.969290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789030756524277:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:37.969019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030241326188:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:37.969230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789030241326189:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:37.968794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030481132341:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:37.969305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789029968738726:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:37.969637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789030481132342:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:37.976460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789029968738652:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(59025031ns) {Configure:104(58997048ns) {ProcessConfigured:135(372757ns) {StartInitialHandshake:157(370089ns) {DropHandshakes:470(1324ns) {DropIncomingHandshake:442(412ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:477(1779ns) {SwitchToState:179(849ns)}}}}}
2025-11-28T14:51:37.980599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789030481132278:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(70799914ns) {Configure:104(70793884ns) {ProcessConfigured:135(783464ns) {StartInitialHandshake:157(781275ns) {DropHandshakes:470(589ns) {DropIncomingHandshake:442(133ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(178ns) PrepareNewSessionHandshake:477(1941ns) {SwitchToState:179(1053ns)}}}}}
2025-11-28T14:51:37.980654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789030241326124:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17655539ns) {Configure:104(17650710ns) {ProcessConfigured:135(419766ns) {StartInitialHandshake:157(417256ns) {DropHandshakes:470(1208ns) {DropIncomingHandshake:442(237ns) DropOutgoingHandshake:457(427ns)} GenerateSessionVirtualId:988(456ns) PrepareNewSessionHandshake:477(1333ns) {SwitchToState:179(594ns)}}}}}
2025-11-28T14:51:37.981302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789029035092888:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20996932ns) {Configure:104(20991968ns) {ProcessConfigured:135(203171ns) {StartInitialHandshake:157(201793ns) {DropHandshakes:470(1027ns) {DropIncomingHandshake:442(296ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:477(1365ns) {SwitchToState:179(665ns)}}}}}
2025-11-28T14:51:37.976944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789029968738651:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(74434368ns) {Configure:104(74429828ns) {ProcessConfigured:135(962217ns) {StartInitialHandshake:157(795624ns) {DropHandshakes:470(1177ns) {DropIncomingHandshake:442(336ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(738ns) PrepareNewSessionHandshake:477(1548ns) {SwitchToState:179(794ns)}}}}}
2025-11-28T14:51:37.981368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789030241326125:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48975093ns) {Configure:104(48969716ns) {ProcessConfigured:135(1109917ns) {StartInitialHandshake:157(1107658ns) {DropHandshakes:470(1148ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(384ns) PrepareNewSessionHandshake:477(1104ns) {SwitchToState:179(302ns)}}}}}
2025-11-28T14:51:37.981853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789029035092887:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(55063858ns) {Configure:104(55057913ns) {ProcessConfigured:135(3668058ns) {StartInitialHandshake:157(3665805ns) {DropHandshakes:470(469ns) {DropIncomingHandshake:442(99ns) DropOutgoingHandshake:457(40ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(1998ns) {SwitchToState:179(303ns)}}}}}
2025-11-28T14:51:37.982385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789030481132277:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(75144642ns) {Configure:104(75139040ns) {ProcessConfigured:135(531912ns) {StartInitialHandshake:157(530190ns) {DropHandshakes:470(1217ns) {DropIncomingHandshake:442(440ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(389ns) PrepareNewSessionHandshake:477(1575ns) {SwitchToState:179(224ns)}}}}}
2025-11-28T14:51:37.982731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789032323462284:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22125203ns) {Configure:104(22122213ns) {ProcessConfigured:135(918640ns) {StartInitialHandshake:157(737672ns) {DropHandshakes:470(1947ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(209ns)} GenerateSessionVirtualId:988(1590ns) PrepareNewSessionHandshake:477(1817ns) {SwitchToState:179(1059ns)}}}}}
2025-11-28T14:51:37.983182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789032323462283:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53120740ns) {Configure:104(53115122ns) {ProcessConfigured:135(1122352ns) {StartInitialHandshake:157(1120878ns) {DropHandshakes:470(759ns) {DropIncomingHandshake:442(183ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(1675ns) {SwitchToState:179(164ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.985139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789030756524213:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28161399ns) {Configure:104(28152845ns) {ProcessConfigured:135(800645ns) {StartInitialHandshake:157(798726ns) {DropHandshakes:470(1134ns) {DropIncomingHandshake:442(271ns) DropOutgoingHandshake:457(169ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:477(1264ns) {SwitchToState:179(536ns)}}}}}
2025-11-28T14:51:37.985790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030241326198:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.989350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789029968738740:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63604
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.995802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030241326202:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:51:37.996228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030481132354:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25590
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.995881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030241326198:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.996360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030481132356:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25590
2025-11-28T14:51:37.996224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789030241326124:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.996053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789030756524287:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52304
2025-11-28T14:51:37.996073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789029968738740:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.996326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789032323462357:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39979
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.996473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789030756524288:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52304
2025-11-28T14:51:37.996738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030481132357:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25590
2025-11-28T14:51:37.997143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789029035092965:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:37.996749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789032323462362:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39979
2025-11-28T14:51:37.996702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789029968738728:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.996903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789030756524291:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52304
2025-11-28T14:51:37.996962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030481132354:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.996731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789029968738728:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:37.998167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030241326202:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.996792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789032323462361:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39979
2025-11-28T14:51:37.997442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789030756524287:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.998241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789030756524292:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52304
2025-11-28T14:51:37.999718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789029968738733:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.999566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789030241326125:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.998482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030481132356:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:37.997471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789032323462357:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:51:37.998426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789030756524213:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.999747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789029968738733:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.002569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030481132350:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:37.998072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789032323462283:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:37.998126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789032323462364:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39979
2025-11-28T14:51:37.998485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789030756524291:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.002601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789030481132350:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.002474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789029968738731:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.014041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789032323462366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39979
2025-11-28T14:51:37.999533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789030756524288:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.015851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789030241326206:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:51:38.010969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030481132344:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.002505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789029968738731:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.011022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789030481132344:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.002635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789029968738728:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.001837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789030756524214:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.013561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789030481132359:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25590
2025-11-28T14:51:38.019155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789029968738731:2121] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.016424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789030481132350:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:38.019834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789029968738733:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:38.020445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789034536293504:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:51:38.020008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789030481132344:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:38.019840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789029968738724:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.021294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789034536293505:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:51:38.020093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789030481132340:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.020036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:38.020269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789029968738652:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.020243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:38.022928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789030241326124:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789030241326198:2115] held: no
2025-11-28T14:51:38.020298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:38.020698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030481132340:2107] [node 4] ICP22 created new session: [2:7577789034776099660:2107]
2025-11-28T14:51:38.023278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789035051491595:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52304
2025-11-28T14:51:38.020595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789029968738724:2107] [node 4] ICP22 created new session: [1:7577789034263706041:2107]
2025-11-28T14:51:38.023587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030241326198:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:38.023456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789036618429664:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39979
2025-11-28T14:51:38.021023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789034776099660:2107] [node 4] ICS09 handshake done sender: [2:7577789030481132350:2113] self: [2:7577789030481132346:0] peer: [4:7577789030241326207:0] socket: 164 qp: -1
2025-11-28T14:51:38.023987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030241326202:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:38.020731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789034263706041:2107] [node 4] ICS09 handshake done sender: [1:7577789029968738728:2120] self: [1:7577789029968738727:0] peer: [4:7577789030241326205:0] socket: 144 qp: -1
2025-11-28T14:51:38.021328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789034776099660:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:38.025137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789032323462357:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:38.020988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789029968738652:2049] [node 3] ICP22 created new session: [1:7577789034263706042:2049]
2025-11-28T14:51:38.024866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789030241326125:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63075) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789030241326202:2116] held: no
2025-11-28T14:51:38.022717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.020858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789034263706041:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:38.025137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789030241326124:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.022730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789030481132278:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.021333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:38.025443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:38.027261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789035051491596:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52304
2025-11-28T14:51:38.027486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789032323462283:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.023319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:38.021908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789034263706042:2049] [node 3] ICS09 handshake done sender: [1:7577789029968738731:2121] self: [1:7577789029968738729:0] peer: [3:7577789030756524296:0] socket: 145 qp: -1
2025-11-28T14:51:38.027586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789033330060262:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:38.027537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:38.025438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789030241326125:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.023519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030481132278:2049] [node 3] ICP22 created new session: [2:7577789034776099662:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.026216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:38.027997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789032323462283:2048] [node 1] ICP22 created new session: [5:7577789036618429665:2048]
2025-11-28T14:51:38.022708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789034263706042:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:38.023782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789034776099662:2049] [node 3] ICS09 handshake done sender: [2:7577789030481132344:2110] self: [2:7577789030481132343:0] peer: [3:7577789035051491593:0] socket: 160 qp: -1
2025-11-28T14:51:38.026484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030241326124:2048] [node 1] ICP22 created new session: [4:7577789034536293506:2048]
2025-11-28T14:51:38.028753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789036618429665:2048] [node 1] ICS09 handshake done sender: [5:7577789032323462357:2115] self: [5:7577789032323462367:0] peer: [1:7577789029968738732:0] socket: 156 qp: -1
2025-11-28T14:51:38.026516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030241326125:2049] [node 2] ICP22 created new session: [4:7577789034536293507:2049]
2025-11-28T14:51:38.024049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789034776099662:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:38.022742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:38.029357Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789030756524287:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:38.028933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789036618429665:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:38.024627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:38.026529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789034536293506:2048] [node 1] ICS09 handshake done sender: [4:7577789030241326198:2115] self: [4:7577789030241326205:0] peer: [1:7577789029968738727:0] socket: 147 qp: -1
2025-11-28T14:51:38.022973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789029968738725:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.026659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789034536293506:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:38.029524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.023012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:38.026822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789034536293507:2049] [node 2] ICS09 handshake done sender: [4:7577789030241326202:2116] self: [4:7577789030241326207:0] peer: [2:7577789030481132346:0] socket: 165 qp: -1
2025-11-28T14:51:38.029588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789034776099664:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25590
2025-11-28T14:51:38.026859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789034536293507:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:38.026946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:38.023204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789029968738725:2109] [node 5] ICP22 created new session: [1:7577789034263706045:2109]
2025-11-28T14:51:38.030647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789030756524213:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.027051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:38.023707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789034263706045:2109] [node 5] ICS09 handshake done sender: [1:7577789029968738733:2122] self: [1:7577789029968738732:0] peer: [5:7577789032323462367:0] socket: 146 qp: -1
2025-11-28T14:51:38.024209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789034263706045:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:38.031092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.024244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:38.031212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789030756524213:2048] [node 1] ICP22 created new session: [3:7577789035051491598:2048]
2025-11-28T14:51:38.030751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789034263706049:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63604
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.031253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789035051491598:2048] [node 1] ICS09 handshake done sender: [3:7577789030756524287:2114] self: [3:7577789030756524296:0] peer: [1:7577789029968738729:0] socket: 148 qp: -1
2025-11-28T14:51:38.031318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789033330060263:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:38.031290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789035051491598:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:38.031090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789034776099664:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.031764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789033330060264:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:38.031980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789033330060263:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.031970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789029968738734:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.031338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:38.031282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789030481132277:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.033511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789029035092888:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.032006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789029968738734:2123] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.033777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789029968738734:2123] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.034335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789029968738651:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.034296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030481132358:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.034335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789030481132358:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.034828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789033330060264:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.034440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:38.035684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789034776099668:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25590
2025-11-28T14:51:38.034539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789029968738651:2048] [node 2] ICP22 created new session: [1:7577789034263706051:2048]
2025-11-28T14:51:38.034768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789034263706051:2048] [node 2] ICS09 handshake done sender: [1:7577789029968738734:2123] self: [1:7577789029968738730:0] peer: [2:7577789034776099667:0] socket: 154 qp: -1
2025-11-28T14:51:38.035702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789034263706051:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:38.035966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.036073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789029035092951:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.036165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789034776099664:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:38.036394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789030756524291:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:38.036605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789034536293512:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:51:38.037157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789030481132277:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.036616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030241326204:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.036642Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789030241326204:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.037383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.037472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789030481132358:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:38.037690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030241326204:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.037581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030481132277:2048] [node 1] ICP22 created new session: [2:7577789034776099670:2048]
2025-11-28T14:51:38.038163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789030241326189:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.038276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.038358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789033330060267:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:38.037753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789034776099670:2048] [node 1] ICS09 handshake done sender: [2:7577789034776099664:2119] self: [2:7577789034776099667:0] peer: [1:7577789029968738730:0] socket: 196 qp: -1
2025-11-28T14:51:38.038614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030241326189:2110] [node 6] ICP22 created new session: [4:7577789034536293514:2110]
2025-11-28T14:51:38.038637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789033330060268:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:38.037797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789034776099670:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:38.038731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789034536293514:2110] [node 6] ICS09 handshake done sender: [4:7577789030241326204:2117] self: [4:7577789030241326203:0] peer: [6:7577789033330060266:0] socket: 201 qp: -1
2025-11-28T14:51:38.038062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:38.038707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789030481132342:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.038796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789034536293514:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:38.038809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.038841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.038851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030481132342:2109] [node 6] ICP22 created new session: [2:7577789034776099672:2109]
2025-11-28T14:51:38.038896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789033330060264:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:38.038894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789034776099672:2109] [node 6] ICS09 handshake done sender: [2:7577789030481132358:2117] self: [2:7577789030481132355:0] peer: [6:7577789033330060265:0] socket: 199 qp: -1
2025-11-28T14:51:38.038913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789030756524214:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.038929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789034776099672:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:38.039042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.039022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:38.039757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789033330060263:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:38.039125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789030756524214:2049] [node 2] ICP22 created new session: [3:7577789035051491600:2049]
2025-11-28T14:51:38.039499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789035051491600:2049] [node 2] ICS09 handshake done sender: [3:7577789030756524291:2117] self: [3:7577789035051491593:0] peer: [2:7577789030481132343:0] socket: 176 qp: -1
2025-11-28T14:51:38.039531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789035051491600:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:38.039618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:38.040087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789029035092951:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.040145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:38.040251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789029035092888:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.040301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:38.040387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789029035092888:2049] [node 2] ICP22 created new session: [6:7577789033330060270:2049]
2025-11-28T14:51:38.040399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789029035092951:2109] [node 4] ICP22 created new session: [6:7577789033330060269:2109]
2025-11-28T14:51:38.040450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789033330060269:2109] [node 4] ICS09 handshake done sender: [6:7577789033330060264:2118] self: [6:7577789033330060266:0] peer: [4:7577789030241326203:0] socket: 203 qp: -1
2025-11-28T14:51:38.040458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789033330060270:2049] [node 2] ICS09 handshake done sender: [6:7577789033330060263:2117] self: [6:7577789033330060265:0] peer: [2:7577789030481132355:0] socket: 200 qp: -1
No RDMA context found for GID: 2025-11-28T14:51:38.040492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789033330060269:2109] [node 4] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:51:38.040494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789033330060270:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:38.040529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:38.040576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:38.041777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789035051491597:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52304
2025-11-28T14:51:38.042734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789035051491597:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.043357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789033330060274:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:38.043475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789034263706053:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63604
2025-11-28T14:51:38.044901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789034263706053:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.045667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789030241326194:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.056170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789029035092956:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.069962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789030481132348:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.081337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789032323462352:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.082214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789032323462362:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.082339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789032323462361:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.082443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789032323462364:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.082621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789032323462348:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.082731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789032323462284:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.083020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030241326194:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.083524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030241326194:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:38.083529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030481132348:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.092439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789030481132348:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:38.092710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789030481132341:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.092751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:38.092792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789030481132341:2108] [node 5] ICP22 created new session: [2:7577789034776099675:2108]
2025-11-28T14:51:38.092827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789034776099675:2108] [node 5] ICS09 handshake done sender: [2:7577789030481132348:2111] self: [2:7577789030481132345:0] peer: [5:7577789036618429670:0] socket: 158 qp: -1
2025-11-28T14:51:38.092853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789034776099675:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:38.089420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789030241326188:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.093104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:38.091363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:38.091428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030241326188:2108] [node 5] ICP22 created new session: [4:7577789034536293517:2108]
2025-11-28T14:51:38.091479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789034536293517:2108] [node 5] ICS09 handshake done sender: [4:7577789030241326194:2112] self: [4:7577789030241326191:0] peer: [5:7577789036618429668:0] socket: 157 qp: -1
2025-11-28T14:51:38.091517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789034536293517:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:38.091561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.093950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789036618429671:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39979
2025-11-28T14:51:38.094515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789032323462362:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.094778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789036618429672:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39979
2025-11-28T14:51:38.095141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789032323462348:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.095220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:38.095261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789032323462348:2108] [node 4] ICP22 created new session: [5:7577789036618429673:2108]
2025-11-28T14:51:38.095300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789036618429673:2108] [node 4] ICS09 handshake done sender: [5:7577789032323462362:2117] self: [5:7577789036618429668:0] peer: [4:7577789030241326191:0] socket: 168 qp: -1
2025-11-28T14:51:38.095327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789036618429673:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:38.095373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:38.096098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789032323462364:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:38.096354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789032323462284:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.096407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:38.096453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789032323462284:2049] [node 2] ICP22 created new session: [5:7577789036618429675:2049]
2025-11-28T14:51:38.096505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789036618429675:2049] [node 2] ICS09 handshake done sender: [5:7577789032323462364:2118] self: [5:7577789036618429670:0] peer: [2:7577789030481132345:0] socket: 171 qp: -1
2025-11-28T14:51:38.096533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789036618429675:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:38.096558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:38.110838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789030756524283:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.120273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789030756524292:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.120290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030481132359:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.120458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789030481132278:2049] [node 3] ICP18 handshake (actor [2:7577789030481132359:2118]) is too old
2025-11-28T14:51:38.131028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789032323462355:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.139006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789029035092965:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.139259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789029035092949:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.139454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789032323462355:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.139908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789032323462355:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:38.140096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789032323462347:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.140232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789033330060276:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:38.140266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.140327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789032323462347:2110] [node 6] ICP22 created new session: [5:7577789036618429678:2110]
2025-11-28T14:51:38.140393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789036618429678:2110] [node 6] ICS09 handshake done sender: [5:7577789032323462355:2113] self: [5:7577789032323462350:0] peer: [6:7577789033330060275:0] socket: 174 qp: -1
2025-11-28T14:51:38.140446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789036618429678:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:38.140486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789029035092965:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:38.140489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.140803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789029035092949:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.140940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:38.140986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789029035092949:2106] [node 5] ICP22 created new session: [6:7577789033330060277:2106]
2025-11-28T14:51:38.141033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789033330060277:2106] [node 5] ICS09 handshake done sender: [6:7577789029035092965:2115] self: [6:7577789033330060275:0] peer: [5:7577789032323462350:0] socket: 175 qp: -1
2025-11-28T14:51:38.141059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789033330060277:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:38.141092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:38.141564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789029035092949:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:38.150156Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789030756524281:2110] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.150902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789032323462366:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.151072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789032323462285:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.151329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789030756524281:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.151723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789030756524281:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:38.157549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789030756524277:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.157957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: 2025-11-28T14:51:38.158028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789030756524277:2108] [node 5] ICP22 created new session: [3:7577789035051491606:2108]
::ffff:127.1.0.0
2025-11-28T14:51:38.158089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789035051491606:2108] [node 5] ICS09 handshake done sender: [3:7577789030756524281:2110] self: [3:7577789030756524279:0] peer: [5:7577789036618429680:0] socket: 182 qp: -1
2025-11-28T14:51:38.158132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789035051491606:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:38.158099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789036618429681:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39979
2025-11-28T14:51:38.158194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:38.158310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789032323462366:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:38.158862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789032323462285:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789032323462366:2120] held: no
2025-11-28T14:51:38.158963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789032323462285:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.159019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:38.159054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789032323462285:2050] [node 3] ICP22 created new session: [5:7577789036618429682:2050]
2025-11-28T14:51:38.159083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789036618429682:2050] [node 3] ICS09 handshake done sender: [5:7577789032323462366:2120] self: [5:7577789036618429680:0] peer: [3:7577789030756524279:0] socket: 184 qp: -1
2025-11-28T14:51:38.159112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789036618429682:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:38.159150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:38.163198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789032323462354:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.170519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789030481132357:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.170769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789030481132341:2108] [node 5] ICP18 handshake (actor [2:7577789030481132357:2116]) is too old
2025-11-28T14:51:38.173442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789030756524284:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.180569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789030241326206:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.180772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789030241326126:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.181015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789030756524284:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.181489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789030756524284:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.181681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789030756524276:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.181782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:38.181781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789034536293520:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:51:38.181836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789030756524276:2107] [node 4] ICP22 created new session: [3:7577789035051491609:2107]
2025-11-28T14:51:38.181895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789035051491609:2107] [node 4] ICS09 handshake done sender: [3:7577789030756524284:2112] self: [3:7577789030756524282:0] peer: [4:7577789034536293519:0] socket: 178 qp: -1
2025-11-28T14:51:38.181946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789035051491609:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:38.181996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:38.182077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789030241326206:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:38.182303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789030241326126:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789030241326206:2118] held: no
2025-11-28T14:51:38.182879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789030241326126:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.182926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:38.182961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789030241326126:2050] [node 3] ICP22 created new session: [4:7577789034536293521:2050]
2025-11-28T14:51:38.182994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789034536293521:2050] [node 3] ICS09 handshake done sender: [4:7577789030241326206:2118] self: [4:7577789034536293519:0] peer: [3:7577789030756524282:0] socket: 179 qp: -1
2025-11-28T14:51:38.183022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789034536293521:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:38.183048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:38.184646Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789029968738739:2124] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.197227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789033330060262:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.197418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789029035092887:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.197692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789029968738739:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.198132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789029968738739:2124] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.198329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789029968738726:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.198311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789033330060280:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:38.198431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.198519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789029968738726:2110] [node 6] ICP22 created new session: [1:7577789034263706056:2110]
2025-11-28T14:51:38.198562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789034263706056:2110] [node 6] ICS09 handshake done sender: [1:7577789029968738739:2124] self: [1:7577789029968738738:0] peer: [6:7577789033330060279:0] socket: 159 qp: -1
2025-11-28T14:51:38.198592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789034263706056:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:38.198622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.198659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789033330060262:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:38.199479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789029035092887:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789033330060262:2116] held: no
2025-11-28T14:51:38.199609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789029035092887:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.199677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:38.199772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789029035092887:2048] [node 1] ICP22 created new session: [6:7577789033330060281:2048]
2025-11-28T14:51:38.199824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789033330060281:2048] [node 1] ICS09 handshake done sender: [6:7577789033330060262:2116] self: [6:7577789033330060279:0] peer: [1:7577789029968738738:0] socket: 163 qp: -1
2025-11-28T14:51:38.199860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789033330060281:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:38.199919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:38.201782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789030481132349:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.214196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789034263706049:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.214363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789032323462365:2119] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.214459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789029968738651:2048] [node 2] ICP18 handshake (actor [1:7577789034263706049:2126]) is too old
2025-11-28T14:51:38.221285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789034536293512:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.221506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789030241326188:2108] [node 5] ICP18 handshake (actor [4:7577789034536293512:2121]) is too old
2025-11-28T14:51:38.224724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789030756524290:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.225553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789033330060274:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.225777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789029035092889:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.226025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789030756524290:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.226524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789030756524290:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.226760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789030756524278:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.226838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.226881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789030756524278:2109] [node 6] ICP22 created new session: [3:7577789035051491612:2109]
2025-11-28T14:51:38.226931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789035051491612:2109] [node 6] ICS09 handshake done sender: [3:7577789030756524290:2116] self: [3:7577789030756524289:0] peer: [6:7577789033330060283:0] socket: 213 qp: -1
2025-11-28T14:51:38.226965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789035051491612:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:38.227008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.226872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789033330060284:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27992
2025-11-28T14:51:38.227210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789033330060274:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:38.227308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789029035092889:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789033330060274:2121] held: no
2025-11-28T14:51:38.227526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789029035092889:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.227560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:38.227819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789029035092889:2050] [node 3] ICP22 created new session: [6:7577789033330060285:2050]
2025-11-28T14:51:38.227893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789033330060285:2050] [node 3] ICS09 handshake done sender: [6:7577789033330060274:2121] self: [6:7577789033330060283:0] peer: [3:7577789030756524289:0] socket: 214 qp: -1
2025-11-28T14:51:38.227917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789033330060285:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:38.227945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:38.252379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:38.252633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789034776099670:2048] [node 1] ICS07 socket disconnect 196 reason# EndOfStream
2025-11-28T14:51:38.252685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789034776099670:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.252799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:38.252867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789034776099670:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:38.253609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:38.253748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789035051491598:2048] [node 1] ICS07 socket disconnect 148 reason# EndOfStream
2025-11-28T14:51:38.253776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789035051491598:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.253848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:38.253893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789035051491598:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:38.255095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789030481132277:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:38.255135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789034776099670:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:38.256130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789030756524213:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:38.256175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789035051491598:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:38.257075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789030481132277:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:38.257329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789034776099670:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:38.257362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789030481132277:2048] [node 1] ICP30 unregister session Session# [2:7577789034776099670:2048] VirtualId# [2:7577789034776099667:0]
2025-11-28T14:51:38.257414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:38.257467Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789030481132277:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.257483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.257895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789030756524213:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:38.257954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789035051491598:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:38.257985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789030756524213:2048] [node 1] ICP30 unregister session Session# [3:7577789035051491598:2048] VirtualId# [3:7577789030756524296:0]
2025-11-28T14:51:38.258109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:38.258036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:38.258067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789030756524213:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.258079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.258362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789034536293506:2048] [node 1] ICS01 socket: 147 reason# EndOfStream
2025-11-28T14:51:38.258412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789030241326124:2048] [node 1] ICP30 unregister session Session# [4:7577789034536293506:2048] VirtualId# [4:7577789030241326205:0]
2025-11-28T14:51:38.258451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789034536293506:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.258520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:38.258573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:38.259105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:38.259349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789036618429665:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-28T14:51:38.259412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789032323462283:2048] [node 1] ICP30 unregister session Session# [5:7577789036618429665:2048] VirtualId# [5:7577789032323462367:0]
2025-11-28T14:51:38.259448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789036618429665:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.259541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:38.259600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:38.259732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:38.259950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789033330060281:2048] [node 1] ICS01 socket: 163 reason# EndOfStream
2025-11-28T14:51:38.259991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789029035092887:2048] [node 1] ICP30 unregister session Session# [6:7577789033330060281:2048] VirtualId# [6:7577789033330060279:0]
2025-11-28T14:51:38.260017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789033330060281:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.260104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:38.260165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:38.268321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789030756524213:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:38.275988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:38.276248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789035051491600:2049] [node 2] ICS01 socket: 176 reason# EndOfStream
2025-11-28T14:51:38.276299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789030756524214:2049] [node 2] ICP30 unregister session Session# [3:7577789035051491600:2049] VirtualId# [3:7577789035051491593:0]
2025-11-28T14:51:38.276330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789035051491600:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.276451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:38.276538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:38.278775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:38.278957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789034536293507:2049] [node 2] ICS01 socket: 165 reason# EndOfStream
2025-11-28T14:51:38.279002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789030241326125:2049] [node 2] ICP30 unregister session Session# [4:7577789034536293507:2049] VirtualId# [4:7577789030241326207:0]
2025-11-28T14:51:38.279041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789034536293507:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.279127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:38.279173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:38.279545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:38.279789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789036618429675:2049] [node 2] ICS07 socket disconnect 171 reason# EndOfStream
2025-11-28T14:51:38.279821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789036618429675:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.279937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:38.279997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789036618429675:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:38.280268Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:38.280469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789033330060270:2049] [node 2] ICS07 socket disconnect 200 reason# EndOfStream
2025-11-28T14:51:38.280508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789033330060270:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.280558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:38.280589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789033330060270:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:38.280663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789032323462284:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63075) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:38.280806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789036618429675:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:38.280878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789032323462284:2049] [node 2] ICP30 unregister session Session# [5:7577789036618429675:2049] VirtualId# [5:7577789036618429670:0]
2025-11-28T14:51:38.280924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:38.280949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789032323462284:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63075) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.280959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63075) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.281402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789029035092888:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63075) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:38.281489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789033330060270:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:38.281539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789029035092888:2049] [node 2] ICP30 unregister session Session# [6:7577789033330060270:2049] VirtualId# [6:7577789033330060265:0]
2025-11-28T14:51:38.281577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:38.281602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789029035092888:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63075) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.281624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63075) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.290858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789032323462284:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:38.291546Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:38.291797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789034536293521:2050] [node 3] ICS01 socket: 179 reason# EndOfStream
2025-11-28T14:51:38.291841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789030241326126:2050] [node 3] ICP30 unregister session Session# [4:7577789034536293521:2050] VirtualId# [4:7577789034536293519:0]
2025-11-28T14:51:38.291871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789034536293521:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.291987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:38.292077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:38.292542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:38.292747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789036618429682:2050] [node 3] ICS01 socket: 184 reason# EndOfStream
2025-11-28T14:51:38.292775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789032323462285:2050] [node 3] ICP30 unregister session Session# [5:7577789036618429682:2050] VirtualId# [5:7577789036618429680:0]
2025-11-28T14:51:38.292800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789036618429682:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.292864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:38.292903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:38.294060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789029035092888:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:38.294152Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:38.294328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789033330060285:2050] [node 3] ICS01 socket: 214 reason# EndOfStream
2025-11-28T14:51:38.294370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789029035092889:2050] [node 3] ICP30 unregister session Session# [6:7577789033330060285:2050] VirtualId# [6:7577789033330060283:0]
2025-11-28T14:51:38.294394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789033330060285:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.294463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:38.294499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:38.317765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:38.318007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789036618429673:2108] [node 4] ICS01 socket: 168 reason# EndOfStream
2025-11-28T14:51:38.318066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789032323462348:2108] [node 4] ICP30 unregister session Session# [5:7577789036618429673:2108] VirtualId# [5:7577789036618429668:0]
2025-11-28T14:51:38.318108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789036618429673:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.318225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:38.318303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:38.318787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:38.318982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789033330060269:2109] [node 4] ICS07 socket disconnect 203 reason# EndOfStream
2025-11-28T14:51:38.319012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789033330060269:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.319104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:38.319147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789033330060269:2109] [node 4] ICS15 start handshake
2025-11-28T14:51:38.319879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789029035092951:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:38.320012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789033330060269:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:38.320049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789029035092951:2109] [node 4] ICP30 unregister session Session# [6:7577789033330060269:2109] VirtualId# [6:7577789033330060266:0]
2025-11-28T14:51:38.320090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:38.320134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789029035092951:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.320164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.329995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789029035092951:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:51:38.330311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:38.330611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789033330060277:2106] [node 5] ICS07 socket disconnect 175 reason# EndOfStream
2025-11-28T14:51:38.330649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789033330060277:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:38.330774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:38.330815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789033330060277:2106] [node 5] ICS15 start handshake
2025-11-28T14:51:38.331963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789029035092949:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11164) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:38.332015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789033330060277:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:38.332342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789029035092949:2106] [node 5] ICP30 unregister session Session# [6:7577789033330060277:2106] VirtualId# [6:7577789033330060275:0]
2025-11-28T14:51:38.332445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:38.332503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789029035092949:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11164) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:38.332554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11164) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:38.400273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789033411364586:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:38.400319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789033411364587:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:38.421366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789033045073342:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:38.421405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789033045073343:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:38.443394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789035619164400:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:38.443664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789035619164401:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:38.463006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789033912355102:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:38.463048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789033912355104:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:38.463059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789033912355103:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:38.481898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789034756142098:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:38.481906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789034756142097:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:38.481937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789034756142099:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:38.507126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789034688005153:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:38.507161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789034688005155:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:38.507454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789034688005154:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:38.606201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789034688005215:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:38.605957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789033411364649:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:38.621315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789035619164465:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:38.621281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789033411364650:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:38.621349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789033411364651:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:38.626933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789033045073406:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:38.627077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789034688005217:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:38.628045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789033912355165:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:38.628354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789034756142160:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:38.628475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789033912355166:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:38.627239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789033045073405:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:38.627913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789035619164467:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:38.628046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789033045073407:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:38.628336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789035619164466:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:38.628715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789034756142161:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:38.631002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789033045073342:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43916498ns) {Configure:104(43905373ns) {ProcessConfigured:135(1463111ns) {StartInitialHandshake:157(1231329ns) {DropHandshakes:470(2181ns) {DropIncomingHandshake:442(682ns) DropOutgoingHandshake:457(317ns)} GenerateSessionVirtualId:988(1024ns) PrepareNewSessionHandshake:477(1515ns) {SwitchToState:179(749ns)}}}}}
2025-11-28T14:51:38.632440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789033045073343:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19394360ns) {Configure:104(19385423ns) {ProcessConfigured:135(265460ns) {StartInitialHandshake:157(262633ns) {DropHandshakes:470(2709ns) {DropIncomingHandshake:442(916ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(1568ns) PrepareNewSessionHandshake:477(1314ns) {SwitchToState:179(659ns)}}}}}
2025-11-28T14:51:38.636199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789033411364649:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18264667ns) {Configure:104(14090572ns) {ProcessConfigured:135(484978ns) {StartInitialHandshake:157(368388ns) {DropHandshakes:470(1049ns) {DropIncomingHandshake:442(307ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:477(882ns) {SwitchToState:179(338ns)}}}}}
2025-11-28T14:51:38.637339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789035619164401:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45947286ns) {Configure:104(45940252ns) {ProcessConfigured:135(463741ns) {StartInitialHandshake:157(462130ns) {DropHandshakes:470(1187ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(1164ns) {SwitchToState:179(679ns)}}}}}
2025-11-28T14:51:38.637948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789033411364587:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49852305ns) {Configure:104(49848589ns) {ProcessConfigured:135(386986ns) {StartInitialHandshake:157(385223ns) {DropHandshakes:470(886ns) {DropIncomingHandshake:442(299ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(295ns) PrepareNewSessionHandshake:477(742ns) {SwitchToState:179(339ns)}}}}}
2025-11-28T14:51:38.637957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789033411364586:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(60002019ns) {Configure:104(59995553ns) {ProcessConfigured:135(620955ns) {StartInitialHandshake:157(618810ns) {DropHandshakes:470(735ns) {DropIncomingHandshake:442(259ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:477(2945ns) {SwitchToState:179(1521ns)}}}}}
2025-11-28T14:51:38.645222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789034688005153:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19329531ns) {Configure:104(15139850ns) {ProcessConfigured:135(417332ns) {StartInitialHandshake:157(415974ns) {DropHandshakes:470(1100ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(846ns) {SwitchToState:179(369ns)}}}}}
2025-11-28T14:51:38.645100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789035619164400:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46613308ns) {Configure:104(46608075ns) {ProcessConfigured:135(539748ns) {StartInitialHandshake:157(537772ns) {DropHandshakes:470(1151ns) {DropIncomingHandshake:442(279ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:477(911ns) {SwitchToState:179(412ns)}}}}}
2025-11-28T14:51:38.645592Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789033912355103:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45761951ns) {Configure:104(45001366ns) {ProcessConfigured:135(907740ns) {StartInitialHandshake:157(906370ns) {DropHandshakes:470(1431ns) {DropIncomingHandshake:442(234ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(1412ns) {SwitchToState:179(698ns)}}}}}
2025-11-28T14:51:38.645164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789034756142097:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42134754ns) {Configure:104(42132703ns) {ProcessConfigured:135(1330929ns) {StartInitialHandshake:157(1230212ns) {DropHandshakes:470(1289ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(1339ns) PrepareNewSessionHandshake:477(1730ns) {SwitchToState:179(912ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.645616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789033912355166:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16093235ns) {Configure:104(16089026ns) {ProcessConfigured:135(6727446ns) {StartInitialHandshake:157(6725696ns) {DropHandshakes:470(938ns) {DropIncomingHandshake:442(248ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(1588ns) {SwitchToState:179(735ns)}}}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:51:38.645865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789033912355102:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44089327ns) {Configure:104(44080840ns) {ProcessConfigured:135(7490050ns) {StartInitialHandshake:157(7488700ns) {DropHandshakes:470(727ns) {DropIncomingHandshake:442(233ns) DropOutgoingHandshake:457(181ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(2339ns) {SwitchToState:179(1093ns)}}}}}
2025-11-28T14:51:38.646094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789034756142098:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19286144ns) {Configure:104(19279575ns) {ProcessConfigured:135(7634796ns) {StartInitialHandshake:157(7632893ns) {DropHandshakes:470(122819ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(356ns) PrepareNewSessionHandshake:477(3172ns) {SwitchToState:179(1149ns)}}}}}
2025-11-28T14:51:38.646829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789033045073421:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29010
2025-11-28T14:51:38.647034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789033912355104:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45281731ns) {Configure:104(45279342ns) {ProcessConfigured:135(1236567ns) {StartInitialHandshake:157(1235144ns) {DropHandshakes:470(1087ns) {DropIncomingHandshake:442(293ns) DropOutgoingHandshake:457(114ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:477(1139ns) {SwitchToState:179(106ns)}}}}}
2025-11-28T14:51:38.647880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789035619164478:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21351
2025-11-28T14:51:38.647871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789033411364662:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64115
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.649110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789035619164478:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.649907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364662:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.650941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789034756142172:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42071
2025-11-28T14:51:38.650924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789035619164401:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.650628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789033411364665:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64115
2025-11-28T14:51:38.652385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789034756142175:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42071
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.656008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789033045073411:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.653894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789033912355177:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53612
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.655150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005231:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-11-28T14:51:38.656019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364665:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.656049Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789033045073411:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.658410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789035619164482:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21351
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.656405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789034756142172:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.656469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789033912355177:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.656447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789033411364667:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64115
2025-11-28T14:51:38.656475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789033045073421:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.658318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789034756142175:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.658497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789033912355181:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53612
2025-11-28T14:51:38.658616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789033045073411:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:38.658426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789034756142178:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42071
2025-11-28T14:51:38.658685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005232:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-11-28T14:51:38.673681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789035619164471:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.673680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789033411364669:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64115
2025-11-28T14:51:38.659069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789033912355178:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53612
2025-11-28T14:51:38.659321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789033045073422:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29010
2025-11-28T14:51:38.658437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789034756142177:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42071
2025-11-28T14:51:38.673736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789035619164471:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.667246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789033912355178:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.673681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789033045073416:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.658694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789034756142098:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.683698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789035619164472:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.683729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789035619164472:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.673890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789033912355104:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.673736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789033045073416:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.658693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789034756142179:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42071
2025-11-28T14:51:38.659098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789034756142099:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.684802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789035619164471:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:38.673889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789033045073343:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.673948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:38.682969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789033045073343:2049] [node 3] ICP22 created new session: [2:7577789033045073425:2049]
2025-11-28T14:51:38.685990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789035619164484:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21351
2025-11-28T14:51:38.683040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789033045073425:2049] [node 3] ICS09 handshake done sender: [2:7577789033045073411:2111] self: [2:7577789033045073410:0] peer: [3:7577789035619164483:0] socket: 151 qp: -1
2025-11-28T14:51:38.686375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789035619164472:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:38.683899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789033045073425:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:38.684065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0
2025-11-28T14:51:38.686770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789035619164465:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.684482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789033045073416:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:38.687137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789035619164478:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:38.689859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789034756142182:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42071
2025-11-28T14:51:38.689978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789033912355185:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53612
2025-11-28T14:51:38.690432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789033912355186:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53612
2025-11-28T14:51:38.687509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:38.685842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789033045073407:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.688789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789035619164467:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.686370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:38.690046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789034756142183:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42071
2025-11-28T14:51:38.689794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005233:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
No RDMA context found for GID: 2025-11-28T14:51:38.689194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789035619164465:2109] [node 5] ICP22 created new session: [3:7577789035619164487:2109]
::ffff:127.1.0.0
2025-11-28T14:51:38.686991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789033045073407:2108] [node 5] ICP22 created new session: [2:7577789033045073429:2108]
2025-11-28T14:51:38.689433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:38.687345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789033045073429:2108] [node 5] ICS09 handshake done sender: [2:7577789033045073416:2112] self: [2:7577789033045073414:0] peer: [5:7577789034756142180:0] socket: 153 qp: -1
2025-11-28T14:51:38.689460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789035619164401:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.690771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789034688005233:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.687385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789033045073429:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:38.689572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789035619164467:2107] [node 4] ICP22 created new session: [3:7577789035619164489:2107]
2025-11-28T14:51:38.691371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005234:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-11-28T14:51:38.688584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:38.691947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789034756142172:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:38.689574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789035619164487:2109] [node 5] ICS09 handshake done sender: [3:7577789035619164471:2112] self: [3:7577789035619164468:0] peer: [5:7577789034756142181:0] socket: 160 qp: -1
2025-11-28T14:51:38.691525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789034688005154:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.692025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789033912355178:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:38.693363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789034756142175:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:38.689617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789035619164487:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:38.690394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789033045073432:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29010
2025-11-28T14:51:38.693434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789033912355104:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.693601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789034756142098:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.692267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789033045073419:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.689618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789035619164489:2107] [node 4] ICS09 handshake done sender: [3:7577789035619164472:2113] self: [3:7577789035619164469:0] peer: [4:7577789033912355184:0] socket: 162 qp: -1
2025-11-28T14:51:38.693597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:38.693644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:38.689739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789035619164489:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:38.692290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789033045073419:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:38.693648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789033912355104:2050] [node 3] ICP22 created new session: [4:7577789033912355187:2050]
2025-11-28T14:51:38.693813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789034756142098:2049] [node 2] ICP22 created new session: [5:7577789034756142184:2049]
2025-11-28T14:51:38.689752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:38.693400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789033045073343:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:38.694064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789034756142184:2049] [node 2] ICS09 handshake done sender: [5:7577789034756142172:2115] self: [5:7577789034756142180:0] peer: [2:7577789033045073414:0] socket: 156 qp: -1
2025-11-28T14:51:38.693862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789033912355187:2050] [node 3] ICS09 handshake done sender: [4:7577789033912355178:2116] self: [4:7577789033912355184:0] peer: [3:7577789035619164469:0] socket: 173 qp: -1
2025-11-28T14:51:38.689769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:38.694252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789034756142184:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:38.693983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789033045073419:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:38.694250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789033912355187:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:38.689793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789035619164401:2049] [node 2] ICP22 created new session: [3:7577789035619164491:2049]
2025-11-28T14:51:38.694623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789033045073406:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.694478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:38.694295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.694668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.689927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789035619164491:2049] [node 2] ICS09 handshake done sender: [3:7577789035619164478:2117] self: [3:7577789035619164483:0] peer: [2:7577789033045073410:0] socket: 154 qp: -1
2025-11-28T14:51:38.695441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789034756142099:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.695169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789033045073406:2109] [node 6] ICP22 created new session: [2:7577789033045073434:2109]
2025-11-28T14:51:38.695488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:38.689945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:38.696049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789034688005154:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789034688005233:2117] held: no
2025-11-28T14:51:38.695609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789034756142099:2050] [node 3] ICP22 created new session: [5:7577789034756142186:2050]
2025-11-28T14:51:38.695217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789033045073434:2109] [node 6] ICS09 handshake done sender: [2:7577789033045073419:2114] self: [2:7577789033045073418:0] peer: [6:7577789034688005235:0] socket: 202 qp: -1
2025-11-28T14:51:38.690328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789035619164491:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:38.696711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005236:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-11-28T14:51:38.690436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:38.695316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789033045073434:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:38.696156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789034756142186:2050] [node 3] ICS09 handshake done sender: [5:7577789034756142175:2116] self: [5:7577789034756142181:0] peer: [3:7577789035619164468:0] socket: 166 qp: -1
2025-11-28T14:51:38.695416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.696250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789034756142186:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.697295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789034688005233:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:38.696276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:38.697689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005237:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-11-28T14:51:38.697928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789034688005154:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.697961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:38.698267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789034688005154:2049] [node 2] ICP22 created new session: [6:7577789034688005239:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.698464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789034688005239:2049] [node 2] ICS09 handshake done sender: [6:7577789034688005233:2117] self: [6:7577789034688005235:0] peer: [2:7577789033045073418:0] socket: 203 qp: -1
2025-11-28T14:51:38.698560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789034688005239:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:38.698616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:38.698691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789033411364671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.700511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789033912355190:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53612
2025-11-28T14:51:38.701345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364671:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.704350Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789033411364661:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.714846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789034688005231:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.715087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789034688005153:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.715382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364661:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.715937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789033411364661:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.716214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789033411364651:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.716204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005242:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-11-28T14:51:38.716281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.716345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789033411364651:2109] [node 6] ICP22 created new session: [1:7577789033411364674:2109]
2025-11-28T14:51:38.716449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789033411364674:2109] [node 6] ICS09 handshake done sender: [1:7577789033411364661:2114] self: [1:7577789033411364654:0] peer: [6:7577789034688005241:0] socket: 171 qp: -1
2025-11-28T14:51:38.716492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789033411364674:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:38.716504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789034688005231:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:38.716528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.716737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789034688005153:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.716807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:38.716941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789034688005153:2048] [node 1] ICP22 created new session: [6:7577789034688005243:2048]
2025-11-28T14:51:38.717105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789034688005243:2048] [node 1] ICS09 handshake done sender: [6:7577789034688005231:2115] self: [6:7577789034688005241:0] peer: [1:7577789033411364654:0] socket: 172 qp: -1
2025-11-28T14:51:38.717139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789034688005243:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.717194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:38.718338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789033045073436:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29010
2025-11-28T14:51:38.726051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789035619164482:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.717752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789034756142166:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.734939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789035619164465:2109] [node 5] ICP18 handshake (actor [3:7577789035619164482:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.735672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789035619164495:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21351
2025-11-28T14:51:38.741248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789034688005225:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.767787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789034756142177:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.769932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789034756142167:2111] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.778335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789034688005232:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.778572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789034688005215:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.778838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789034756142167:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.779356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789034756142167:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.779580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789034756142160:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.779539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005247:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-11-28T14:51:38.779719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.779782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789034756142160:2109] [node 6] ICP22 created new session: [5:7577789034756142191:2109]
2025-11-28T14:51:38.779847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789034756142191:2109] [node 6] ICS09 handshake done sender: [5:7577789034756142167:2111] self: [5:7577789034756142162:0] peer: [6:7577789034688005246:0] socket: 177 qp: -1
2025-11-28T14:51:38.779857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789034688005232:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:38.779951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789034756142191:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:38.780008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.780271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789034688005215:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22436) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789034688005232:2116] held: no
2025-11-28T14:51:38.780467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789034688005215:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.780509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:38.780546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789034688005215:2107] [node 5] ICP22 created new session: [6:7577789034688005248:2107]
2025-11-28T14:51:38.780586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789034688005248:2107] [node 5] ICS09 handshake done sender: [6:7577789034688005232:2116] self: [6:7577789034688005246:0] peer: [5:7577789034756142162:0] socket: 179 qp: -1
2025-11-28T14:51:38.780617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789034688005248:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:38.780660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:38.792739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789033411364660:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.801398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789034756142178:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.801573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789034756142097:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.801843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364660:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.802411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789033411364660:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.802580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789033411364650:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.802647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:38.802709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789033411364650:2108] [node 5] ICP22 created new session: [1:7577789033411364678:2108]
2025-11-28T14:51:38.802769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789033411364678:2108] [node 5] ICS09 handshake done sender: [1:7577789033411364660:2113] self: [1:7577789033411364658:0] peer: [5:7577789034756142193:0] socket: 169 qp: -1
2025-11-28T14:51:38.802814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789033411364678:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:38.802861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:38.802730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789034756142194:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42071
2025-11-28T14:51:38.803009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789034756142178:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:38.803270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789034756142097:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.803384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:38.803429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789034756142097:2048] [node 1] ICP22 created new session: [5:7577789034756142195:2048]
2025-11-28T14:51:38.803479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789034756142195:2048] [node 1] ICS09 handshake done sender: [5:7577789034756142178:2118] self: [5:7577789034756142193:0] peer: [1:7577789033411364658:0] socket: 181 qp: -1
2025-11-28T14:51:38.803516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789034756142195:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.803553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:38.803603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789035619164496:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21351
2025-11-28T14:51:38.811952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789034756142169:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.835312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364667:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.835566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789033411364650:2108] [node 5] ICP18 handshake (actor [1:7577789033411364667:2117]) is too old
2025-11-28T14:51:38.847833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789033411364655:2110] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.875780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789033912355181:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.875946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789033912355102:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.876214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364655:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.876767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789033411364655:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:38.876861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789033912355192:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53612
2025-11-28T14:51:38.877031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789033411364649:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.877089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:38.877134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789033411364649:2107] [node 4] ICP22 created new session: [1:7577789033411364682:2107]
2025-11-28T14:51:38.877265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789033411364682:2107] [node 4] ICS09 handshake done sender: [1:7577789033411364655:2110] self: [1:7577789033411364652:0] peer: [4:7577789033912355191:0] socket: 175 qp: -1
2025-11-28T14:51:38.877313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789033411364682:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:38.877624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:38.877652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789033912355181:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:38.877898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789033912355102:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.877948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:38.877985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789033912355102:2048] [node 1] ICP22 created new session: [4:7577789033912355193:2048]
2025-11-28T14:51:38.878026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789033912355193:2048] [node 1] ICS09 handshake done sender: [4:7577789033912355181:2117] self: [4:7577789033912355191:0] peer: [1:7577789033411364652:0] socket: 178 qp: -1
2025-11-28T14:51:38.878069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789033912355193:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:38.878102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:38.880674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789033912355174:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.891067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789034756142179:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.891302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789034756142161:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.891516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789033912355174:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.892061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789033912355174:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.892222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789033912355165:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.892245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789034756142198:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42071
2025-11-28T14:51:38.892284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:38.892341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789033912355165:2108] [node 5] ICP22 created new session: [4:7577789033912355196:2108]
2025-11-28T14:51:38.892412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789033912355196:2108] [node 5] ICS09 handshake done sender: [4:7577789033912355174:2112] self: [4:7577789033912355170:0] peer: [5:7577789034756142197:0] socket: 188 qp: -1
2025-11-28T14:51:38.892456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789033912355196:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:38.892496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:38.892542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789034756142179:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:38.892735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789034756142161:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.892917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:38.892980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789034756142161:2108] [node 4] ICP22 created new session: [5:7577789034756142199:2108]
2025-11-28T14:51:38.893036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789034756142199:2108] [node 4] ICS09 handshake done sender: [5:7577789034756142179:2119] self: [5:7577789034756142197:0] peer: [4:7577789033912355170:0] socket: 190 qp: -1
2025-11-28T14:51:38.893082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789034756142199:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:38.893125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:38.894116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789033912355172:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.901693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789033045073422:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.904526Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789033912355175:2113] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.912399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364669:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.912612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789033411364649:2107] [node 4] ICP18 handshake (actor [1:7577789033411364669:2118]) is too old
2025-11-28T14:51:38.914430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789033045073417:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.922955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789033912355185:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.923150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789033912355103:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.923408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789033045073417:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:38.923909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789033045073417:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:38.923998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789033912355199:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53612
2025-11-28T14:51:38.924163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789033045073405:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.924258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789033912355185:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:38.924279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:38.924539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789033912355103:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.924336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789033045073405:2107] [node 4] ICP22 created new session: [2:7577789033045073439:2107]
2025-11-28T14:51:38.924503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789033045073439:2107] [node 4] ICS09 handshake done sender: [2:7577789033045073417:2113] self: [2:7577789033045073415:0] peer: [4:7577789033912355198:0] socket: 197 qp: -1
2025-11-28T14:51:38.924903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:38.924707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789033045073439:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:38.924982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789033912355103:2049] [node 2] ICP22 created new session: [4:7577789033912355200:2049]
2025-11-28T14:51:38.925076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789033912355200:2049] [node 2] ICS09 handshake done sender: [4:7577789033912355185:2118] self: [4:7577789033912355198:0] peer: [2:7577789033045073415:0] socket: 198 qp: -1
2025-11-28T14:51:38.924859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:38.925115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789033912355200:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:38.925179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:38.925565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789033912355103:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:38.927912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789035619164477:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.954726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789034688005234:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.954927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789034688005155:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.955135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789035619164477:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.955598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789035619164477:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.955789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789035619164466:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.955826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.955859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789035619164466:2110] [node 6] ICP22 created new session: [3:7577789035619164498:2110]
2025-11-28T14:51:38.955848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005251:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-11-28T14:51:38.955923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789035619164498:2110] [node 6] ICS09 handshake done sender: [3:7577789035619164477:2116] self: [3:7577789035619164475:0] peer: [6:7577789034688005250:0] socket: 206 qp: -1
2025-11-28T14:51:38.955967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789035619164498:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:38.956030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.956064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789034688005234:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:38.956187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789034688005155:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.956274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:38.956396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789034688005155:2050] [node 3] ICP22 created new session: [6:7577789034688005252:2050]
2025-11-28T14:51:38.956446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789034688005252:2050] [node 3] ICS09 handshake done sender: [6:7577789034688005234:2118] self: [6:7577789034688005250:0] peer: [3:7577789035619164475:0] socket: 207 qp: -1
2025-11-28T14:51:38.956485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789034688005252:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:38.956520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:38.960853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789033912355176:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:38.988013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789034688005237:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.988200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789034688005217:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:38.988458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789033912355176:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:38.988989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789033912355176:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:38.989175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789034688005255:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-11-28T14:51:38.989207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789033912355166:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:38.989299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:38.989350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789033912355166:2109] [node 6] ICP22 created new session: [4:7577789033912355203:2109]
2025-11-28T14:51:38.989399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789033912355203:2109] [node 6] ICS09 handshake done sender: [4:7577789033912355176:2114] self: [4:7577789033912355171:0] peer: [6:7577789034688005254:0] socket: 159 qp: -1
2025-11-28T14:51:38.989491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789033912355203:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:38.989406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789034688005237:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:38.989594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789034688005217:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:38.989612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:38.989775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:38.989952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789034688005217:2108] [node 4] ICP22 created new session: [6:7577789034688005256:2108]
2025-11-28T14:51:38.990090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789034688005256:2108] [node 4] ICS09 handshake done sender: [6:7577789034688005237:2120] self: [6:7577789034688005254:0] peer: [4:7577789033912355171:0] socket: 208 qp: -1
2025-11-28T14:51:38.990108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789034688005256:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:38.990138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:38.990841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789034688005217:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:38.992141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789034756142170:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:39.004139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789033912355190:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.004346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789033912355165:2108] [node 5] ICP18 handshake (actor [4:7577789033912355190:2120]) is too old
2025-11-28T14:51:39.007053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789033411364656:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:39.026098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789033045073436:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.026362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789033045073342:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.026589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364656:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.027039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789033411364656:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:39.027225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789033411364586:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.027344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:39.027397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789033411364586:2048] [node 2] ICP22 created new session: [1:7577789037706331981:2048]
2025-11-28T14:51:39.027454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789037706331981:2048] [node 2] ICS09 handshake done sender: [1:7577789033411364656:2111] self: [1:7577789033411364653:0] peer: [2:7577789037340040737:0] socket: 214 qp: -1
2025-11-28T14:51:39.027443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789037340040738:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29010
2025-11-28T14:51:39.027495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789037706331981:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:39.027539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:39.027756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789033045073436:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:39.027921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789033045073342:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789033045073436:2118] held: no
2025-11-28T14:51:39.028233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789033045073342:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.028306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.028429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789033045073342:2048] [node 1] ICP22 created new session: [2:7577789037340040739:2048]
2025-11-28T14:51:39.028489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789037340040739:2048] [node 1] ICS09 handshake done sender: [2:7577789033045073436:2118] self: [2:7577789037340040737:0] peer: [1:7577789033411364653:0] socket: 215 qp: -1
2025-11-28T14:51:39.028528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789037340040739:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:39.028562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:39.030254Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789034688005227:2114] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:39.038450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789035619164495:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.038656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789035619164466:2110] [node 6] ICP18 handshake (actor [3:7577789035619164495:2120]) is too old
2025-11-28T14:51:39.041221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789033411364659:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:39.041970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789035619164496:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.042173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789035619164400:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.042383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789033411364659:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.042818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789033411364659:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.042939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789039914131797:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21351
2025-11-28T14:51:39.043065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789033411364587:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.043146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:39.043194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789033411364587:2049] [node 3] ICP22 created new session: [1:7577789037706331984:2049]
2025-11-28T14:51:39.043217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789035619164496:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:39.043243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789037706331984:2049] [node 3] ICS09 handshake done sender: [1:7577789033411364659:2112] self: [1:7577789033411364657:0] peer: [3:7577789039914131796:0] socket: 176 qp: -1
2025-11-28T14:51:39.043350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789037706331984:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:39.043406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:39.043464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789035619164400:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.043522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.043800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789035619164400:2048] [node 1] ICP22 created new session: [3:7577789039914131798:2048]
2025-11-28T14:51:39.043887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789039914131798:2048] [node 1] ICS09 handshake done sender: [3:7577789035619164496:2121] self: [3:7577789039914131796:0] peer: [1:7577789033411364657:0] socket: 217 qp: -1
2025-11-28T14:51:39.043919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789039914131798:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:39.043975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:39.045070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789035619164400:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:39.048204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:39.048456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789037340040739:2048] [node 1] ICS01 socket: 215 reason# EndOfStream
2025-11-28T14:51:39.048503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789033045073342:2048] [node 1] ICP30 unregister session Session# [2:7577789037340040739:2048] VirtualId# [2:7577789037340040737:0]
2025-11-28T14:51:39.048593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789037340040739:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.048684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.048732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.049008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:39.049204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789039914131798:2048] [node 1] ICS07 socket disconnect 217 reason# EndOfStream
2025-11-28T14:51:39.049245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789039914131798:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.049339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.049387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789039914131798:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:39.050087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789035619164400:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.050138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789039914131798:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.050427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789035619164400:2048] [node 1] ICP30 unregister session Session# [3:7577789039914131798:2048] VirtualId# [3:7577789039914131796:0]
2025-11-28T14:51:39.050472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.050505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789035619164400:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.050530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.050629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:39.050819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789033912355193:2048] [node 1] ICS07 socket disconnect 178 reason# EndOfStream
2025-11-28T14:51:39.050855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789033912355193:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.050933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.050980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789033912355193:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:39.051330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:39.051548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789034756142195:2048] [node 1] ICS07 socket disconnect 181 reason# EndOfStream
2025-11-28T14:51:39.051596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789034756142195:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.051558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789033912355102:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.051662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789033912355193:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.051704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789033912355102:2048] [node 1] ICP30 unregister session Session# [4:7577789033912355193:2048] VirtualId# [4:7577789033912355191:0]
2025-11-28T14:51:39.051726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.051745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.051773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789033912355102:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.051779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789034756142195:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:39.051785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.051951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:39.052151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789034688005243:2048] [node 1] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-28T14:51:39.052176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789034688005243:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.052235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.052289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789034688005243:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:39.052512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789034756142097:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.052563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789034756142195:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.052587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789034756142097:2048] [node 1] ICP30 unregister session Session# [5:7577789034756142195:2048] VirtualId# [5:7577789034756142193:0]
2025-11-28T14:51:39.052714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.052778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789034756142097:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.052792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.053007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789034688005153:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.053171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789034688005243:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.053207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789034688005153:2048] [node 1] ICP30 unregister session Session# [6:7577789034688005243:2048] VirtualId# [6:7577789034688005241:0]
2025-11-28T14:51:39.053259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.053296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789034688005153:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.053327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29690) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.059885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789035619164400:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:39.061925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789033912355102:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:39.062916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789034756142097:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:39.062945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789034688005153:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:39.085965Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:39.086234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789035619164491:2049] [node 2] ICS07 socket disconnect 154 reason# EndOfStream
2025-11-28T14:51:39.086265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789035619164491:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.086350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:39.086404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789035619164491:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:39.087197Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789035619164401:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.087256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789035619164491:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.087307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789035619164401:2049] [node 2] ICP30 unregister session Session# [3:7577789035619164491:2049] VirtualId# [3:7577789035619164483:0]
2025-11-28T14:51:39.087382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:39.087418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789035619164401:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.087433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.088728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:39.088927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789033912355200:2049] [node 2] ICS07 socket disconnect 198 reason# EndOfStream
2025-11-28T14:51:39.088966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789033912355200:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.089049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:39.089091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789033912355200:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:39.089398Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:39.089640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789034756142184:2049] [node 2] ICS07 socket disconnect 156 reason# EndOfStream
2025-11-28T14:51:39.089688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789034756142184:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.089746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789033912355103:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.089812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:39.089852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789033912355200:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.089900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789033912355103:2049] [node 2] ICP30 unregister session Session# [4:7577789033912355200:2049] VirtualId# [4:7577789033912355198:0]
2025-11-28T14:51:39.089894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789034756142184:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:39.089945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:39.089900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:39.089969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789033912355103:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.089990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.090086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789034688005239:2049] [node 2] ICS01 socket: 203 reason# EndOfStream
2025-11-28T14:51:39.090163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789034688005154:2049] [node 2] ICP30 unregister session Session# [6:7577789034688005239:2049] VirtualId# [6:7577789034688005235:0]
2025-11-28T14:51:39.090208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789034688005239:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.090287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:39.090340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:39.090577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789034756142098:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.090753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789034756142184:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.090786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789034756142098:2049] [node 2] ICP30 unregister session Session# [5:7577789034756142184:2049] VirtualId# [5:7577789034756142180:0]
2025-11-28T14:51:39.090838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:39.090864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789034756142098:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.090877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21105) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.099808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789033912355103:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:39.100813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789034756142098:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:39.121067Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:39.121293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789033912355187:2050] [node 3] ICS07 socket disconnect 173 reason# EndOfStream
2025-11-28T14:51:39.121329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789033912355187:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.121428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:39.121480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789033912355187:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:39.121824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:39.122081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789034756142186:2050] [node 3] ICS07 socket disconnect 166 reason# EndOfStream
2025-11-28T14:51:39.122109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789034756142186:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.122204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789033912355104:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26451) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.122202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:39.122249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789034756142186:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:39.122360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789033912355187:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.122367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:39.122420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789033912355104:2050] [node 3] ICP30 unregister session Session# [4:7577789033912355187:2050] VirtualId# [4:7577789033912355184:0]
2025-11-28T14:51:39.122509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:39.122523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789034688005252:2050] [node 3] ICS07 socket disconnect 207 reason# EndOfStream
2025-11-28T14:51:39.122567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789033912355104:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26451) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.122544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789034688005252:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.122581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26451) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.122595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:39.122627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789034688005252:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:39.123024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789034756142099:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26451) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.123193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789034688005155:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26451) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.123085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789034756142186:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.123242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789034688005252:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.123122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789034756142099:2050] [node 3] ICP30 unregister session Session# [5:7577789034756142186:2050] VirtualId# [5:7577789034756142181:0]
2025-11-28T14:51:39.123342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789034688005155:2050] [node 3] ICP30 unregister session Session# [6:7577789034688005252:2050] VirtualId# [6:7577789034688005250:0]
2025-11-28T14:51:39.123163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:39.123385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:39.123215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789034756142099:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26451) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.123419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789034688005155:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26451) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.123234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26451) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.123434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26451) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.131677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:39.131926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789034756142199:2108] [node 4] ICS07 socket disconnect 190 reason# EndOfStream
2025-11-28T14:51:39.131970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789034756142199:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.132080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:39.132140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789034756142199:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:39.132492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:39.132628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789034688005256:2108] [node 4] ICS01 socket: 208 reason# EndOfStream
2025-11-28T14:51:39.132672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789034688005217:2108] [node 4] ICP30 unregister session Session# [6:7577789034688005256:2108] VirtualId# [6:7577789034688005254:0]
2025-11-28T14:51:39.132698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789034688005256:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.132758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:39.132794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:39.132869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789034756142161:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27857) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.132980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789034756142199:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.133028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789034756142161:2108] [node 4] ICP30 unregister session Session# [5:7577789034756142199:2108] VirtualId# [5:7577789034756142197:0]
2025-11-28T14:51:39.133069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:39.133106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789034756142161:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27857) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.133121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27857) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.133524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789034688005155:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:39.133557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789034756142099:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:39.142008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:39.142163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789034688005248:2107] [node 5] ICS01 socket: 179 reason# EndOfStream
2025-11-28T14:51:39.142198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789034688005215:2107] [node 5] ICP30 unregister session Session# [6:7577789034688005248:2107] VirtualId# [6:7577789034688005246:0]
2025-11-28T14:51:39.142224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789034688005248:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.142305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:39.142340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:39.240951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789037875807125:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:39.240985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789037875807126:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:39.262711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789040495774328:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.262746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789040495774329:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:39.283889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789037614770495:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.283946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789037614770496:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:39.305850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789037821030696:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.305907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789037821030697:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:39.305943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789037821030698:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:39.327309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789037735880959:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.327341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789037735880960:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:39.327361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789037735880961:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:39.348055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789040381691283:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.348121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789040381691284:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:39.348165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789040381691285:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:39.397481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789040381691342:2105] [node 5] ICP01 ready to work
2025-11-28T14:51:39.443172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789037735881023:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:39.444584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789040495774391:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:39.444686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789037614770558:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:39.444983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789040495774392:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:39.444969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789037614770559:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:39.444896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789037735881024:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:39.446364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789037614770560:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:39.446371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789040495774393:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:39.446417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789037875807125:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48467453ns) {Configure:104(48458792ns) {ProcessConfigured:135(435198ns) {StartInitialHandshake:157(432032ns) {DropHandshakes:470(3420ns) {DropIncomingHandshake:442(839ns) DropOutgoingHandshake:457(661ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:477(110091ns) {SwitchToState:179(888ns)}}}}}
2025-11-28T14:51:39.446793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789037875807190:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:39.447363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789037875807191:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:39.447601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789037875807192:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:39.447655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789037821030696:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39623884ns) {Configure:104(39615661ns) {ProcessConfigured:135(453329ns) {StartInitialHandshake:157(448795ns) {DropHandshakes:470(2764ns) {DropIncomingHandshake:442(764ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(985ns) PrepareNewSessionHandshake:477(1209ns) {SwitchToState:179(447ns)}}}}}
2025-11-28T14:51:39.447669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789040381691342:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47813852ns) {Configure:104(47805053ns) {ProcessConfigured:135(503360ns) {StartInitialHandshake:157(500458ns) {DropHandshakes:470(2105ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:477(2047ns) {SwitchToState:179(641ns)}}}}}
2025-11-28T14:51:39.448162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789037821030761:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:39.448303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789040381691349:2106] [node 4] ICP01 ready to work
2025-11-28T14:51:39.448513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789037821030762:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.456501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789037614770571:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23635
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.456854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789037821030771:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.458072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789037735881035:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45951
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.461133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789040495774404:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55618
2025-11-28T14:51:39.461191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789037821030775:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.461747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789040495774405:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55618
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.463490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789040495774406:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55618
2025-11-28T14:51:39.463583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789040495774407:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55618
2025-11-28T14:51:39.463843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789040495774404:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.464195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789040495774411:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55618
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.464197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789037875807204:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-28T14:51:39.464399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789040495774405:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.464714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789037875807205:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.464761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789040495774328:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.464965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789037875807206:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.465082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789037875807207:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
2025-11-28T14:51:39.465256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807204:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.465424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789037875807208:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40204
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.465552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807206:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.465632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789037614770575:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23635
2025-11-28T14:51:39.465737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789037614770576:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23635
2025-11-28T14:51:39.465872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789037614770571:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.466598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789037614770577:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23635
2025-11-28T14:51:39.466795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789037614770495:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.466815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789037735881035:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.466851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807205:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.466834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789037821030775:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.466913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789037821030771:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.467011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807194:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.467038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789037875807194:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.467157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789040495774406:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.467200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789037614770575:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.467297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789037614770576:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.467335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789037821030776:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-28T14:51:39.467395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789037821030698:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.467432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789037821030696:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.467592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789037614770562:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.467614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789037614770562:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.467628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807189:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.467652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789037875807189:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.468757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789037614770562:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:39.468878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807207:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.469140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789037821030779:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-28T14:51:39.469073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807196:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.469098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789037875807196:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.469281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789037614770558:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.469333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:39.469398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789037614770558:2107] [node 4] ICP22 created new session: [3:7577789037614770582:2107]
2025-11-28T14:51:39.469448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789037614770582:2107] [node 4] ICS09 handshake done sender: [3:7577789037614770562:2110] self: [3:7577789037614770561:0] peer: [4:7577789037821030777:0] socket: 168 qp: -1
2025-11-28T14:51:39.469488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789037614770582:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:39.469954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:39.470141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789037821030775:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:39.470508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789037821030698:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.470530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789037875807194:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:39.470555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:39.470618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789037821030698:2050] [node 3] ICP22 created new session: [4:7577789037821030780:2050]
2025-11-28T14:51:39.470661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789037821030780:2050] [node 3] ICS09 handshake done sender: [4:7577789037821030775:2116] self: [4:7577789037821030777:0] peer: [3:7577789037614770561:0] socket: 173 qp: -1
2025-11-28T14:51:39.470696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789037821030780:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:39.470762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:39.470916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789037875807189:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:39.471138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789037875807126:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.471180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:39.471207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789037875807126:2049] [node 3] ICP22 created new session: [1:7577789037875807216:2049]
2025-11-28T14:51:39.471234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789037875807216:2049] [node 3] ICS09 handshake done sender: [1:7577789037875807194:2111] self: [1:7577789037875807193:0] peer: [3:7577789037614770578:0] socket: 157 qp: -1
2025-11-28T14:51:39.471251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789037875807216:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:39.471272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:39.471574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789037875807125:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:39.471666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-28T14:51:39.471705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789037875807125:2048] [node 2] ICP22 created new session: [1:7577789037875807218:2048]
2025-11-28T14:51:39.472530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789037614770584:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23635
2025-11-28T14:51:39.472616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789040495774407:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.472314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789037875807218:2048] [node 2] ICS09 handshake done sender: [1:7577789037875807189:2110] self: [1:7577789037875807188:0] peer: [2:7577789040495774412:0] socket: 156 qp: -1
2025-11-28T14:51:39.472330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789037875807218:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.472350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:39.472790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789040495774411:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.472795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789037735881040:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45951
2025-11-28T14:51:39.472923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789037875807196:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:39.473106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789037614770571:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.473570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789037614770577:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.473586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789037821030782:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-28T14:51:39.473605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789037614770495:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.473653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.473680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789037614770495:2048] [node 1] ICP22 created new session: [3:7577789037614770586:2048]
2025-11-28T14:51:39.473715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789037614770586:2048] [node 1] ICS09 handshake done sender: [3:7577789037614770571:2115] self: [3:7577789037614770578:0] peer: [1:7577789037875807193:0] socket: 160 qp: -1
2025-11-28T14:51:39.473782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789037614770586:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:51:39.473808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
::ffff:127.1.0.0
2025-11-28T14:51:39.473959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789040495774414:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55618
2025-11-28T14:51:39.474088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789037821030776:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.474142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807208:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.474188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789037875807190:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.474233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:39.474322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789037875807190:2107] [node 4] ICP22 created new session: [1:7577789037875807220:2107]
2025-11-28T14:51:39.474454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789040495774404:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:39.474481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789037821030771:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:39.474472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789037875807220:2107] [node 4] ICS09 handshake done sender: [1:7577789037875807196:2112] self: [1:7577789037875807195:0] peer: [4:7577789037821030778:0] socket: 158 qp: -1
2025-11-28T14:51:39.474508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789037875807220:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.474571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:39.474950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789040495774328:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.474947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789037735881041:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45951
2025-11-28T14:51:39.474999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.474959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789037821030696:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.475003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.475035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789040495774328:2048] [node 1] ICP22 created new session: [2:7577789040495774418:2048]
2025-11-28T14:51:39.475036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789037821030696:2048] [node 1] ICP22 created new session: [4:7577789037821030784:2048]
2025-11-28T14:51:39.475074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789037821030784:2048] [node 1] ICS09 handshake done sender: [4:7577789037821030771:2115] self: [4:7577789037821030778:0] peer: [1:7577789037875807195:0] socket: 161 qp: -1
2025-11-28T14:51:39.475074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789040495774418:2048] [node 1] ICS09 handshake done sender: [2:7577789040495774404:2115] self: [2:7577789040495774412:0] peer: [1:7577789037875807188:0] socket: 167 qp: -1
2025-11-28T14:51:39.475099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789037821030784:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:39.475112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789040495774418:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:39.475127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:39.475141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:39.475909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789037821030697:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.478226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789037735881040:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.478226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789037614770496:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.478327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789037875807125:2048] [node 2] ICP18 handshake (actor [1:7577789037875807208:2119]) is too old
2025-11-28T14:51:39.478516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789040495774399:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.478548Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789040495774399:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.478870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691361:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.478915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789037735880961:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.478973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691362:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
No RDMA context found for GID: 2025-11-28T14:51:39.479020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789037735881042:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45951
::ffff:127.1.0.0
2025-11-28T14:51:39.479095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789037735881043:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45951
2025-11-28T14:51:39.480024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789040495774397:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.480036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789037614770568:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.480051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789040495774397:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.480654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789037735881041:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.480057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789037614770568:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.480833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789037735881023:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.481499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691363:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
2025-11-28T14:51:39.481640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789040381691362:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.481903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789040381691285:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.482138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789037821030768:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.482172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789037614770570:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.482202Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789037614770570:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.482225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789037735881046:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45951
2025-11-28T14:51:39.482168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789037821030768:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.483096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789040495774399:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:39.483482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789040381691363:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.483693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789037735881042:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.483776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789040381691361:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.483822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789037614770568:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:39.483901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789040381691349:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.483992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789040495774397:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.484078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789040381691283:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.484079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789037821030788:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53299
2025-11-28T14:51:39.484321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789037614770591:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23635
2025-11-28T14:51:39.484353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789037735880959:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.484386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807200:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.484419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789037875807200:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.484729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789037821030768:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:39.484692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789040495774391:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.485749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:39.485762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789037614770559:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.485828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:39.485955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789037614770559:2108] [node 5] ICP22 created new session: [3:7577789037614770593:2108]
2025-11-28T14:51:39.485924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789040495774391:2107] [node 4] ICP22 created new session: [2:7577789040495774423:2107]
2025-11-28T14:51:39.485930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789037735881040:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:39.485973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789037875807198:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.486001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789040495774423:2107] [node 4] ICS09 handshake done sender: [2:7577789040495774399:2112] self: [2:7577789040495774398:0] peer: [4:7577789037821030786:0] socket: 193 qp: -1
2025-11-28T14:51:39.486047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789040495774423:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:39.486005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789037614770593:2108] [node 5] ICS09 handshake done sender: [3:7577789037614770568:2113] self: [3:7577789037614770567:0] peer: [5:7577789037735881044:0] socket: 204 qp: -1
2025-11-28T14:51:39.486001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789037875807198:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.486083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:39.486038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789037614770593:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:39.486868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789037821030770:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.486111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:39.486910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789037821030770:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.487090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789037735881043:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.487509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789037821030761:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.487548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:39.487516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789037735880961:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.487617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:39.487617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789037821030761:2108] [node 5] ICP22 created new session: [4:7577789037821030790:2108]
2025-11-28T14:51:39.487608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789037875807200:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:39.487597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789040495774329:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.487664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789037821030790:2108] [node 5] ICS09 handshake done sender: [4:7577789037821030768:2113] self: [4:7577789037821030767:0] peer: [5:7577789037735881045:0] socket: 189 qp: -1
2025-11-28T14:51:39.487662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789037735880961:2050] [node 3] ICP22 created new session: [5:7577789037735881049:2050]
2025-11-28T14:51:39.487663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:39.487700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789037821030790:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:39.487721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789037735881049:2050] [node 3] ICS09 handshake done sender: [5:7577789037735881040:2117] self: [5:7577789037735881044:0] peer: [3:7577789037614770567:0] socket: 205 qp: -1
2025-11-28T14:51:39.487701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789040495774329:2049] [node 3] ICP22 created new session: [2:7577789040495774425:2049]
2025-11-28T14:51:39.487742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:39.487757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789037735881049:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:51:39.487739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789040495774425:2049] [node 3] ICS09 handshake done sender: [2:7577789040495774397:2111] self: [2:7577789040495774396:0] peer: [3:7577789037614770588:0] socket: 192 qp: -1
::ffff:127.1.0.0
2025-11-28T14:51:39.487776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789040495774425:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:39.487875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:39.487913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:39.487952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691367:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
2025-11-28T14:51:39.488125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789037875807198:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:39.488291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789037614770577:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:39.488946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789037875807192:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.489069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789037821030776:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:39.489002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:39.489135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789037875807192:2109] [node 6] ICP22 created new session: [1:7577789037875807226:2109]
2025-11-28T14:51:39.489187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789037875807226:2109] [node 6] ICS09 handshake done sender: [1:7577789037875807200:2114] self: [1:7577789037875807199:0] peer: [6:7577789040381691366:0] socket: 170 qp: -1
2025-11-28T14:51:39.489308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789037875807226:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:39.489528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:39.489964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789040381691361:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:39.490407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691368:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
::ffff:127.1.0.0
2025-11-28T14:51:39.490499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789040381691283:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.490473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789037875807191:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.490516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789037614770570:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:39.490541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789037735881048:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45951
2025-11-28T14:51:39.490528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:39.490564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789037875807191:2108] [node 5] ICP22 created new session: [1:7577789037875807228:2108]
2025-11-28T14:51:39.490550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.490600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789037875807228:2108] [node 5] ICS09 handshake done sender: [1:7577789037875807198:2113] self: [1:7577789037875807197:0] peer: [5:7577789037735881047:0] socket: 165 qp: -1
2025-11-28T14:51:39.490627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789037875807228:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:39.490592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789040381691283:2048] [node 1] ICP22 created new session: [6:7577789040381691369:2048]
2025-11-28T14:51:39.490655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:39.490635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789040381691369:2048] [node 1] ICS09 handshake done sender: [6:7577789040381691361:2115] self: [6:7577789040381691366:0] peer: [1:7577789037875807199:0] socket: 207 qp: -1
2025-11-28T14:51:39.490663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789040381691369:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:39.490773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:39.491749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789037614770496:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.491821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:39.491977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789037821030697:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.491884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789037614770496:2049] [node 2] ICP22 created new session: [3:7577789037614770595:2049]
2025-11-28T14:51:39.492788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:39.492794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789037614770595:2049] [node 2] ICS09 handshake done sender: [3:7577789037614770577:2118] self: [3:7577789037614770588:0] peer: [2:7577789040495774396:0] socket: 196 qp: -1
2025-11-28T14:51:39.493009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789037821030697:2049] [node 2] ICP22 created new session: [4:7577789037821030793:2049]
2025-11-28T14:51:39.493063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789037821030793:2049] [node 2] ICS09 handshake done sender: [4:7577789037821030776:2117] self: [4:7577789037821030786:0] peer: [2:7577789040495774398:0] socket: 197 qp: -1
2025-11-28T14:51:39.493022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789037614770595:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:39.493093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789037821030793:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:39.493066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:39.493186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:39.493417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789037821030697:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:39.493821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789037735880960:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.494100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789037821030770:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:39.496805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789037614770560:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.496904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:39.496954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789037614770560:2109] [node 6] ICP22 created new session: [3:7577789037614770597:2109]
2025-11-28T14:51:39.497000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789037614770597:2109] [node 6] ICS09 handshake done sender: [3:7577789037614770570:2114] self: [3:7577789037614770569:0] peer: [6:7577789040381691364:0] socket: 209 qp: -1
2025-11-28T14:51:39.497031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789037614770597:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:39.497014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789037821030762:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.497070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:39.497170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:39.497281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789037821030762:2109] [node 6] ICP22 created new session: [4:7577789037821030795:2109]
2025-11-28T14:51:39.497324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789037821030795:2109] [node 6] ICS09 handshake done sender: [4:7577789037821030770:2114] self: [4:7577789037821030769:0] peer: [6:7577789040381691365:0] socket: 194 qp: -1
2025-11-28T14:51:39.497353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789037821030795:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:39.497323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789037735881041:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:39.497377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:39.497888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789037614770496:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:39.497937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789037735880959:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3229) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789037735881042:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:39.497948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789040381691362:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:39.497999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789040495774401:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.498052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789040495774401:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.498122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789037735881051:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45951
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.498302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691371:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
2025-11-28T14:51:39.498973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789037735881023:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.499050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:39.499089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789037735881023:2108] [node 4] ICP22 created new session: [5:7577789037735881053:2108]
2025-11-28T14:51:39.499138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789037735881053:2108] [node 4] ICS09 handshake done sender: [5:7577789037735881041:2118] self: [5:7577789037735881045:0] peer: [4:7577789037821030767:0] socket: 206 qp: -1
2025-11-28T14:51:39.499108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691372:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
2025-11-28T14:51:39.499169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789037735881053:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:39.499198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:39.499217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789040381691285:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.499275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:39.499314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789040381691285:2050] [node 3] ICP22 created new session: [6:7577789040381691373:2050]
2025-11-28T14:51:39.499273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789040495774401:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:39.499353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789040381691373:2050] [node 3] ICS09 handshake done sender: [6:7577789040381691362:2116] self: [6:7577789040381691364:0] peer: [3:7577789037614770569:0] socket: 210 qp: -1
2025-11-28T14:51:39.499381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789040381691373:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:39.499414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:39.499675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789040495774392:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.499726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:39.499769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789040495774392:2108] [node 5] ICP22 created new session: [2:7577789040495774429:2108]
2025-11-28T14:51:39.499806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789040495774429:2108] [node 5] ICS09 handshake done sender: [2:7577789040495774401:2113] self: [2:7577789040495774400:0] peer: [5:7577789037735881052:0] socket: 188 qp: -1
2025-11-28T14:51:39.499835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789040495774429:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:39.499865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:39.500509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789037735881042:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:39.500561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789040381691372:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.500708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789040381691363:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:39.501099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789040381691349:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.501167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:39.501215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789040381691349:2106] [node 4] ICP22 created new session: [6:7577789040381691376:2106]
2025-11-28T14:51:39.501264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789040381691376:2106] [node 4] ICS09 handshake done sender: [6:7577789040381691363:2117] self: [6:7577789040381691365:0] peer: [4:7577789037821030769:0] socket: 211 qp: -1
2025-11-28T14:51:39.501299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789040381691376:2106] [node 4] ICS10 traffic start
2025-11-28T14:51:39.501329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:39.502049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789037735880959:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.502090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.502131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789037735880959:2048] [node 1] ICP22 created new session: [5:7577789037735881058:2048]
2025-11-28T14:51:39.502172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789037735881058:2048] [node 1] ICS09 handshake done sender: [5:7577789037735881042:2119] self: [5:7577789037735881047:0] peer: [1:7577789037875807197:0] socket: 191 qp: -1
2025-11-28T14:51:39.502205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789037735881058:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:39.502229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:39.502746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789040381691284:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.502775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789037735880960:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789037735881043:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.502931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789037735881056:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45951
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.503095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691378:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
2025-11-28T14:51:39.503201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789040495774403:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.503240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789040495774403:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.503432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789037735881043:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:39.504107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789040495774403:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.504398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789040495774393:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.504358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691381:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
2025-11-28T14:51:39.504444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:39.504482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789040495774393:2109] [node 6] ICP22 created new session: [2:7577789040495774432:2109]
2025-11-28T14:51:39.504539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789040495774432:2109] [node 6] ICS09 handshake done sender: [2:7577789040495774403:2114] self: [2:7577789040495774402:0] peer: [6:7577789040381691380:0] socket: 184 qp: -1
2025-11-28T14:51:39.504578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789040495774432:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:39.504610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:39.504845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789040381691372:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:39.505310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789040381691284:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.505314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789037735880960:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.505358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:39.505379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:39.505397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789037735880960:2049] [node 2] ICP22 created new session: [5:7577789037735881060:2049]
2025-11-28T14:51:39.505427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789040381691284:2049] [node 2] ICP22 created new session: [6:7577789040381691382:2049]
2025-11-28T14:51:39.505448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789037735881060:2049] [node 2] ICS09 handshake done sender: [5:7577789037735881043:2120] self: [5:7577789037735881052:0] peer: [2:7577789040495774400:0] socket: 208 qp: -1
2025-11-28T14:51:39.505466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789040381691382:2049] [node 2] ICS09 handshake done sender: [6:7577789040381691372:2121] self: [6:7577789040381691380:0] peer: [2:7577789040495774402:0] socket: 180 qp: -1
2025-11-28T14:51:39.505484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789037735881060:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:39.505502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789040381691382:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:39.505509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:39.505552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:39.507591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789040381691378:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.507772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789040381691342:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.508011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789037735881034:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.508044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789037735881034:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.508794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789037735881034:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.508960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789040381691385:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25468
2025-11-28T14:51:39.509122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789037735881024:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.509177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:39.509221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789037735881024:2110] [node 6] ICP22 created new session: [5:7577789037735881063:2110]
2025-11-28T14:51:39.509327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789037735881063:2110] [node 6] ICS09 handshake done sender: [5:7577789037735881034:2115] self: [5:7577789037735881033:0] peer: [6:7577789040381691384:0] socket: 177 qp: -1
2025-11-28T14:51:39.509367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789037735881063:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:39.509397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:39.509388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789040381691378:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:39.509600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789040381691342:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.509662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:39.509708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789040381691342:2105] [node 5] ICP22 created new session: [6:7577789040381691386:2105]
2025-11-28T14:51:39.509752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789040381691386:2105] [node 5] ICS09 handshake done sender: [6:7577789040381691378:2122] self: [6:7577789040381691384:0] peer: [5:7577789037735881033:0] socket: 181 qp: -1
2025-11-28T14:51:39.509782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789040381691386:2105] [node 5] ICS10 traffic start
2025-11-28T14:51:39.509805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:39.526941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:39.527169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789040495774418:2048] [node 1] ICS01 socket: 167 reason# EndOfStream
2025-11-28T14:51:39.527212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789040495774328:2048] [node 1] ICP30 unregister session Session# [2:7577789040495774418:2048] VirtualId# [2:7577789040495774412:0]
2025-11-28T14:51:39.527244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789040495774418:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.527336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.527405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.527954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:39.528132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789037614770586:2048] [node 1] ICS01 socket: 160 reason# EndOfStream
2025-11-28T14:51:39.528169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789037614770495:2048] [node 1] ICP30 unregister session Session# [3:7577789037614770586:2048] VirtualId# [3:7577789037614770578:0]
2025-11-28T14:51:39.528194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789037614770586:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.528276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.528336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.530989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:39.531231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789037821030784:2048] [node 1] ICS01 socket: 161 reason# EndOfStream
2025-11-28T14:51:39.531274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789037821030696:2048] [node 1] ICP30 unregister session Session# [4:7577789037821030784:2048] VirtualId# [4:7577789037821030778:0]
2025-11-28T14:51:39.531307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789037821030784:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.531380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.531435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.531596Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:39.531880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789037735881058:2048] [node 1] ICS01 socket: 191 reason# EndOfStream
2025-11-28T14:51:39.531918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789037735880959:2048] [node 1] ICP30 unregister session Session# [5:7577789037735881058:2048] VirtualId# [5:7577789037735881047:0]
2025-11-28T14:51:39.531945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789037735881058:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.532013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.532057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.532504Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:39.532696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789040381691369:2048] [node 1] ICS01 socket: 207 reason# EndOfStream
2025-11-28T14:51:39.532736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789040381691283:2048] [node 1] ICP30 unregister session Session# [6:7577789040381691369:2048] VirtualId# [6:7577789040381691366:0]
2025-11-28T14:51:39.532758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789040381691369:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.532827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:39.532892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:39.550910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:39.551160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789037614770595:2049] [node 2] ICS07 socket disconnect 196 reason# EndOfStream
2025-11-28T14:51:39.551206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789037614770595:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.551331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:39.551402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789037614770595:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:39.553552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:39.553789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789037821030793:2049] [node 2] ICS07 socket disconnect 197 reason# EndOfStream
2025-11-28T14:51:39.553825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789037821030793:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.553899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:39.553955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789037821030793:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:39.554107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789037614770496:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:39.554151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789037614770595:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:39.554360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:39.554496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789037735881060:2049] [node 2] ICS01 socket: 208 reason# EndOfStream
2025-11-28T14:51:39.554525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789037735880960:2049] [node 2] ICP30 unregister session Session# [5:7577789037735881060:2049] VirtualId# [5:7577789037735881052:0]
2025-11-28T14:51:39.554544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789037735881060:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.554614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:39.554648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:39.554890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789037821030697:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.554928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789037614770496:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.554939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789037821030793:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.555008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789037821030697:2049] [node 2] ICP30 unregister session Session# [4:7577789037821030793:2049] VirtualId# [4:7577789037821030786:0]
2025-11-28T14:51:39.555053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:39.555065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789037614770595:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.555088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789037821030697:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.555101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.555116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789037614770496:2049] [node 2] ICP30 unregister session Session# [3:7577789037614770595:2049] VirtualId# [3:7577789037614770588:0]
2025-11-28T14:51:39.555158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:39.555187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789037614770496:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.555221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.555560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:39.555766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789040381691382:2049] [node 2] ICS07 socket disconnect 180 reason# EndOfStream
2025-11-28T14:51:39.555795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789040381691382:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.555860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:39.555902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789040381691382:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:39.556661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789040381691284:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.556707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789040381691382:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.556735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789040381691284:2049] [node 2] ICP30 unregister session Session# [6:7577789040381691382:2049] VirtualId# [6:7577789040381691380:0]
2025-11-28T14:51:39.556772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:39.556794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789040381691284:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.556807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.564716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789037821030697:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:39.566770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789040381691284:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:39.573211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:39.573458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789037821030780:2050] [node 3] ICS01 socket: 173 reason# EndOfStream
2025-11-28T14:51:39.573498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789037821030698:2050] [node 3] ICP30 unregister session Session# [4:7577789037821030780:2050] VirtualId# [4:7577789037821030777:0]
2025-11-28T14:51:39.573533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789037821030780:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.573610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:39.573685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:39.574149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:39.574367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789037735881049:2050] [node 3] ICS01 socket: 205 reason# EndOfStream
2025-11-28T14:51:39.574402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789037735880961:2050] [node 3] ICP30 unregister session Session# [5:7577789037735881049:2050] VirtualId# [5:7577789037735881044:0]
2025-11-28T14:51:39.574426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789037735881049:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.574517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:39.574579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:39.574954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:39.575139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789040381691373:2050] [node 3] ICS07 socket disconnect 210 reason# EndOfStream
2025-11-28T14:51:39.575168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789040381691373:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.575235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:39.575274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789040381691373:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:39.576398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789040381691285:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.576451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789040381691373:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.576482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789040381691285:2050] [node 3] ICP30 unregister session Session# [6:7577789040381691373:2050] VirtualId# [6:7577789040381691364:0]
2025-11-28T14:51:39.576524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:39.576549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789040381691285:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.576562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.587288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789040381691285:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:39.596008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:39.596238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789037735881053:2108] [node 4] ICS07 socket disconnect 206 reason# EndOfStream
2025-11-28T14:51:39.596271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789037735881053:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.596356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:39.596425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789037735881053:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:39.596999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:39.597162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789040381691376:2106] [node 4] ICS07 socket disconnect 211 reason# EndOfStream
2025-11-28T14:51:39.597195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789040381691376:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.597259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:39.597285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789040381691376:2106] [node 4] ICS15 start handshake
2025-11-28T14:51:39.597246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789037735881023:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13264) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.597306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789037735881053:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.597342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789037735881023:2108] [node 4] ICP30 unregister session Session# [5:7577789037735881053:2108] VirtualId# [5:7577789037735881045:0]
2025-11-28T14:51:39.597381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:39.597436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789037735881023:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.597452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.597983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789040381691349:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13264) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:39.598020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789040381691376:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:39.598042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789040381691349:2106] [node 4] ICP30 unregister session Session# [6:7577789040381691376:2106] VirtualId# [6:7577789040381691365:0]
2025-11-28T14:51:39.598071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:39.598088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789040381691349:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.598098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:39.607716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789040381691349:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:51:39.620976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:39.621200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789040381691386:2105] [node 5] ICS01 socket: 181 reason# EndOfStream
2025-11-28T14:51:39.621253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789040381691342:2105] [node 5] ICP30 unregister session Session# [6:7577789040381691386:2105] VirtualId# [6:7577789040381691384:0]
2025-11-28T14:51:39.621289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789040381691386:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:39.621399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:39.621465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:39.707092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789039139135624:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:39.707094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789039139135623:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:39.724503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789039187491637:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.724517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789039187491638:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:39.742422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789040188125923:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.742450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789040188125924:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:39.760291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789040534840282:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.760302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789040534840283:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:39.760333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789040534840284:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:39.779334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789038751367939:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.779368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789038751367941:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:39.779456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789038751367940:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:39.797980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789039253553126:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:39.798026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789039253553127:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:39.798055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789039253553128:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:39.854662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789039253553188:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:39.901940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789039187491710:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:39.901964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789038751368002:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:39.902135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789038751368003:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:39.902369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789039187491711:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:39.902788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789039187491712:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:39.903053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789039139135623:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49627068ns) {Configure:104(49622682ns) {ProcessConfigured:135(184900ns) {StartInitialHandshake:157(182578ns) {DropHandshakes:470(2556ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(563ns)} GenerateSessionVirtualId:988(602ns) PrepareNewSessionHandshake:477(1847ns) {SwitchToState:179(813ns)}}}}}
2025-11-28T14:51:39.903305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789039139135689:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:39.903529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789039139135691:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:39.903745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789039139135693:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:39.903776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789039187491637:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46334099ns) {Configure:104(46327375ns) {ProcessConfigured:135(43481ns) {StartInitialHandshake:157(41492ns) {DropHandshakes:470(1354ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(512ns) PrepareNewSessionHandshake:477(933ns) {SwitchToState:179(329ns)}}}}}
2025-11-28T14:51:39.903819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789039139135624:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48858423ns) {Configure:104(48854606ns) {ProcessConfigured:135(173297ns) {StartInitialHandshake:157(171476ns) {DropHandshakes:470(6081ns) {DropIncomingHandshake:442(409ns) DropOutgoingHandshake:457(508ns)} GenerateSessionVirtualId:988(1067ns) PrepareNewSessionHandshake:477(855ns) {SwitchToState:179(414ns)}}}}}
2025-11-28T14:51:39.904305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789040188125924:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43842510ns) {Configure:104(43838965ns) {ProcessConfigured:135(48166ns) {StartInitialHandshake:157(44251ns) {DropHandshakes:470(1227ns) {DropIncomingHandshake:442(390ns) DropOutgoingHandshake:457(174ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:477(864ns) {SwitchToState:179(400ns)}}}}}
2025-11-28T14:51:39.904628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789040188125923:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44934437ns) {Configure:104(44932203ns) {ProcessConfigured:135(48297ns) {StartInitialHandshake:157(46119ns) {DropHandshakes:470(1224ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(662ns) PrepareNewSessionHandshake:477(964ns) {SwitchToState:179(520ns)}}}}}
2025-11-28T14:51:39.904698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789040188125990:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:39.905028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789040188125991:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:39.905194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789040188125992:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:39.905340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789039253553188:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49595833ns) {Configure:104(49592767ns) {ProcessConfigured:135(125741ns) {StartInitialHandshake:157(123775ns) {DropHandshakes:470(1171ns) {DropIncomingHandshake:442(387ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(569ns) PrepareNewSessionHandshake:477(804ns) {SwitchToState:179(390ns)}}}}}
2025-11-28T14:51:39.905501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789039253553126:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38924915ns) {Configure:104(38921271ns) {ProcessConfigured:135(192129ns) {StartInitialHandshake:157(130968ns) {DropHandshakes:470(1496ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(771ns) PrepareNewSessionHandshake:477(955ns) {SwitchToState:179(437ns)}}}}}
2025-11-28T14:51:39.906208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789040534840282:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42213935ns) {Configure:104(42202672ns) {ProcessConfigured:135(267498ns) {StartInitialHandshake:157(265232ns) {DropHandshakes:470(1089ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(146ns)} GenerateSessionVirtualId:988(368ns) PrepareNewSessionHandshake:477(1844ns) {SwitchToState:179(804ns)}}}}}
2025-11-28T14:51:39.905926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789039253553194:2109] [node 4] ICP01 ready to work
2025-11-28T14:51:39.907252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789040534840361:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:39.907920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789040534840362:2121] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.910800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789039139135703:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15922
2025-11-28T14:51:39.911077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789039187491724:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.917300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789038751368012:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2122
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.918645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789040188125997:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3907
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.918930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789040188125998:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3907
2025-11-28T14:51:39.917728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789039139135704:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15922
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.917738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789039187491724:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.917854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789039139135705:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15922
2025-11-28T14:51:39.919750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789040534840367:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20555
2025-11-28T14:51:39.920692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789040188126003:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3907
2025-11-28T14:51:39.920167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789038751368013:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2122
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.921110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789038751368012:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.921234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789040188125997:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.921491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135703:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.921919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789038751367939:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.922055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789040534840373:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20555
2025-11-28T14:51:39.921881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789040188125998:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:51:39.921936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789039139135706:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15922
::ffff:127.1.0.0
2025-11-28T14:51:39.922027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789039139135707:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15922
2025-11-28T14:51:39.923613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135695:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.924870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789040188126004:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3907
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.926109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789039187491637:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.925440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789040188125923:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.923638Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789039139135695:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.927314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789039187491728:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.925750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789039139135695:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:39.927334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789040188125924:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.927638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789039187491729:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
2025-11-28T14:51:39.927633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789040534840367:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.928664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789039187491727:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
2025-11-28T14:51:39.928132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789040188126003:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.928706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789040534840283:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.930713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135692:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.929237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789039187491716:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.938269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789039253553204:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
2025-11-28T14:51:39.930746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789039139135692:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.929261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789039187491716:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.929387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789039187491718:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.933579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135706:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.938562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789040188126004:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.938117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135704:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.929409Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789039187491718:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.934549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789039187491718:2122] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.938157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789039187491716:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:39.938550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135688:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.941016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789039187491638:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.938582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789039139135688:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.939067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789039139135691:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.941059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:39.939102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:39.941042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789040534840375:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20555
2025-11-28T14:51:39.939221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789039139135691:2109] [node 5] ICP22 created new session: [1:7577789039139135710:2109]
2025-11-28T14:51:39.941177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789039187491710:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.941179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789039187491638:2049] [node 3] ICP22 created new session: [2:7577789039187491733:2049]
2025-11-28T14:51:39.940962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789039139135710:2109] [node 5] ICS09 handshake done sender: [1:7577789039139135695:2113] self: [1:7577789039139135694:0] peer: [5:7577789038751368019:0] socket: 165 qp: -1
2025-11-28T14:51:39.940988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789039139135710:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:39.941050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:39.941207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789039187491733:2049] [node 3] ICS09 handshake done sender: [2:7577789039187491716:2121] self: [2:7577789039187491715:0] peer: [3:7577789040188126007:0] socket: 166 qp: -1
2025-11-28T14:51:39.941216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:39.941244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789039187491733:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:39.941271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:39.941308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789039187491710:2107] [node 4] ICP22 created new session: [2:7577789039187491734:2107]
2025-11-28T14:51:39.941403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789039187491734:2107] [node 4] ICS09 handshake done sender: [2:7577789039187491718:2122] self: [2:7577789039187491717:0] peer: [4:7577789040534840374:0] socket: 167 qp: -1
2025-11-28T14:51:39.942134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789039139135692:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:39.941433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789039187491734:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:39.941466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:39.942226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789040534840367:2126] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:39.942805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789040534840283:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.942833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789039139135688:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:39.942862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:39.943116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789039139135624:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.943165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:39.942913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789040534840283:2049] [node 2] ICP22 created new session: [4:7577789040534840376:2049]
2025-11-28T14:51:39.944151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789039139135624:2049] [node 3] ICP22 created new session: [1:7577789039139135717:2049]
2025-11-28T14:51:39.944160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789039139135623:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.943144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789040534840376:2049] [node 2] ICS09 handshake done sender: [4:7577789040534840367:2126] self: [4:7577789040534840374:0] peer: [2:7577789039187491717:0] socket: 175 qp: -1
2025-11-28T14:51:39.944197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789039139135717:2049] [node 3] ICS09 handshake done sender: [1:7577789039139135692:2112] self: [1:7577789039139135690:0] peer: [3:7577789040188126006:0] socket: 164 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.944236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:39.943256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789040534840376:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:39.944269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789039139135623:2048] [node 2] ICP22 created new session: [1:7577789039139135718:2048]
2025-11-28T14:51:39.944056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:39.944303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789039139135718:2048] [node 2] ICS09 handshake done sender: [1:7577789039139135688:2111] self: [1:7577789039139135687:0] peer: [2:7577789039187491730:0] socket: 163 qp: -1
2025-11-28T14:51:39.944305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789039139135717:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:39.944571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789039187491737:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
2025-11-28T14:51:39.944326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789039139135718:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:39.944409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:39.944418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.944980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789039253553205:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.945638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789040534840378:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20555
2025-11-28T14:51:39.945743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789039187491724:2124] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.946649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789039253553206:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
2025-11-28T14:51:39.948774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789039187491637:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.948839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.948889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789039187491637:2048] [node 1] ICP22 created new session: [2:7577789039187491740:2048]
2025-11-28T14:51:39.948930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789039187491740:2048] [node 1] ICS09 handshake done sender: [2:7577789039187491724:2124] self: [2:7577789039187491730:0] peer: [1:7577789039139135687:0] socket: 168 qp: -1
2025-11-28T14:51:39.948914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789040534840378:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.948974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789039187491740:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:39.949004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:39.949090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789039253553205:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.949107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789039253553206:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.949046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789040534840282:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.949232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789039253553127:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.949251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789039253553126:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:39.949279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789039187491637:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:39.949251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135697:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.949276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789039139135697:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.949482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789039187491726:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.949528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135702:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.949518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789039187491726:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.949561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789039139135702:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:39.950660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789039139135702:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:39.950726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789039187491726:2125] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.951006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789039187491712:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.951051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.951196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789039187491712:2110] [node 6] ICP22 created new session: [2:7577789039187491743:2110]
2025-11-28T14:51:39.951287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789039187491743:2110] [node 6] ICS09 handshake done sender: [2:7577789039187491726:2125] self: [2:7577789039187491725:0] peer: [6:7577789039253553207:0] socket: 215 qp: -1
2025-11-28T14:51:39.951314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789039253553209:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
2025-11-28T14:51:39.951500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789039253553210:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
2025-11-28T14:51:39.951320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789039187491743:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:39.951533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:39.951559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789039139135693:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.951635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:39.951685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789039139135693:2110] [node 6] ICP22 created new session: [1:7577789039139135724:2110]
2025-11-28T14:51:39.951736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789039139135724:2110] [node 6] ICS09 handshake done sender: [1:7577789039139135702:2115] self: [1:7577789039139135700:0] peer: [6:7577789039253553208:0] socket: 169 qp: -1
2025-11-28T14:51:39.951783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789039139135724:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:39.951829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:39.953210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789039139135697:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.953137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789039253553205:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:39.953912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789039253553206:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:39.953992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789040534840380:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20555
2025-11-28T14:51:39.954219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789039139135689:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:39.954452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789039253553127:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.954297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:39.954605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:39.954339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789039139135689:2107] [node 4] ICP22 created new session: [1:7577789039139135726:2107]
2025-11-28T14:51:39.954754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789039253553127:2049] [node 2] ICP22 created new session: [6:7577789039253553211:2049]
2025-11-28T14:51:39.954609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789040534840378:2130] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:39.954594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789039139135726:2107] [node 4] ICS09 handshake done sender: [1:7577789039139135697:2114] self: [1:7577789039139135696:0] peer: [4:7577789040534840379:0] socket: 212 qp: -1
2025-11-28T14:51:39.954804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789039253553211:2049] [node 2] ICS09 handshake done sender: [6:7577789039253553205:2116] self: [6:7577789039253553207:0] peer: [2:7577789039187491725:0] socket: 216 qp: -1
2025-11-28T14:51:39.954833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789039253553211:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:39.954858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:39.954628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789039139135726:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:39.954723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:39.955414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789040534840282:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.955463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.955504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789040534840282:2048] [node 1] ICP22 created new session: [4:7577789040534840381:2048]
2025-11-28T14:51:39.955547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789040534840381:2048] [node 1] ICS09 handshake done sender: [4:7577789040534840378:2130] self: [4:7577789040534840379:0] peer: [1:7577789039139135696:0] socket: 217 qp: -1
2025-11-28T14:51:39.955580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789040534840381:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:39.955686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:39.956004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789039253553126:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:39.956078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:39.956119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789039253553126:2048] [node 1] ICP22 created new session: [6:7577789039253553213:2048]
2025-11-28T14:51:39.956176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789039253553213:2048] [node 1] ICS09 handshake done sender: [6:7577789039253553206:2117] self: [6:7577789039253553208:0] peer: [1:7577789039139135700:0] socket: 218 qp: -1
2025-11-28T14:51:39.956206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789039253553213:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:39.956256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:39.956579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789039253553126:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:39.959472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789040534840360:2123] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:39.968396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789039187491729:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.968604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789039187491710:2107] [node 4] ICP18 handshake (actor [2:7577789039187491729:2128]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.969181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789040534840383:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20555
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:39.969784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789039253553216:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
2025-11-28T14:51:39.988021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789038751368009:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:39.996801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789040534840373:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:39.998749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789040188125987:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.008703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789039187491727:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.008903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789039187491638:2049] [node 3] ICP18 handshake (actor [2:7577789039187491727:2126]) is too old
2025-11-28T14:51:40.028511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789038751368005:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.037083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789038751368013:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.037091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789039187491728:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.037543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789038751368020:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2122
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.038132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789043046335317:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2122
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.038288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789043046335318:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2122
2025-11-28T14:51:40.038397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789038751368012:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:40.038682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789043046335317:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.038725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789038751367939:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.038771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:40.038807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789038751367939:2048] [node 1] ICP22 created new session: [5:7577789043046335320:2048]
2025-11-28T14:51:40.038839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789043046335320:2048] [node 1] ICS09 handshake done sender: [5:7577789038751368012:2114] self: [5:7577789038751368019:0] peer: [1:7577789039139135694:0] socket: 172 qp: -1
2025-11-28T14:51:40.038871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789043046335320:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:40.038904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:40.039523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789043046335318:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.039766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789038751367940:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.039844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789038751368002:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.040084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789039187491720:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.040118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789039187491720:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:40.040344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789040534840366:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.040396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789040534840366:2125] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.041570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789039187491720:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:40.041637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789043046335324:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2122
2025-11-28T14:51:40.041747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789040534840366:2125] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:40.041878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789039187491711:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.041982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:40.042033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789039187491711:2109] [node 5] ICP22 created new session: [2:7577789043482459043:2109]
2025-11-28T14:51:40.042073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789043482459043:2109] [node 5] ICS09 handshake done sender: [2:7577789039187491720:2123] self: [2:7577789039187491719:0] peer: [5:7577789043046335322:0] socket: 214 qp: -1
2025-11-28T14:51:40.042362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789043046335317:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:40.042114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789043482459043:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:40.042443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789040534840361:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.042157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:40.042501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:40.042704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789038751367940:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.042551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789040534840361:2108] [node 5] ICP22 created new session: [4:7577789044829807684:2108]
2025-11-28T14:51:40.042738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:40.042600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789044829807684:2108] [node 5] ICS09 handshake done sender: [4:7577789040534840366:2125] self: [4:7577789040534840365:0] peer: [5:7577789043046335323:0] socket: 185 qp: -1
2025-11-28T14:51:40.042784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789038751367940:2049] [node 2] ICP22 created new session: [5:7577789043046335326:2049]
2025-11-28T14:51:40.042824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789043046335326:2049] [node 2] ICS09 handshake done sender: [5:7577789043046335317:2118] self: [5:7577789043046335322:0] peer: [2:7577789039187491719:0] socket: 191 qp: -1
2025-11-28T14:51:40.042700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789044829807684:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:40.042857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789043046335326:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:40.042737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:40.042880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.043746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789043046335325:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2122
2025-11-28T14:51:40.044261Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789038751367940:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:40.044380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789038751368002:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19280) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789043046335318:2119] held: no
2025-11-28T14:51:40.044526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789043046335318:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:40.044759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789038751368002:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.044785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:40.044805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789038751368002:2108] [node 4] ICP22 created new session: [5:7577789043046335330:2108]
2025-11-28T14:51:40.044828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789043046335330:2108] [node 4] ICS09 handshake done sender: [5:7577789043046335318:2119] self: [5:7577789043046335323:0] peer: [4:7577789040534840365:0] socket: 198 qp: -1
2025-11-28T14:51:40.044848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789043046335330:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:40.044864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.045897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789043548520513:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
2025-11-28T14:51:40.055538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789040188125989:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.064122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135705:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.064315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789039139135624:2049] [node 3] ICP18 handshake (actor [1:7577789039139135705:2118]) is too old
2025-11-28T14:51:40.065378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789040188125924:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789040188125998:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.066109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789040188126009:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3907
2025-11-28T14:51:40.066617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789040188125923:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789040188125997:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.066820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789044483093306:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3907
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.067148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789044483093307:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3907
2025-11-28T14:51:40.067363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789040188125998:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:40.067763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789040188125924:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.067813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:40.067851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789040188125924:2049] [node 2] ICP22 created new session: [3:7577789044483093311:2049]
2025-11-28T14:51:40.067893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789044483093311:2049] [node 2] ICS09 handshake done sender: [3:7577789040188125998:2115] self: [3:7577789040188126007:0] peer: [2:7577789039187491715:0] socket: 173 qp: -1
2025-11-28T14:51:40.067930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789044483093311:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:40.067962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.068094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789044829807686:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20555
2025-11-28T14:51:40.068378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789040188125997:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:40.068691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789040188125923:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.068734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:40.068763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789040188125923:2048] [node 1] ICP22 created new session: [3:7577789044483093313:2048]
2025-11-28T14:51:40.068796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789044483093313:2048] [node 1] ICS09 handshake done sender: [3:7577789040188125997:2114] self: [3:7577789040188126006:0] peer: [1:7577789039139135690:0] socket: 171 qp: -1
2025-11-28T14:51:40.068818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789044483093313:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:40.068844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:40.078803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789039253553202:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.086663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789040534840383:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.103436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789040534840371:2127] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.111544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789039253553216:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.111762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789039253553194:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.111961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789040534840371:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.112467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789040534840371:2127] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.112598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789043548520515:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
2025-11-28T14:51:40.112634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789040534840362:2121] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.112748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:40.112805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789040534840362:2121] [node 6] ICP22 created new session: [4:7577789044829807689:2121]
2025-11-28T14:51:40.112877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789044829807689:2121] [node 6] ICS09 handshake done sender: [4:7577789040534840371:2127] self: [4:7577789040534840370:0] peer: [6:7577789043548520514:0] socket: 188 qp: -1
2025-11-28T14:51:40.112913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789044829807689:2121] [node 6] ICS10 traffic start
2025-11-28T14:51:40.112879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789039253553216:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:40.113020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:40.113310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789039253553194:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.113512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:40.113552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789039253553194:2109] [node 4] ICP22 created new session: [6:7577789043548520516:2109]
2025-11-28T14:51:40.113597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789043548520516:2109] [node 4] ICS09 handshake done sender: [6:7577789039253553216:2120] self: [6:7577789043548520514:0] peer: [4:7577789040534840370:0] socket: 192 qp: -1
2025-11-28T14:51:40.113635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789043548520516:2109] [node 4] ICS10 traffic start
2025-11-28T14:51:40.113668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:40.113881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789039253553194:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:40.117562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789038751368007:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.118440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789039139135707:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.118634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789039139135691:2109] [node 5] ICP18 handshake (actor [1:7577789039139135707:2120]) is too old
2025-11-28T14:51:40.142484Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789040188125994:2112] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.161739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789039253553204:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.161968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789039253553128:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.162197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789040188125994:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.162709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789040188125994:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.163041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789043548520519:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
2025-11-28T14:51:40.163334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789040188125991:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.163401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:40.163423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789039253553204:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:40.163458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789040188125991:2109] [node 6] ICP22 created new session: [3:7577789044483093316:2109]
2025-11-28T14:51:40.163510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789044483093316:2109] [node 6] ICS09 handshake done sender: [3:7577789040188125994:2112] self: [3:7577789040188125993:0] peer: [6:7577789043548520518:0] socket: 202 qp: -1
2025-11-28T14:51:40.163634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789044483093316:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:40.163781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:40.163822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789039253553128:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.163880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:40.163939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789039253553128:2050] [node 3] ICP22 created new session: [6:7577789043548520520:2050]
2025-11-28T14:51:40.164002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789043548520520:2050] [node 3] ICS09 handshake done sender: [6:7577789039253553204:2115] self: [6:7577789043548520518:0] peer: [3:7577789040188125993:0] socket: 204 qp: -1
2025-11-28T14:51:40.164046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789043548520520:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:40.164084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:40.166610Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789038751368018:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.175061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789043548520513:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.175249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789039253553188:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.175935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789038751368018:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.176148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789043046335332:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2122
2025-11-28T14:51:40.176610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789038751368018:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.177140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789043548520523:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31828
2025-11-28T14:51:40.177333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789038751368003:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.177409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:40.177420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789043548520513:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:40.177453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789038751368003:2109] [node 6] ICP22 created new session: [5:7577789043046335334:2109]
2025-11-28T14:51:40.177575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789043046335334:2109] [node 6] ICS09 handshake done sender: [5:7577789038751368018:2116] self: [5:7577789038751368017:0] peer: [6:7577789043548520522:0] socket: 190 qp: -1
2025-11-28T14:51:40.177680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789043046335334:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:40.177716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789039253553188:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.177721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:40.177778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:40.177835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789039253553188:2106] [node 5] ICP22 created new session: [6:7577789043548520524:2106]
2025-11-28T14:51:40.177902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789043548520524:2106] [node 5] ICS09 handshake done sender: [6:7577789043548520513:2121] self: [6:7577789043548520522:0] peer: [5:7577789038751368017:0] socket: 195 qp: -1
2025-11-28T14:51:40.177945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789043548520524:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:40.177985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:40.181526Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789040188125996:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.193528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789044829807686:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.193802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789040534840284:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.194201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789040188125996:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.195500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789040188125996:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:40.195514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789038751368011:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.204358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789044829807692:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20555
2025-11-28T14:51:40.204452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789044483093307:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.204525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789040188125990:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.204596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:40.204666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789040188125990:2107] [node 4] ICP22 created new session: [3:7577789044483093319:2107]
2025-11-28T14:51:40.204714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789044483093319:2107] [node 4] ICS09 handshake done sender: [3:7577789040188125996:2113] self: [3:7577789040188125995:0] peer: [4:7577789044829807691:0] socket: 181 qp: -1
2025-11-28T14:51:40.204754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789044483093319:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:40.204744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789044829807686:2133] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:40.204890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:40.205257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789040534840284:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.205319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:40.205371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789040534840284:2050] [node 3] ICP22 created new session: [4:7577789044829807693:2050]
2025-11-28T14:51:40.205424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789044829807693:2050] [node 3] ICS09 handshake done sender: [4:7577789044829807686:2133] self: [4:7577789044829807691:0] peer: [3:7577789040188125995:0] socket: 183 qp: -1
2025-11-28T14:51:40.205453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789044829807693:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:40.205487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:40.211152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789040188126002:2116] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.212109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789043046335332:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.212274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789038751367941:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.212530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789040188126002:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.213019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789040188126002:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.213141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789043046335337:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2122
2025-11-28T14:51:40.213222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789040188125992:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.213291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:40.213337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789043046335332:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:40.213375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789040188125992:2108] [node 5] ICP22 created new session: [3:7577789044483093323:2108]
2025-11-28T14:51:40.213438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789044483093323:2108] [node 5] ICS09 handshake done sender: [3:7577789040188126002:2116] self: [3:7577789040188126001:0] peer: [5:7577789043046335336:0] socket: 182 qp: -1
2025-11-28T14:51:40.213470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789044483093323:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:40.213612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:40.213550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789038751367941:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.213865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:40.213995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789038751367941:2050] [node 3] ICP22 created new session: [5:7577789043046335338:2050]
2025-11-28T14:51:40.214041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789043046335338:2050] [node 3] ICS09 handshake done sender: [5:7577789043046335332:2122] self: [5:7577789043046335336:0] peer: [3:7577789040188126001:0] socket: 179 qp: -1
2025-11-28T14:51:40.214074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789043046335338:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:40.214158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:40.251891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:40.252129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789039187491740:2048] [node 1] ICS07 socket disconnect 168 reason# EndOfStream
2025-11-28T14:51:40.252166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789039187491740:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.252261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:40.252317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789039187491740:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:40.253086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:40.253291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789044483093313:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-11-28T14:51:40.253336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789040188125923:2048] [node 1] ICP30 unregister session Session# [3:7577789044483093313:2048] VirtualId# [3:7577789040188126006:0]
2025-11-28T14:51:40.253376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789044483093313:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.253450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:40.253496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:40.254588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789039187491637:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:40.254637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789039187491740:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:40.256450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789039187491637:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.256507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789039187491740:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.256537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789039187491637:2048] [node 1] ICP30 unregister session Session# [2:7577789039187491740:2048] VirtualId# [2:7577789039187491730:0]
2025-11-28T14:51:40.256583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:40.256626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789039187491637:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.256670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.256724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:40.256970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789040534840381:2048] [node 1] ICS07 socket disconnect 217 reason# EndOfStream
2025-11-28T14:51:40.257005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789040534840381:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.257086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:40.257136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789040534840381:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:40.257885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789040534840282:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.257850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:40.257972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789040534840381:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.258098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789040534840282:2048] [node 1] ICP30 unregister session Session# [4:7577789040534840381:2048] VirtualId# [4:7577789040534840379:0]
2025-11-28T14:51:40.258147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:40.258188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789040534840282:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.258201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.258273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789043046335320:2048] [node 1] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-28T14:51:40.258306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789043046335320:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.258389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:40.258456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789043046335320:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:40.258998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:40.259092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789038751367939:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.259136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789043046335320:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.259206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789038751367939:2048] [node 1] ICP30 unregister session Session# [5:7577789043046335320:2048] VirtualId# [5:7577789038751368019:0]
2025-11-28T14:51:40.259261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:40.259236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789039253553213:2048] [node 1] ICS07 socket disconnect 218 reason# EndOfStream
2025-11-28T14:51:40.259289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789038751367939:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.259257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789039253553213:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.259300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.259318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:40.259350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789039253553213:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:40.259925Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789039253553126:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.260000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789039253553213:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.260025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789039253553126:2048] [node 1] ICP30 unregister session Session# [6:7577789039253553213:2048] VirtualId# [6:7577789039253553208:0]
2025-11-28T14:51:40.260056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:40.260088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789039253553126:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.260430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.268219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789040534840282:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:40.269210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789038751367939:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:40.270223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789039253553126:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:40.295556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:40.295756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789044483093311:2049] [node 2] ICS01 socket: 173 reason# EndOfStream
2025-11-28T14:51:40.295788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789040188125924:2049] [node 2] ICP30 unregister session Session# [3:7577789044483093311:2049] VirtualId# [3:7577789040188126007:0]
2025-11-28T14:51:40.295811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789044483093311:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.295891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:40.295932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:40.299499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:40.299639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789040534840376:2049] [node 2] ICS07 socket disconnect 175 reason# EndOfStream
2025-11-28T14:51:40.299664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789040534840376:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.299716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:40.299757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789040534840376:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:40.300431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789040534840283:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.300504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789040534840376:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.300540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789040534840283:2049] [node 2] ICP30 unregister session Session# [4:7577789040534840376:2049] VirtualId# [4:7577789040534840374:0]
2025-11-28T14:51:40.300585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:40.300616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789040534840283:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.300632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.300610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:40.300728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789043046335326:2049] [node 2] ICS07 socket disconnect 191 reason# EndOfStream
2025-11-28T14:51:40.300744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789043046335326:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.300784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:40.300808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789043046335326:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:40.301391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789038751367940:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.301423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789043046335326:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.301442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789038751367940:2049] [node 2] ICP30 unregister session Session# [5:7577789043046335326:2049] VirtualId# [5:7577789043046335322:0]
2025-11-28T14:51:40.301473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:40.301491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789038751367940:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.301502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.301624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:40.301729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789039253553211:2049] [node 2] ICS07 socket disconnect 216 reason# EndOfStream
2025-11-28T14:51:40.301742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789039253553211:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.301782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:40.301806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789039253553211:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:40.302330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789039253553127:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.302357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789039253553211:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.302426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789039253553127:2049] [node 2] ICP30 unregister session Session# [6:7577789039253553211:2049] VirtualId# [6:7577789039253553207:0]
2025-11-28T14:51:40.302459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:40.302477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789039253553127:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.302483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.310219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789040534840283:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:40.311156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789038751367940:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:40.312232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789039253553127:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:40.313535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:40.313701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789044829807693:2050] [node 3] ICS01 socket: 183 reason# EndOfStream
2025-11-28T14:51:40.313747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789040534840284:2050] [node 3] ICP30 unregister session Session# [4:7577789044829807693:2050] VirtualId# [4:7577789044829807691:0]
2025-11-28T14:51:40.313775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789044829807693:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.313844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:40.313878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:40.314560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:40.314683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789043046335338:2050] [node 3] ICS07 socket disconnect 179 reason# EndOfStream
2025-11-28T14:51:40.314703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789043046335338:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.314745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:40.314774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789043046335338:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:40.315422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789038751367941:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17167) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.315459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789043046335338:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.315511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789038751367941:2050] [node 3] ICP30 unregister session Session# [5:7577789043046335338:2050] VirtualId# [5:7577789043046335336:0]
2025-11-28T14:51:40.315549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:40.315569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789038751367941:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.315583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.315585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:40.315873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789043548520520:2050] [node 3] ICS07 socket disconnect 204 reason# EndOfStream
2025-11-28T14:51:40.315905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789043548520520:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.315971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:40.316012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789043548520520:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:40.316604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789039253553128:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17167) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.316699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789043548520520:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.316727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789039253553128:2050] [node 3] ICP30 unregister session Session# [6:7577789043548520520:2050] VirtualId# [6:7577789043548520518:0]
2025-11-28T14:51:40.316764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:40.316788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789039253553128:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.316807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.325515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789038751367941:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:40.326569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789039253553128:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:40.339414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:40.339643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789043046335330:2108] [node 4] ICS01 socket: 198 reason# EndOfStream
2025-11-28T14:51:40.339687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789038751368002:2108] [node 4] ICP30 unregister session Session# [5:7577789043046335330:2108] VirtualId# [5:7577789043046335323:0]
2025-11-28T14:51:40.339722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789043046335330:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.339805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:40.339853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:40.340346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:40.340529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789043548520516:2109] [node 4] ICS07 socket disconnect 192 reason# EndOfStream
2025-11-28T14:51:40.340561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789043548520516:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.340624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:40.340666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789043548520516:2109] [node 4] ICS15 start handshake
2025-11-28T14:51:40.341259Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789039253553194:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19280) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.341300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789043548520516:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.341359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789039253553194:2109] [node 4] ICP30 unregister session Session# [6:7577789043548520516:2109] VirtualId# [6:7577789043548520514:0]
2025-11-28T14:51:40.341389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:40.341415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789039253553194:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19280) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.341424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19280) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.351186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789039253553194:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:51:40.356702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:40.356920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789043548520524:2106] [node 5] ICS07 socket disconnect 195 reason# EndOfStream
2025-11-28T14:51:40.356956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789043548520524:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:40.357027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:40.357070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789043548520524:2106] [node 5] ICS15 start handshake
2025-11-28T14:51:40.357781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789039253553188:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18952) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:40.357856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789043548520524:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:40.357894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789039253553188:2106] [node 5] ICP30 unregister session Session# [6:7577789043548520524:2106] VirtualId# [6:7577789043548520522:0]
2025-11-28T14:51:40.357938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:40.357969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789039253553188:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18952) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:40.357984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18952) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:40.409232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789043599600586:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:40.409285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789043599600587:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:40.427361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789042457963621:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:40.427386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789042457963622:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:40.445581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789045636591363:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:40.445670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789045636591364:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:40.466851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789044190312750:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:40.466875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789044190312751:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:40.466901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789044190312752:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:40.489389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789044648702072:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:40.489422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789044648702074:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:40.489615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789044648702073:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:40.512015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789042132330557:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:40.512022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789042132330558:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:40.512178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789042132330559:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:40.584601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789043599600647:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:40.597482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789042457963683:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:40.608274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789043599600650:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:40.625152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789042457963686:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:40.654965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789043599600651:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:40.681414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789042132330621:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:40.654906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789042457963685:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:40.689203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789044648702135:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:40.689411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789045636591428:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:40.691581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789042132330623:2109] [node 4] ICP01 ready to work
2025-11-28T14:51:40.692695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789042457963621:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(106227677ns) {Configure:104(106212967ns) {ProcessConfigured:135(63923ns) {StartInitialHandshake:157(61097ns) {DropHandshakes:470(2775ns) {DropIncomingHandshake:442(611ns) DropOutgoingHandshake:457(767ns)} GenerateSessionVirtualId:988(1169ns) PrepareNewSessionHandshake:477(1199ns) {SwitchToState:179(348ns)}}}}}
2025-11-28T14:51:40.690294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789045636591429:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:40.690569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789045636591430:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:40.691807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789044190312813:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:40.700487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789044190312814:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:40.692827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789044648702136:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:40.706638Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789042132330559:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(124978258ns) {Configure:104(124968884ns) {ProcessConfigured:135(12353991ns) {StartInitialHandshake:157(12350415ns) {DropHandshakes:470(3218ns) {DropIncomingHandshake:442(898ns) DropOutgoingHandshake:457(480ns)} GenerateSessionVirtualId:988(987ns) PrepareNewSessionHandshake:477(2966ns) {SwitchToState:179(1854ns)}}}}}
2025-11-28T14:51:40.714092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789045636591428:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23241009ns) {Configure:104(23236599ns) {ProcessConfigured:135(71019ns) {StartInitialHandshake:157(67833ns) {DropHandshakes:470(2410ns) {DropIncomingHandshake:442(638ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(1937ns) {SwitchToState:179(757ns)}}}}}
2025-11-28T14:51:40.709324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789043599600586:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(131548007ns) {Configure:104(131540465ns) {ProcessConfigured:135(59885ns) {StartInitialHandshake:157(57361ns) {DropHandshakes:470(2581ns) {DropIncomingHandshake:442(667ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(636ns) PrepareNewSessionHandshake:477(1438ns) {SwitchToState:179(642ns)}}}}}
2025-11-28T14:51:40.729243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789044648702074:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(99343131ns) {Configure:104(91900417ns) {ProcessConfigured:135(65974ns) {StartInitialHandshake:157(63651ns) {DropHandshakes:470(2122ns) {DropIncomingHandshake:442(538ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(947ns) PrepareNewSessionHandshake:477(1064ns) {SwitchToState:179(547ns)}}}}}
2025-11-28T14:51:40.726138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789042457963622:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(96951703ns) {Configure:104(96943294ns) {ProcessConfigured:135(13055983ns) {StartInitialHandshake:157(13052214ns) {DropHandshakes:470(1252ns) {DropIncomingHandshake:442(389ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:477(5108ns) {SwitchToState:179(1772ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.736684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789045636591364:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(118493742ns) {Configure:104(118052512ns) {ProcessConfigured:135(5613078ns) {StartInitialHandshake:157(5610161ns) {DropHandshakes:470(1301ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(479ns) PrepareNewSessionHandshake:477(2661ns) {SwitchToState:179(1171ns)}}}}}
2025-11-28T14:51:40.736819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789045636591363:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(117434170ns) {Configure:104(117427557ns) {ProcessConfigured:135(6222409ns) {StartInitialHandshake:157(6219252ns) {DropHandshakes:470(1714ns) {DropIncomingHandshake:442(664ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:477(3389ns) {SwitchToState:179(1209ns)}}}}}
2025-11-28T14:51:40.751301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789042132330558:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44479967ns) {Configure:104(44475864ns) {ProcessConfigured:135(34634ns) {StartInitialHandshake:157(33052ns) {DropHandshakes:470(5965ns) {DropIncomingHandshake:442(525ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(765ns) PrepareNewSessionHandshake:477(799ns) {SwitchToState:179(389ns)}}}}}
2025-11-28T14:51:40.732788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789044648702072:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32086118ns) {Configure:104(32080556ns) {ProcessConfigured:135(384567ns) {StartInitialHandshake:157(379966ns) {DropHandshakes:470(1199ns) {DropIncomingHandshake:442(353ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(650ns) PrepareNewSessionHandshake:477(1193ns) {SwitchToState:179(600ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.730766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789043599600587:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(147066941ns) {Configure:104(147061722ns) {ProcessConfigured:135(14431050ns) {StartInitialHandshake:157(14428917ns) {DropHandshakes:470(1258ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(518ns) PrepareNewSessionHandshake:477(2241ns) {SwitchToState:179(1003ns)}}}}}
2025-11-28T14:51:40.734883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789044648702143:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23597
2025-11-28T14:51:40.756491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789044648702143:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.761334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789044648702073:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(124051139ns) {Configure:104(124041586ns) {ProcessConfigured:135(8600152ns) {StartInitialHandshake:157(8596916ns) {DropHandshakes:470(1130ns) {DropIncomingHandshake:442(106ns) DropOutgoingHandshake:457(373ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(3434ns) {SwitchToState:179(1610ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.753795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789042132330557:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(94260383ns) {Configure:104(94255102ns) {ProcessConfigured:135(7683220ns) {StartInitialHandshake:157(7679645ns) {DropHandshakes:470(2760ns) {DropIncomingHandshake:442(700ns) DropOutgoingHandshake:457(471ns)} GenerateSessionVirtualId:988(584ns) PrepareNewSessionHandshake:477(2935ns) {SwitchToState:179(1517ns)}}}}}
2025-11-28T14:51:40.750869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789045636591430:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21046437ns) {Configure:104(21042243ns) {ProcessConfigured:135(10143220ns) {StartInitialHandshake:157(10140407ns) {DropHandshakes:470(1963ns) {DropIncomingHandshake:442(684ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(670ns) PrepareNewSessionHandshake:477(3502ns) {SwitchToState:179(1809ns)}}}}}
2025-11-28T14:51:40.765629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789043599600650:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(62067953ns) {Configure:104(62058451ns) {ProcessConfigured:135(18332935ns) {StartInitialHandshake:157(18330451ns) {DropHandshakes:470(889ns) {DropIncomingHandshake:442(234ns) DropOutgoingHandshake:457(178ns)} GenerateSessionVirtualId:988(383ns) PrepareNewSessionHandshake:477(2854ns) {SwitchToState:179(1231ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.766610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789043599600657:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46939
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.757111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789042457963683:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(81458682ns) {Configure:104(81449329ns) {ProcessConfigured:135(10495590ns) {StartInitialHandshake:157(10492758ns) {DropHandshakes:470(3107ns) {DropIncomingHandshake:442(669ns) DropOutgoingHandshake:457(465ns)} GenerateSessionVirtualId:988(733ns) PrepareNewSessionHandshake:477(3455ns) {SwitchToState:179(1490ns)}}}}}
2025-11-28T14:51:40.754638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789045636591441:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6515
2025-11-28T14:51:40.764973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789042457963700:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-11-28T14:51:40.776804Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789044190312750:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(86896999ns) {Configure:104(86879653ns) {ProcessConfigured:135(13772700ns) {StartInitialHandshake:157(13770338ns) {DropHandshakes:470(1584ns) {DropIncomingHandshake:442(395ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(1136ns) PrepareNewSessionHandshake:477(2934ns) {SwitchToState:179(79ns)}}}}}
2025-11-28T14:51:40.781596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789044190312751:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(57295557ns) {Configure:104(57288712ns) {ProcessConfigured:135(13955334ns) {StartInitialHandshake:157(13951318ns) {DropHandshakes:470(1388ns) {DropIncomingHandshake:442(571ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(347ns) PrepareNewSessionHandshake:477(4647ns) {SwitchToState:179(2245ns)}}}}}
2025-11-28T14:51:40.785144Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789044648702135:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55649648ns) {Configure:104(55645649ns) {ProcessConfigured:135(63055ns) {StartInitialHandshake:157(61804ns) {DropHandshakes:470(2925ns) {DropIncomingHandshake:442(562ns) DropOutgoingHandshake:457(584ns)} GenerateSessionVirtualId:988(750ns) PrepareNewSessionHandshake:477(1186ns) {SwitchToState:179(295ns)}}}}}
2025-11-28T14:51:40.785393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789044648702073:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.784533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789043599600659:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46939
2025-11-28T14:51:40.777061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789042132330621:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51886102ns) {Configure:104(51881163ns) {ProcessConfigured:135(21304471ns) {StartInitialHandshake:157(21300507ns) {DropHandshakes:470(2178ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(614ns) PrepareNewSessionHandshake:477(3536ns) {SwitchToState:179(1713ns)}}}}}
2025-11-28T14:51:40.787689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789044190312752:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(83886169ns) {Configure:104(83882237ns) {ProcessConfigured:135(16174386ns) {StartInitialHandshake:157(16170110ns) {DropHandshakes:470(3919ns) {DropIncomingHandshake:442(984ns) DropOutgoingHandshake:457(818ns)} GenerateSessionVirtualId:988(1063ns) PrepareNewSessionHandshake:477(2959503ns) {SwitchToState:179(5787ns)}}}}}
2025-11-28T14:51:40.787649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789043599600657:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.786583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789042132330632:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:40.787664Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789043599600647:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(64708493ns) {Configure:104(64703013ns) {ProcessConfigured:135(10315593ns) {StartInitialHandshake:157(5612285ns) {DropHandshakes:470(4863ns) {DropIncomingHandshake:442(1009ns) DropOutgoingHandshake:457(445ns)} GenerateSessionVirtualId:988(2379ns) PrepareNewSessionHandshake:477(2691ns) {SwitchToState:179(1040ns)}}}}}
2025-11-28T14:51:40.789821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789042457963692:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.786710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789042132330634:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:40.791811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789045636591441:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.786793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789042132330635:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:40.792197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789043599600659:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.791830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789042132330634:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.789871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789042457963692:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.793123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789044648702152:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23597
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.792974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789042457963692:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.792394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789042132330632:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.796992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789045636591445:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6515
2025-11-28T14:51:40.797183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789042132330621:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:51:40.797604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789042457963702:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
::ffff:127.1.0.0
2025-11-28T14:51:40.797741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789042132330558:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.797753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789044648702154:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23597
2025-11-28T14:51:40.797835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789042457963685:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.805487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:40.800482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789044648702153:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23597
2025-11-28T14:51:40.805831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789042457963690:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.805857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789042457963690:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.812470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789043599600651:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18133709ns) {Configure:104(18130093ns) {ProcessConfigured:135(65424ns) {StartInitialHandshake:157(61719ns) {DropHandshakes:470(6889ns) {DropIncomingHandshake:442(637ns) DropOutgoingHandshake:457(300ns)} GenerateSessionVirtualId:988(582ns) PrepareNewSessionHandshake:477(1272ns) {SwitchToState:179(477ns)}}}}}
2025-11-28T14:51:40.817285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789044648702142:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.821172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789042457963685:2108] [node 5] ICP22 created new session: [2:7577789042457963704:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.842985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789042132330641:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:40.817340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789044648702142:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:40.830368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789044190312814:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(35577275ns) {Configure:104(35571106ns) {ProcessConfigured:135(4017048ns) {StartInitialHandshake:157(4013379ns) {DropHandshakes:470(2287ns) {DropIncomingHandshake:442(600ns) DropOutgoingHandshake:457(449ns)} GenerateSessionVirtualId:988(872ns) PrepareNewSessionHandshake:477(3410ns) {SwitchToState:179(1411ns)}}}}}
2025-11-28T14:51:40.826827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789042457963703:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-11-28T14:51:40.830827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789044648702143:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:40.830727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789044190312813:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33855187ns) {Configure:104(33844512ns) {ProcessConfigured:135(4051852ns) {StartInitialHandshake:157(4047907ns) {DropHandshakes:470(3082ns) {DropIncomingHandshake:442(648ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(850ns) PrepareNewSessionHandshake:477(2148ns) {SwitchToState:179(837ns)}}}}}
2025-11-28T14:51:40.852881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789044648702073:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.853172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789044648702142:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:40.826838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789042457963704:2108] [node 5] ICS09 handshake done sender: [2:7577789042457963692:2112] self: [2:7577789042457963691:0] peer: [5:7577789044648702151:0] socket: 170 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.863710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789045636591447:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6515
2025-11-28T14:51:40.853449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789044648702136:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.863591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789042132330632:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:40.830137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789042457963704:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:40.864821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789045636591447:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.853480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.869460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789044190312821:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42843
2025-11-28T14:51:40.869639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789044190312828:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42843
2025-11-28T14:51:40.869755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789044190312827:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42843
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.866659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789045636591364:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.830211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:40.853531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789044648702136:2109] [node 6] ICP22 created new session: [5:7577789044648702156:2109]
2025-11-28T14:51:40.830400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789042457963690:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:40.870857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789044190312828:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.853578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789044648702156:2109] [node 6] ICS09 handshake done sender: [5:7577789044648702142:2112] self: [5:7577789044648702141:0] peer: [6:7577789042132330639:0] socket: 176 qp: -1
2025-11-28T14:51:40.876554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789042132330558:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.876620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:40.876695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789042132330558:2049] [node 2] ICP22 created new session: [6:7577789042132330643:2049]
2025-11-28T14:51:40.876744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789042132330643:2049] [node 2] ICS09 handshake done sender: [6:7577789042132330632:2115] self: [6:7577789042132330640:0] peer: [2:7577789042457963689:0] socket: 179 qp: -1
2025-11-28T14:51:40.876785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789042132330643:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:40.876826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:40.871010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789044190312827:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.836985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789042457963685:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(38928530ns) {HandleHandshakeStatus:329(38926131ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(787ns) SwitchToState:179(297ns) ProcessPendingSessionEvents:495(155688ns) {ForwardSessionEventToSession:612(103806ns) ForwardSessionEventToSession:612(4399ns) ForwardSessionEventToSession:612(2360ns) ForwardSessionEventToSession:612(7993ns) ForwardSessionEventToSession:612(2118ns) ForwardSessionEventToSession:612(1987ns) ForwardSessionEventToSession:612(4672ns) ForwardSessionEventToSession:612(4687ns) ForwardSessionEventToSession:612(8243ns) ForwardSessionEventToSession:612(3055ns)}}}
2025-11-28T14:51:40.853598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789044648702156:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:40.873683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789044190312826:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42843
2025-11-28T14:51:40.874445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789044190312751:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:40.857488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789042457963686:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.853846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:40.857575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:40.853917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:40.857641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789042457963686:2109] [node 6] ICP22 created new session: [2:7577789042457963708:2109]
2025-11-28T14:51:40.861242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789044648702073:2049] [node 2] ICP22 created new session: [5:7577789044648702157:2049]
2025-11-28T14:51:40.857687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789042457963708:2109] [node 6] ICS09 handshake done sender: [2:7577789042457963690:2111] self: [2:7577789042457963689:0] peer: [6:7577789042132330640:0] socket: 169 qp: -1
2025-11-28T14:51:40.861310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789044648702157:2049] [node 2] ICS09 handshake done sender: [5:7577789044648702143:2113] self: [5:7577789044648702151:0] peer: [2:7577789042457963691:0] socket: 173 qp: -1
2025-11-28T14:51:40.857726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789042457963708:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:40.861753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789044648702157:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:40.857787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:40.862185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:40.867362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789042457963711:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-11-28T14:51:40.873666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789042457963697:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.873703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789042457963697:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:40.877974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789042457963699:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.878002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789042457963699:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:40.885581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789042457963697:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.889283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789044190312834:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42843
2025-11-28T14:51:40.885886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789042457963699:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:40.885905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789042457963622:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.885949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:40.886064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789042457963622:2049] [node 3] ICP22 created new session: [2:7577789042457963714:2049]
2025-11-28T14:51:40.888653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789042457963714:2049] [node 3] ICS09 handshake done sender: [2:7577789042457963697:2113] self: [2:7577789042457963696:0] peer: [3:7577789045636591448:0] socket: 206 qp: -1
2025-11-28T14:51:40.888785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789042457963714:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:40.888838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:40.893454Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789045636591438:2115] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.889398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789042457963683:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:40.889571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:40.889763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789042457963683:2107] [node 4] ICP22 created new session: [2:7577789042457963716:2107]
2025-11-28T14:51:40.895628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789044190312827:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:40.896078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789044190312751:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.896285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:40.896345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789044190312751:2049] [node 2] ICP22 created new session: [4:7577789044190312835:2049]
2025-11-28T14:51:40.905512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789044190312835:2049] [node 2] ICS09 handshake done sender: [4:7577789044190312827:2117] self: [4:7577789044190312833:0] peer: [2:7577789042457963698:0] socket: 212 qp: -1
2025-11-28T14:51:40.905584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789044190312835:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:40.905663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.895212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789042457963716:2107] [node 4] ICS09 handshake done sender: [2:7577789042457963699:2114] self: [2:7577789042457963698:0] peer: [4:7577789044190312833:0] socket: 210 qp: -1
2025-11-28T14:51:40.905863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789045636591449:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6515
2025-11-28T14:51:40.895487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789042457963700:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.895620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789042457963716:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:40.895694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:40.896561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789042457963622:2049] [node 3] ICP18 handshake (actor [2:7577789042457963700:2115]) is too old
2025-11-28T14:51:40.905870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789042457963683:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16226354ns) {HandleHandshakeStatus:329(16221760ns) {DropIncomingHandshake:442(648ns) DropOutgoingHandshake:457(552ns) SwitchToState:179(939ns) ProcessPendingSessionEvents:495(9304935ns) {ForwardSessionEventToSession:612(9228873ns) ForwardSessionEventToSession:612(16757ns) ForwardSessionEventToSession:612(3356ns) ForwardSessionEventToSession:612(6576ns) ForwardSessionEventToSession:612(3285ns) ForwardSessionEventToSession:612(6536ns) ForwardSessionEventToSession:612(3050ns) ForwardSessionEventToSession:612(2326ns) ForwardSessionEventToSession:612(5244ns) ForwardSessionEventToSession:612(5287ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.929895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789045636591364:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31669) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789045636591447:2119] held: no
2025-11-28T14:51:40.930143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789045636591450:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6515
2025-11-28T14:51:40.915972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789044190312751:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19817246ns) {HandleHandshakeStatus:329(19814712ns) {DropIncomingHandshake:442(1377ns) DropOutgoingHandshake:457(4977ns) SwitchToState:179(842ns) ProcessPendingSessionEvents:495(9822754ns) {ForwardSessionEventToSession:612(9762935ns) ForwardSessionEventToSession:612(8641ns) ForwardSessionEventToSession:612(11152ns) ForwardSessionEventToSession:612(3348ns) ForwardSessionEventToSession:612(5428ns) ForwardSessionEventToSession:612(4743ns) ForwardSessionEventToSession:612(3644ns) ForwardSessionEventToSession:612(2326ns) ForwardSessionEventToSession:612(4898ns) ForwardSessionEventToSession:612(3293ns)}}}
2025-11-28T14:51:40.932326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789045636591447:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:40.936792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789045636591364:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:40.936857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:40.936911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789045636591364:2049] [node 2] ICP22 created new session: [3:7577789045636591452:2049]
2025-11-28T14:51:40.936958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789045636591452:2049] [node 2] ICS09 handshake done sender: [3:7577789045636591447:2119] self: [3:7577789045636591448:0] peer: [2:7577789042457963696:0] socket: 208 qp: -1
2025-11-28T14:51:40.936993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789045636591452:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:40.937023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:40.938618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789045636591440:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.976274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789042132330629:2113] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:40.986027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789042132330635:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.986471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789042457963702:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:40.986634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789042457963686:2109] [node 6] ICP18 handshake (actor [2:7577789042457963702:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:40.994159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789042132330633:2116] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.018832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789042132330642:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:41.021846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789042132330634:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:41.021964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789042132330559:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.022180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789042132330621:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.022227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:41.022266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789042132330621:2107] [node 5] ICP22 created new session: [6:7577789046427297943:2107]
2025-11-28T14:51:41.022320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789046427297943:2107] [node 5] ICS09 handshake done sender: [6:7577789042132330634:2117] self: [6:7577789042132330639:0] peer: [5:7577789044648702141:0] socket: 180 qp: -1
2025-11-28T14:51:41.022354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789046427297943:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:41.022396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:41.022086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789044648702152:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.022292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789044648702136:2109] [node 6] ICP18 handshake (actor [5:7577789044648702152:2116]) is too old
2025-11-28T14:51:41.023089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789045636591440:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.023478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789049931558750:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6515
2025-11-28T14:51:41.023564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789045636591440:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:41.023782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789045636591430:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:41.023885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.023926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789045636591430:2109] [node 6] ICP22 created new session: [3:7577789049931558752:2109]
2025-11-28T14:51:41.023968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789049931558752:2109] [node 6] ICS09 handshake done sender: [3:7577789045636591440:2116] self: [3:7577789045636591439:0] peer: [6:7577789046427297941:0] socket: 184 qp: -1
2025-11-28T14:51:41.024028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789049931558752:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:41.024048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:41.024115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789046427297945:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:41.024778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789042132330635:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.025592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789042132330559:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.025682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:41.025837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789048943669457:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23597
2025-11-28T14:51:41.025722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789042132330559:2050] [node 3] ICP22 created new session: [6:7577789046427297947:2050]
2025-11-28T14:51:41.026085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789046427297947:2050] [node 3] ICS09 handshake done sender: [6:7577789042132330635:2118] self: [6:7577789046427297941:0] peer: [3:7577789045636591439:0] socket: 187 qp: -1
2025-11-28T14:51:41.026113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789046427297947:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:41.026272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:41.029902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789045636591430:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:41.059730Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789043599600655:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:41.069440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789047894567965:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46939
2025-11-28T14:51:41.069575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789045636591445:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.069795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789045636591363:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.069780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789047894567966:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46939
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.069927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789047894567967:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46939
2025-11-28T14:51:41.070107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789043599600655:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.071008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789043599600655:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:41.071368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789043599600587:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:41.071432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:41.071477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789043599600587:2049] [node 3] ICP22 created new session: [1:7577789047894567969:2049]
2025-11-28T14:51:41.071523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789047894567969:2049] [node 3] ICS09 handshake done sender: [1:7577789043599600655:2111] self: [1:7577789043599600654:0] peer: [3:7577789049931558755:0] socket: 194 qp: -1
2025-11-28T14:51:41.071558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789047894567969:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:41.071599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:41.082905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789043599600653:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.100663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789042457963703:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.100935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789042457963621:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.101156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789043599600653:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.101566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789043599600653:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:41.101771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789043599600586:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.101843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:41.101894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789043599600586:2048] [node 2] ICP22 created new session: [1:7577789047894567973:2048]
2025-11-28T14:51:41.101943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789047894567973:2048] [node 2] ICS09 handshake done sender: [1:7577789043599600653:2110] self: [1:7577789043599600652:0] peer: [2:7577789046752931014:0] socket: 192 qp: -1
2025-11-28T14:51:41.101974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789047894567973:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:41.102015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:41.101931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789046752931015:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-11-28T14:51:41.102207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789042457963703:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.102701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789042457963621:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.102752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:41.102800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789042457963621:2048] [node 1] ICP22 created new session: [2:7577789046752931016:2048]
2025-11-28T14:51:41.102836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789046752931016:2048] [node 1] ICS09 handshake done sender: [2:7577789042457963703:2117] self: [2:7577789046752931014:0] peer: [1:7577789043599600652:0] socket: 199 qp: -1
2025-11-28T14:51:41.103241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789046752931016:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:41.103584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789046427297949:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:41.103278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:41.103646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789042457963621:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:41.108423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789043599600658:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.112692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789044648702153:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.113255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789043599600658:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.116104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789043599600658:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.112938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789044648702072:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.116435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789048943669459:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23597
2025-11-28T14:51:41.116516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789043599600650:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:41.120878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:41.122553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789044648702153:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.121155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789043599600650:2108] [node 5] ICP22 created new session: [1:7577789047894567977:2108]
2025-11-28T14:51:41.128768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789045636591432:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.124584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789047894567977:2108] [node 5] ICS09 handshake done sender: [1:7577789043599600658:2113] self: [1:7577789043599600656:0] peer: [5:7577789048943669458:0] socket: 196 qp: -1
2025-11-28T14:51:41.129993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789049931558756:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6515
2025-11-28T14:51:41.124632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789047894567977:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:41.124699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:41.137979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789045636591445:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:41.136483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789044648702072:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.149162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789045636591363:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.149264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:41.149325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789045636591363:2048] [node 1] ICP22 created new session: [3:7577789049931558757:2048]
2025-11-28T14:51:41.149373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789049931558757:2048] [node 1] ICS09 handshake done sender: [3:7577789045636591445:2118] self: [3:7577789049931558755:0] peer: [1:7577789043599600654:0] socket: 195 qp: -1
2025-11-28T14:51:41.149411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789049931558757:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:41.136641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:41.149448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:41.136704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789044648702072:2048] [node 1] ICP22 created new session: [5:7577789048943669460:2048]
2025-11-28T14:51:41.148482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789048943669460:2048] [node 1] ICS09 handshake done sender: [5:7577789044648702153:2117] self: [5:7577789048943669458:0] peer: [1:7577789043599600656:0] socket: 198 qp: -1
2025-11-28T14:51:41.148569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789048943669460:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:41.148621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:41.170237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789042132330631:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.196203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789044190312817:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.197061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789044190312821:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.197145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789044190312826:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.197169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789042457963711:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.197349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789042457963683:2107] [node 4] ICP18 handshake (actor [2:7577789042457963711:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.198196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789048485280133:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42843
2025-11-28T14:51:41.198355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789044190312752:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.198699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789045636591432:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.200128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789045636591432:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.218486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789048943669462:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23597
2025-11-28T14:51:41.219892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789045636591428:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.220074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:41.220135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789045636591428:2107] [node 4] ICP22 created new session: [3:7577789049931558760:2107]
2025-11-28T14:51:41.220180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789049931558760:2107] [node 4] ICS09 handshake done sender: [3:7577789045636591432:2112] self: [3:7577789045636591431:0] peer: [4:7577789048485280134:0] socket: 182 qp: -1
2025-11-28T14:51:41.220217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789049931558760:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:41.220253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:41.220329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789048485280137:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42843
2025-11-28T14:51:41.229829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789044190312820:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.230487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789044190312821:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:41.231046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789044190312752:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.231101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:41.231148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789044190312752:2050] [node 3] ICP22 created new session: [4:7577789048485280138:2050]
2025-11-28T14:51:41.231190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789048485280138:2050] [node 3] ICS09 handshake done sender: [4:7577789044190312821:2113] self: [4:7577789048485280134:0] peer: [3:7577789045636591431:0] socket: 203 qp: -1
2025-11-28T14:51:41.231218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789048485280138:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:41.231253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:41.230549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789045636591449:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.230746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789045636591428:2107] [node 4] ICP18 handshake (actor [3:7577789045636591449:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.234602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789046427297950:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:41.268407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789045636591434:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.282849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789045636591434:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.283374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789045636591434:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.282260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789048943669457:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.282492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789044648702074:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.283872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789048943669464:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23597
2025-11-28T14:51:41.283857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789045636591429:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:41.283946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:41.283998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789045636591429:2108] [node 5] ICP22 created new session: [3:7577789049931558763:2108]
2025-11-28T14:51:41.290870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789044190312818:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.284069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789049931558763:2108] [node 5] ICS09 handshake done sender: [3:7577789045636591434:2113] self: [3:7577789045636591433:0] peer: [5:7577789048943669463:0] socket: 225 qp: -1
2025-11-28T14:51:41.284101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789049931558763:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:41.291703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789047894567965:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.288336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789048943669457:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:41.284143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:41.303962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789044648702074:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.304061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:41.304118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789044648702074:2050] [node 3] ICP22 created new session: [5:7577789048943669465:2050]
2025-11-28T14:51:41.319849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789048943669465:2050] [node 3] ICS09 handshake done sender: [5:7577789048943669457:2119] self: [5:7577789048943669463:0] peer: [3:7577789045636591433:0] socket: 226 qp: -1
2025-11-28T14:51:41.323578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789048943669465:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:41.323685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:41.324078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789044648702074:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(31340254ns) {HandleHandshakeStatus:329(20046454ns) {DropIncomingHandshake:442(1161ns) DropOutgoingHandshake:457(8494ns) SwitchToState:179(497ns) ProcessPendingSessionEvents:495(92464ns) {ForwardSessionEventToSession:612(56093ns) ForwardSessionEventToSession:612(2216ns) ForwardSessionEventToSession:612(3967ns) ForwardSessionEventToSession:612(3680ns) ForwardSessionEventToSession:612(6879ns) ForwardSessionEventToSession:612(1907ns) ForwardSessionEventToSession:612(1235ns) ForwardSessionEventToSession:612(3510ns) ForwardSessionEventToSession:612(1241ns) ForwardSessionEventToSession:612(1177ns)}}}
2025-11-28T14:51:41.324147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789044648702074:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:41.329035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789042132330626:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.330024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789047894567967:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.362198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789045636591437:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.370124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789047894567966:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.370356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789043599600587:2049] [node 3] ICP18 handshake (actor [1:7577789047894567966:2118]) is too old
2025-11-28T14:51:41.374880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789043599600668:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.397961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789046427297949:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.398197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789042132330557:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.398654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789043599600668:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.399091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789043599600668:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.399288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789043599600651:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:41.399423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789046427297952:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:41.399703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789046427297949:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:41.399354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:41.399443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789043599600651:2109] [node 6] ICP22 created new session: [1:7577789047894567982:2109]
2025-11-28T14:51:41.399529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789047894567982:2109] [node 6] ICS09 handshake done sender: [1:7577789043599600668:2116] self: [1:7577789043599600667:0] peer: [6:7577789046427297951:0] socket: 172 qp: -1
2025-11-28T14:51:41.399825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789042132330557:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.399561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789047894567982:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:41.399599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:41.399953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:41.400055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789042132330557:2048] [node 1] ICP22 created new session: [6:7577789046427297953:2048]
2025-11-28T14:51:41.400228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789046427297953:2048] [node 1] ICS09 handshake done sender: [6:7577789046427297949:2122] self: [6:7577789046427297951:0] peer: [1:7577789043599600667:0] socket: 232 qp: -1
2025-11-28T14:51:41.400589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789046427297953:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:41.400628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:41.402501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789043599600663:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.403912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789048485280133:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.404070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789044190312750:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.404399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789043599600663:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.404764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789043599600663:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.407183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789043599600647:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:41.410818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:41.410798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789048485280142:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42843
2025-11-28T14:51:41.410891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789043599600647:2107] [node 4] ICP22 created new session: [1:7577789047894567985:2107]
2025-11-28T14:51:41.410961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789047894567985:2107] [node 4] ICS09 handshake done sender: [1:7577789043599600663:2115] self: [1:7577789043599600660:0] peer: [4:7577789048485280141:0] socket: 229 qp: -1
2025-11-28T14:51:41.410999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789047894567985:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:41.411063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:41.412321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789048485280133:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:41.412683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789044190312750:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789048485280133:2120] held: no
2025-11-28T14:51:41.412796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789044190312750:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.412859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:41.412912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789044190312750:2048] [node 1] ICP22 created new session: [4:7577789048485280143:2048]
2025-11-28T14:51:41.412972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789048485280143:2048] [node 1] ICS09 handshake done sender: [4:7577789048485280133:2120] self: [4:7577789048485280141:0] peer: [1:7577789043599600660:0] socket: 175 qp: -1
2025-11-28T14:51:41.413014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789048485280143:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:41.413050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:41.415860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789044190312825:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.424102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789048943669462:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.424397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789044648702135:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.424760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789044190312825:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.425364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789044190312825:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.425738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789048943669468:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23597
2025-11-28T14:51:41.427951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789046427297950:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:41.426794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789044190312814:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:41.426702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789048943669462:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:41.430239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789042132330623:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:41.430089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789044648702135:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23463) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789048943669462:2121] held: no
2025-11-28T14:51:41.431066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789044648702135:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.431139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:41.427091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789044190312824:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:41.431196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789044648702135:2108] [node 4] ICP22 created new session: [5:7577789048943669469:2108]
2025-11-28T14:51:41.428598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:41.428665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789044190312814:2108] [node 5] ICP22 created new session: [4:7577789048485280146:2108]
2025-11-28T14:51:41.431251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789048943669469:2108] [node 4] ICS09 handshake done sender: [5:7577789048943669462:2121] self: [5:7577789048943669467:0] peer: [4:7577789044190312823:0] socket: 217 qp: -1
2025-11-28T14:51:41.428717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789048485280146:2108] [node 5] ICS09 handshake done sender: [4:7577789044190312825:2115] self: [4:7577789044190312823:0] peer: [5:7577789048943669467:0] socket: 178 qp: -1
2025-11-28T14:51:41.429260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789048485280146:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:41.431313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789048943669469:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:41.429319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:41.431370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:41.431125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789044190312824:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.432138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789046427297956:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55388
2025-11-28T14:51:41.432553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789044190312824:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:41.433188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789046427297950:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:41.433443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789042132330623:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:41.433497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:41.433550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789042132330623:2109] [node 4] ICP22 created new session: [6:7577789046427297957:2109]
2025-11-28T14:51:41.433613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789046427297957:2109] [node 4] ICS09 handshake done sender: [6:7577789046427297950:2123] self: [6:7577789046427297955:0] peer: [4:7577789044190312822:0] socket: 219 qp: -1
2025-11-28T14:51:41.433648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789046427297957:2109] [node 4] ICS10 traffic start
2025-11-28T14:51:41.433680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:41.435046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789044190312813:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:41.435115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:41.435160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789044190312813:2109] [node 6] ICP22 created new session: [4:7577789048485280149:2109]
2025-11-28T14:51:41.435203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789048485280149:2109] [node 6] ICS09 handshake done sender: [4:7577789044190312824:2114] self: [4:7577789044190312822:0] peer: [6:7577789046427297955:0] socket: 218 qp: -1
2025-11-28T14:51:41.435234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789048485280149:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:41.435266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:41.440556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:41.440775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789046752931016:2048] [node 1] ICS07 socket disconnect 199 reason# EndOfStream
2025-11-28T14:51:41.440802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789046752931016:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.440896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:41.440941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789046752931016:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:41.441205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:41.441520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789049931558757:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-11-28T14:51:41.441568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789045636591363:2048] [node 1] ICP30 unregister session Session# [3:7577789049931558757:2048] VirtualId# [3:7577789049931558755:0]
2025-11-28T14:51:41.441598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789049931558757:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.441678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:41.441724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:41.444923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:41.445204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789048485280143:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-28T14:51:41.445235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789044190312750:2048] [node 1] ICP30 unregister session Session# [4:7577789048485280143:2048] VirtualId# [4:7577789048485280141:0]
2025-11-28T14:51:41.445259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789048485280143:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.445365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:41.445395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:41.446718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789042457963621:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:41.446769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789046752931016:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:41.447476Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:41.447690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789048943669460:2048] [node 1] ICS07 socket disconnect 198 reason# EndOfStream
2025-11-28T14:51:41.447728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789048943669460:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.447808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:41.447853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789048943669460:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:41.449241Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:41.449551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789046427297953:2048] [node 1] ICS07 socket disconnect 232 reason# EndOfStream
2025-11-28T14:51:41.449581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789046427297953:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.449703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789044648702072:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:41.449913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789048943669460:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:41.449966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789044648702072:2048] [node 1] ICP30 unregister session Session# [5:7577789048943669460:2048] VirtualId# [5:7577789048943669458:0]
2025-11-28T14:51:41.450016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:41.450049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789044648702072:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.450060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.449969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:41.450025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789046427297953:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:41.451267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789042457963621:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:41.451458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789046752931016:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:41.452617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789042457963621:2048] [node 1] ICP30 unregister session Session# [2:7577789046752931016:2048] VirtualId# [2:7577789046752931014:0]
2025-11-28T14:51:41.452698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:41.454479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789042457963621:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.454508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.456011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789042132330557:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:41.456171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789046427297953:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:41.456433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789042132330557:2048] [node 1] ICP30 unregister session Session# [6:7577789046427297953:2048] VirtualId# [6:7577789046427297951:0]
2025-11-28T14:51:41.460038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:41.460203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789042132330557:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.460219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23479) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.460222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789044648702072:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:41.470427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789042132330557:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:41.505001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:41.505245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789045636591452:2049] [node 2] ICS01 socket: 208 reason# EndOfStream
2025-11-28T14:51:41.505288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789045636591364:2049] [node 2] ICP30 unregister session Session# [3:7577789045636591452:2049] VirtualId# [3:7577789045636591448:0]
2025-11-28T14:51:41.505319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789045636591452:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.505405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:41.505466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:41.508079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:41.508294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789044190312835:2049] [node 2] ICS01 socket: 212 reason# EndOfStream
2025-11-28T14:51:41.508334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789044190312751:2049] [node 2] ICP30 unregister session Session# [4:7577789044190312835:2049] VirtualId# [4:7577789044190312833:0]
2025-11-28T14:51:41.508373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789044190312835:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.508467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:41.508536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:41.509086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:41.509326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789044648702157:2049] [node 2] ICS07 socket disconnect 173 reason# EndOfStream
2025-11-28T14:51:41.509363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789044648702157:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.509441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:41.509501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789044648702157:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:41.509963Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:41.510232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789044648702073:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31669) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:41.510166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789042132330643:2049] [node 2] ICS01 socket: 179 reason# EndOfStream
2025-11-28T14:51:41.510210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789042132330558:2049] [node 2] ICP30 unregister session Session# [6:7577789042132330643:2049] VirtualId# [6:7577789042132330640:0]
2025-11-28T14:51:41.510236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789042132330643:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.510432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789044648702157:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:41.510304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:41.510349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:41.510497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789044648702073:2049] [node 2] ICP30 unregister session Session# [5:7577789044648702157:2049] VirtualId# [5:7577789044648702151:0]
2025-11-28T14:51:41.510528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:41.510566Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789044648702073:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31669) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.510576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31669) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.520581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789044648702073:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:41.522738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:41.522999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789048485280138:2050] [node 3] ICS01 socket: 203 reason# EndOfStream
2025-11-28T14:51:41.523039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789044190312752:2050] [node 3] ICP30 unregister session Session# [4:7577789048485280138:2050] VirtualId# [4:7577789048485280134:0]
2025-11-28T14:51:41.523095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789048485280138:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.523196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:41.523238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:41.524561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:41.524810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789046427297947:2050] [node 3] ICS07 socket disconnect 187 reason# EndOfStream
2025-11-28T14:51:41.524846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789046427297947:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.524937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:41.525000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789046427297947:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:41.525741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789042132330559:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29465) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:41.525798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789046427297947:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:41.525874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789042132330559:2050] [node 3] ICP30 unregister session Session# [6:7577789046427297947:2050] VirtualId# [6:7577789046427297941:0]
2025-11-28T14:51:41.525923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:41.525969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789042132330559:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29465) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.525991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29465) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.528570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:41.528889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789048943669465:2050] [node 3] ICS07 socket disconnect 226 reason# EndOfStream
2025-11-28T14:51:41.528938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789048943669465:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.529040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:41.529088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789048943669465:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:41.531826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789044648702074:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29465) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:41.531916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789048943669465:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:41.532058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789044648702074:2050] [node 3] ICP30 unregister session Session# [5:7577789048943669465:2050] VirtualId# [5:7577789048943669463:0]
2025-11-28T14:51:41.532113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:41.532211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789044648702074:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29465) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.532226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29465) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:41.534481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:41.534620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789048943669469:2108] [node 4] ICS01 socket: 217 reason# EndOfStream
2025-11-28T14:51:41.534652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789044648702135:2108] [node 4] ICP30 unregister session Session# [5:7577789048943669469:2108] VirtualId# [5:7577789048943669467:0]
2025-11-28T14:51:41.534671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789048943669469:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.534729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:41.534759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:41.535438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:41.535682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789046427297957:2109] [node 4] ICS01 socket: 219 reason# EndOfStream
2025-11-28T14:51:41.535731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789042132330623:2109] [node 4] ICP30 unregister session Session# [6:7577789046427297957:2109] VirtualId# [6:7577789046427297955:0]
2025-11-28T14:51:41.535774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789046427297957:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.535884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:41.535900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789042132330559:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:41.535951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:41.592609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:41.592968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789046427297943:2107] [node 5] ICS07 socket disconnect 180 reason# EndOfStream
2025-11-28T14:51:41.593025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789046427297943:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:41.593199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:41.642514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789049603681714:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:41.642551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789049603681715:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:41.669651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789046234734960:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:41.671773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789046234734961:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:41.694243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789047307494137:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:41.694244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789047307494138:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:41.718094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789047721368428:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:41.718135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789047721368429:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:41.718423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789047721368427:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:41.739717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789047889674332:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:41.739753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789047889674333:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:41.739779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789047889674334:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:41.762287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789047470271531:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:41.762336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789047470271533:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:41.762938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789047470271532:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:41.861490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789046234735023:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:41.861352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789047470271572:2086] [node 4] ICP01 ready to work
2025-11-28T14:51:41.893050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789046234735024:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:41.892983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789047889674395:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:41.892892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789049603681777:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:41.892147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789047307494200:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:41.894138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789049603681778:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:41.893667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789047307494201:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:41.901229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789047721368490:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:41.900717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789047470271582:2092] [node 5] ICP01 ready to work
2025-11-28T14:51:41.900919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789046234735025:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:41.894265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789047307494202:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:41.902171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789047889674396:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:41.901683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789047721368491:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:41.902445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789049603681779:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:41.917548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789047721368429:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(63661980ns) {Configure:104(55831744ns) {ProcessConfigured:135(98014ns) {StartInitialHandshake:157(95325ns) {DropHandshakes:470(2672ns) {DropIncomingHandshake:442(923ns) DropOutgoingHandshake:457(418ns)} GenerateSessionVirtualId:988(1043ns) PrepareNewSessionHandshake:477(1649ns) {SwitchToState:179(806ns)}}}}}
2025-11-28T14:51:41.927073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789049603681714:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(97211481ns) {Configure:104(97203194ns) {ProcessConfigured:135(4879286ns) {StartInitialHandshake:157(4875285ns) {DropHandshakes:470(3369ns) {DropIncomingHandshake:442(907ns) DropOutgoingHandshake:457(1022ns)} GenerateSessionVirtualId:988(1215ns) PrepareNewSessionHandshake:477(3776ns) {SwitchToState:179(1854ns)}}}}}
2025-11-28T14:51:41.928860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789046234734961:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48566062ns) {Configure:104(48559723ns) {ProcessConfigured:135(856309ns) {StartInitialHandshake:157(854051ns) {DropHandshakes:470(2151ns) {DropIncomingHandshake:442(664ns) DropOutgoingHandshake:457(342ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:477(1936ns) {SwitchToState:179(860ns)}}}}}
2025-11-28T14:51:41.937070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789049603681715:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28612656ns) {Configure:104(28605978ns) {ProcessConfigured:135(1730444ns) {StartInitialHandshake:157(1728320ns) {DropHandshakes:470(1525ns) {DropIncomingHandshake:442(382ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(642ns) PrepareNewSessionHandshake:477(1859ns) {SwitchToState:179(730ns)}}}}}
2025-11-28T14:51:41.939343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789047470271532:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(35322127ns) {Configure:104(35318793ns) {ProcessConfigured:135(736696ns) {StartInitialHandshake:157(734607ns) {DropHandshakes:470(996ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(102ns)} GenerateSessionVirtualId:988(292ns) PrepareNewSessionHandshake:477(1704ns) {SwitchToState:179(722ns)}}}}}
2025-11-28T14:51:41.938747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789047721368428:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(50947818ns) {Configure:104(50927202ns) {ProcessConfigured:135(1131694ns) {StartInitialHandshake:157(1128052ns) {DropHandshakes:470(2769ns) {DropIncomingHandshake:442(646ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(1426ns) PrepareNewSessionHandshake:477(3364ns) {SwitchToState:179(1496ns)}}}}}
2025-11-28T14:51:41.940185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789047889674332:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43840502ns) {Configure:104(43834995ns) {ProcessConfigured:135(6532144ns) {StartInitialHandshake:157(6529515ns) {DropHandshakes:470(1223ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(569ns) PrepareNewSessionHandshake:477(3310ns) {SwitchToState:179(1404ns)}}}}}
2025-11-28T14:51:41.940299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789049603681777:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36667616ns) {Configure:104(36663128ns) {ProcessConfigured:135(1781854ns) {StartInitialHandshake:157(1234639ns) {DropHandshakes:470(2865ns) {DropIncomingHandshake:442(473ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(974ns) PrepareNewSessionHandshake:477(1199ns) {SwitchToState:179(637ns)}}}}}
2025-11-28T14:51:41.939816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789046234734960:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(93157566ns) {Configure:104(93148114ns) {ProcessConfigured:135(10066501ns) {StartInitialHandshake:157(10063769ns) {DropHandshakes:470(1225ns) {DropIncomingHandshake:442(338ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(473ns) PrepareNewSessionHandshake:477(1817ns) {SwitchToState:179(642ns)}}}}}
2025-11-28T14:51:41.936216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789047307494137:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(88846769ns) {Configure:104(88839922ns) {ProcessConfigured:135(1265175ns) {StartInitialHandshake:157(1261544ns) {DropHandshakes:470(3072ns) {DropIncomingHandshake:442(1071ns) DropOutgoingHandshake:457(467ns)} GenerateSessionVirtualId:988(827ns) PrepareNewSessionHandshake:477(12822ns) {SwitchToState:179(1346ns)}}}}}
2025-11-28T14:51:41.939466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789047470271533:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(83930962ns) {Configure:104(83925861ns) {ProcessConfigured:135(2466448ns) {StartInitialHandshake:157(2464408ns) {DropHandshakes:470(2176ns) {DropIncomingHandshake:442(629ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(676ns) PrepareNewSessionHandshake:477(1851ns) {SwitchToState:179(668ns)}}}}}
2025-11-28T14:51:41.944627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789047889674334:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(37153645ns) {Configure:104(37143635ns) {ProcessConfigured:135(1093411ns) {StartInitialHandshake:157(1091534ns) {DropHandshakes:470(911ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(107ns)} GenerateSessionVirtualId:988(277ns) PrepareNewSessionHandshake:477(1183ns) {SwitchToState:179(499ns)}}}}}
2025-11-28T14:51:41.952566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789047721368490:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36369948ns) {Configure:104(36363000ns) {ProcessConfigured:135(1237688ns) {StartInitialHandshake:157(1235658ns) {DropHandshakes:470(1226ns) {DropIncomingHandshake:442(393ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(480ns) PrepareNewSessionHandshake:477(1505ns) {SwitchToState:179(571ns)}}}}}
2025-11-28T14:51:41.940582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789047470271572:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48264486ns) {Configure:104(48255462ns) {ProcessConfigured:135(2803825ns) {StartInitialHandshake:157(2719019ns) {DropHandshakes:470(1018ns) {DropIncomingHandshake:442(283ns) DropOutgoingHandshake:457(183ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(12371ns) {SwitchToState:179(1259ns)}}}}}
2025-11-28T14:51:41.939096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789047307494200:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(34446057ns) {Configure:104(33589494ns) {ProcessConfigured:135(457228ns) {StartInitialHandshake:157(454220ns) {DropHandshakes:470(1558ns) {DropIncomingHandshake:442(423ns) DropOutgoingHandshake:457(226ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:477(1842ns) {SwitchToState:179(869ns)}}}}}
2025-11-28T14:51:41.953096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789047889674333:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46613511ns) {Configure:104(46607555ns) {ProcessConfigured:135(2574695ns) {StartInitialHandshake:157(2572787ns) {DropHandshakes:470(1545ns) {DropIncomingHandshake:442(446ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(635ns) PrepareNewSessionHandshake:477(1843ns) {SwitchToState:179(529ns)}}}}}
2025-11-28T14:51:41.953419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789049603681778:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34664909ns) {Configure:104(34660093ns) {ProcessConfigured:135(2059040ns) {StartInitialHandshake:157(2058061ns) {DropHandshakes:470(733ns) {DropIncomingHandshake:442(225ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(1341ns) {SwitchToState:179(99ns)}}}}}
2025-11-28T14:51:41.954256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789046234735023:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(37856478ns) {Configure:104(37849027ns) {ProcessConfigured:135(2607564ns) {StartInitialHandshake:157(2606048ns) {DropHandshakes:470(1485ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(339ns) PrepareNewSessionHandshake:477(1555ns) {SwitchToState:179(160ns)}}}}}
2025-11-28T14:51:41.953154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789047721368427:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(37066717ns) {Configure:104(37062700ns) {ProcessConfigured:135(2596191ns) {StartInitialHandshake:157(2593844ns) {DropHandshakes:470(1511ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(693ns) PrepareNewSessionHandshake:477(2131ns) {SwitchToState:179(917ns)}}}}}
2025-11-28T14:51:41.944613Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789047470271531:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44522973ns) {Configure:104(44517571ns) {ProcessConfigured:135(726436ns) {StartInitialHandshake:157(724053ns) {DropHandshakes:470(1182ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(433ns) PrepareNewSessionHandshake:477(1215ns) {SwitchToState:179(592ns)}}}}}
2025-11-28T14:51:41.939493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789047307494138:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(90214730ns) {Configure:104(90209619ns) {ProcessConfigured:135(2646135ns) {StartInitialHandshake:157(2643743ns) {DropHandshakes:470(1351ns) {DropIncomingHandshake:442(325ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(920ns) PrepareNewSessionHandshake:477(3048ns) {SwitchToState:179(1376ns)}}}}}
2025-11-28T14:51:41.954784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789046234735024:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(37821511ns) {Configure:104(37818461ns) {ProcessConfigured:135(2697575ns) {StartInitialHandshake:157(2696654ns) {DropHandshakes:470(324473ns) {DropIncomingHandshake:442(654ns) DropOutgoingHandshake:457(122ns)} GenerateSessionVirtualId:988(464ns) PrepareNewSessionHandshake:477(1257ns) {SwitchToState:179(160ns)}}}}}
2025-11-28T14:51:41.952732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789047307494201:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36624217ns) {Configure:104(36619825ns) {ProcessConfigured:135(1365668ns) {StartInitialHandshake:157(1364215ns) {DropHandshakes:470(719ns) {DropIncomingHandshake:442(208ns) DropOutgoingHandshake:457(108ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(1463ns) {SwitchToState:179(484ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.969888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789047307494212:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5721
2025-11-28T14:51:41.959411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789047721368491:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30418217ns) {Configure:104(30409726ns) {ProcessConfigured:135(1303952ns) {StartInitialHandshake:157(1301822ns) {DropHandshakes:470(402305ns) {DropIncomingHandshake:442(884ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(1671ns) PrepareNewSessionHandshake:477(2269ns) {SwitchToState:179(391ns)}}}}}
2025-11-28T14:51:41.966020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789047889674395:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(51161135ns) {Configure:104(51157566ns) {ProcessConfigured:135(3413451ns) {StartInitialHandshake:157(3410764ns) {DropHandshakes:470(4127ns) {DropIncomingHandshake:442(957ns) DropOutgoingHandshake:457(1057ns)} GenerateSessionVirtualId:988(1209ns) PrepareNewSessionHandshake:477(2679ns) {SwitchToState:179(1381ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.968559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789049603681779:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29086576ns) {Configure:104(29082845ns) {ProcessConfigured:135(1361381ns) {StartInitialHandshake:157(1359803ns) {DropHandshakes:470(1182ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(602ns) PrepareNewSessionHandshake:477(1968ns) {SwitchToState:179(508ns)}}}}}
2025-11-28T14:51:41.989875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789047889674396:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32738502ns) {Configure:104(32732564ns) {ProcessConfigured:135(77508ns) {StartInitialHandshake:157(74787ns) {DropHandshakes:470(3004ns) {DropIncomingHandshake:442(584ns) DropOutgoingHandshake:457(572ns)} GenerateSessionVirtualId:988(681ns) PrepareNewSessionHandshake:477(2500ns) {SwitchToState:179(1085ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.985653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789047721368501:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14683
2025-11-28T14:51:41.998836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789047470271582:2092] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(53963334ns) {Configure:104(53956543ns) {ProcessConfigured:135(1992248ns) {StartInitialHandshake:157(1989460ns) {DropHandshakes:470(2461ns) {DropIncomingHandshake:442(507ns) DropOutgoingHandshake:457(638ns)} GenerateSessionVirtualId:988(301ns) PrepareNewSessionHandshake:477(2481ns) {SwitchToState:179(989ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.997095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789046234735025:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(50022178ns) {Configure:104(50014298ns) {ProcessConfigured:135(9663530ns) {StartInitialHandshake:157(9660349ns) {DropHandshakes:470(3045ns) {DropIncomingHandshake:442(805ns) DropOutgoingHandshake:457(493ns)} GenerateSessionVirtualId:988(997ns) PrepareNewSessionHandshake:477(11671ns) {SwitchToState:179(8294ns)}}}}}
2025-11-28T14:51:41.999187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789047307494211:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5721
2025-11-28T14:51:41.995193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789047721368501:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.000784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789047307494202:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(54888591ns) {Configure:104(54882004ns) {ProcessConfigured:135(2043051ns) {StartInitialHandshake:157(2039332ns) {DropHandshakes:470(2665ns) {DropIncomingHandshake:442(657ns) DropOutgoingHandshake:457(576ns)} GenerateSessionVirtualId:988(774ns) PrepareNewSessionHandshake:477(2211ns) {SwitchToState:179(994ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:41.999362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789047721368427:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.007704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789047721368502:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14683
2025-11-28T14:51:42.009916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789047721368502:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.008589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789049603681786:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.008653Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789049603681786:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.011043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789047889674410:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10093
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.010986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789049603681795:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26664
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.011522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789047307494211:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.012211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789046234735035:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17506
2025-11-28T14:51:42.011002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789049603681794:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26664
2025-11-28T14:51:42.013733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789047307494216:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5721
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.012495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789046234735034:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17506
2025-11-28T14:51:42.011176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789049603681793:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26664
2025-11-28T14:51:42.016581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789046234735036:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17506
2025-11-28T14:51:42.016782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789046234735039:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17506
2025-11-28T14:51:42.013852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789047307494219:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5721
2025-11-28T14:51:42.017970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052016335805:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14683
2025-11-28T14:51:42.014006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789047307494212:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.017051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789046234735035:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.011814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789049603681786:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.018019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789047889674410:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.014146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789047307494137:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.017701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789046234735040:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17506
2025-11-28T14:51:42.011855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789049603681794:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.020477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789047307494216:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.020511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789047889674334:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.018598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789046234735040:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.017702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789049603681777:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.020565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789047307494219:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.028478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789052184641707:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10093
2025-11-28T14:51:42.021097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789046234735039:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.017748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:42.028612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789051602461516:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5721
2025-11-28T14:51:42.029249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789047307494208:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.017863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789049603681777:2107] [node 4] ICP22 created new session: [1:7577789053898649095:2107]
2025-11-28T14:51:42.029294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789047307494208:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.018634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789053898649095:2107] [node 4] ICS09 handshake done sender: [1:7577789049603681786:2112] self: [1:7577789049603681784:0] peer: [4:7577789047721368507:0] socket: 176 qp: -1
2025-11-28T14:51:42.020447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789049603681783:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.031379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789047721368501:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:42.020478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789049603681783:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.031680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789047721368427:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.031745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:42.020516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789053898649095:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:42.031807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789047721368427:2048] [node 1] ICP22 created new session: [4:7577789052016335806:2048]
2025-11-28T14:51:42.020563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:42.028384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789049603681795:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.031877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789052016335806:2048] [node 1] ICS09 handshake done sender: [4:7577789047721368501:2114] self: [4:7577789047721368507:0] peer: [1:7577789049603681784:0] socket: 179 qp: -1
2025-11-28T14:51:42.032549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789047307494208:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:42.031323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789053898649093:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26664
2025-11-28T14:51:42.031925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789052016335806:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.031790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789049603681793:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.031956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:42.032808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789049603681777:2107] [node 4] ICP18 handshake (actor [1:7577789049603681795:2117]) is too old
2025-11-28T14:51:42.033078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789052184641709:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10093
2025-11-28T14:51:42.033229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789047307494200:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.033551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789049603681783:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.033643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:42.034272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789047307494200:2108] [node 5] ICP22 created new session: [3:7577789051602461524:2108]
2025-11-28T14:51:42.034997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789049603681715:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.034818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789047889674410:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:42.035868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238905:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.034582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789051602461524:2108] [node 5] ICS09 handshake done sender: [3:7577789047307494208:2112] self: [3:7577789047307494207:0] peer: [5:7577789052184641708:0] socket: 195 qp: -1
2025-11-28T14:51:42.036590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:42.034626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789051602461524:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:42.036179Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789047889674334:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789047889674410:2115] held: no
2025-11-28T14:51:42.036872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789049603681715:2049] [node 3] ICP22 created new session: [1:7577789053898649100:2049]
2025-11-28T14:51:42.035921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.036660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789047889674334:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.037151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789053898649100:2049] [node 3] ICS09 handshake done sender: [1:7577789049603681783:2111] self: [1:7577789049603681782:0] peer: [3:7577789051602461517:0] socket: 175 qp: -1
2025-11-28T14:51:42.036691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.037409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789053898649100:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:42.037342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789047889674334:2050] [node 3] ICP22 created new session: [5:7577789052184641711:2050]
2025-11-28T14:51:42.037445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:42.037529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052016335809:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14683
2025-11-28T14:51:42.036846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789051602461523:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5721
2025-11-28T14:51:42.037122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789047307494137:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789047307494211:2114] held: no
2025-11-28T14:51:42.037695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789052184641711:2050] [node 3] ICS09 handshake done sender: [5:7577789047889674410:2115] self: [5:7577789052184641708:0] peer: [3:7577789047307494207:0] socket: 201 qp: -1
2025-11-28T14:51:42.037939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789047307494211:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:42.037774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789052184641710:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10093
2025-11-28T14:51:42.038043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789052184641711:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.038076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:42.039094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789047307494137:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.039145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.039265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789047307494137:2048] [node 1] ICP22 created new session: [3:7577789051602461526:2048]
2025-11-28T14:51:42.039142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789052184641713:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10093
2025-11-28T14:51:42.039311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789051602461526:2048] [node 1] ICS09 handshake done sender: [3:7577789047307494211:2114] self: [3:7577789051602461517:0] peer: [1:7577789049603681782:0] socket: 181 qp: -1
2025-11-28T14:51:42.039341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789051602461526:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:42.039385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:42.039340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238906:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.039739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789051765238906:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.039909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238907:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.039924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789047470271582:2092] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.040171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789047889674404:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.040287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789047889674404:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.041997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789047889674404:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:42.042394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789047889674395:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.042453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.042630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789047889674395:2109] [node 6] ICP22 created new session: [5:7577789052184641716:2109]
2025-11-28T14:51:42.042639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789052184641713:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.042743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789052184641716:2109] [node 6] ICS09 handshake done sender: [5:7577789047889674404:2113] self: [5:7577789047889674403:0] peer: [6:7577789051765238908:0] socket: 190 qp: -1
2025-11-28T14:51:42.043025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789051765238907:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.042756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789047889674332:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.043053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238909:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.043296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789049603681792:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.043223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789047470271533:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.042783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789052184641716:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:42.043332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789049603681792:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.042860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:42.043571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789047307494218:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.043611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789047307494218:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.044290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052016335810:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14683
2025-11-28T14:51:42.044853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789051765238906:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:42.045556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789047307494218:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:42.045767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789047470271582:2092] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.045750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789049603681792:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:42.046283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:42.046494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789052016335810:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.047219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789047470271582:2092] [node 5] ICP22 created new session: [6:7577789051765238911:2092]
2025-11-28T14:51:42.046523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789047307494202:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.047393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789051765238911:2092] [node 5] ICS09 handshake done sender: [6:7577789051765238906:2116] self: [6:7577789051765238908:0] peer: [5:7577789047889674403:0] socket: 225 qp: -1
2025-11-28T14:51:42.047403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789049603681779:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.047423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789051765238911:2092] [node 5] ICS10 traffic start
2025-11-28T14:51:42.046603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:42.047479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:42.047794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:42.046690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789047307494202:2109] [node 6] ICP22 created new session: [3:7577789051602461530:2109]
2025-11-28T14:51:42.047769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789049603681779:2108] [node 5] ICP22 created new session: [1:7577789053898649104:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.046738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789051602461530:2109] [node 6] ICS09 handshake done sender: [3:7577789047307494218:2117] self: [3:7577789047307494217:0] peer: [6:7577789051765238910:0] socket: 189 qp: -1
2025-11-28T14:51:42.047836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789047470271533:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789051765238907:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.047980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789053898649104:2108] [node 5] ICS09 handshake done sender: [1:7577789049603681792:2114] self: [1:7577789049603681791:0] peer: [5:7577789052184641718:0] socket: 202 qp: -1
2025-11-28T14:51:42.046776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789051602461530:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:42.048009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789053898649104:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:42.048766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789047889674332:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789052184641713:2119] held: no
2025-11-28T14:51:42.048815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789052184641720:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10093
2025-11-28T14:51:42.048995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238912:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.046892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:42.048049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:42.050001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789052184641713:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.050603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789047889674332:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.050707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789051765238907:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:42.050744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:42.051127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789053898649106:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26664
2025-11-28T14:51:42.051159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789047470271533:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.051213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:42.050965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789047889674332:2048] [node 1] ICP22 created new session: [5:7577789052184641721:2048]
2025-11-28T14:51:42.051265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789047470271533:2050] [node 3] ICP22 created new session: [6:7577789051765238915:2050]
2025-11-28T14:51:42.051309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789051765238915:2050] [node 3] ICS09 handshake done sender: [6:7577789051765238907:2117] self: [6:7577789051765238910:0] peer: [3:7577789047307494217:0] socket: 227 qp: -1
2025-11-28T14:51:42.051143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789052184641721:2048] [node 1] ICS09 handshake done sender: [5:7577789052184641713:2119] self: [5:7577789052184641718:0] peer: [1:7577789049603681791:0] socket: 226 qp: -1
2025-11-28T14:51:42.051340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789051765238915:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:42.051253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789052184641721:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:42.051380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:42.051303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:42.051918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789053898649106:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.054407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789049603681781:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.073059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789046234735034:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.073231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789046234734960:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.073564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789049603681781:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.074113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789049603681781:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.074255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789050529702344:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17506
2025-11-28T14:51:42.074356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789049603681714:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.074504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:42.074591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789049603681714:2048] [node 2] ICP22 created new session: [1:7577789053898649109:2048]
2025-11-28T14:51:42.074662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789053898649109:2048] [node 2] ICS09 handshake done sender: [1:7577789049603681781:2110] self: [1:7577789049603681780:0] peer: [2:7577789050529702343:0] socket: 174 qp: -1
2025-11-28T14:51:42.074697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789053898649109:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:42.074756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:42.074879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789046234735034:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:42.075111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789046234734960:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.075155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:42.075182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789046234734960:2048] [node 1] ICP22 created new session: [2:7577789050529702345:2048]
2025-11-28T14:51:42.075211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789050529702345:2048] [node 1] ICS09 handshake done sender: [2:7577789046234735034:2114] self: [2:7577789050529702343:0] peer: [1:7577789049603681780:0] socket: 185 qp: -1
2025-11-28T14:51:42.075234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789050529702345:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:42.075278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:42.075832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789047721368495:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.083637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789046234735036:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.087736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789046234735027:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.109700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789051602461516:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.109912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789047307494138:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.110117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789046234735027:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.110636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789046234735027:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.110822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789046234734961:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.110804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789051602461533:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5721
2025-11-28T14:51:42.110887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:42.110955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789046234734961:2049] [node 3] ICP22 created new session: [2:7577789050529702349:2049]
2025-11-28T14:51:42.111007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789050529702349:2049] [node 3] ICS09 handshake done sender: [2:7577789046234735027:2110] self: [2:7577789046234735026:0] peer: [3:7577789051602461532:0] socket: 209 qp: -1
2025-11-28T14:51:42.111053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789050529702349:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:42.111119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:42.111118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789051602461516:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:42.111381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789047307494138:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.111862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:42.111922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789047307494138:2049] [node 2] ICP22 created new session: [3:7577789051602461534:2049]
2025-11-28T14:51:42.112055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789051602461534:2049] [node 2] ICS09 handshake done sender: [3:7577789051602461516:2119] self: [3:7577789051602461532:0] peer: [2:7577789046234735026:0] socket: 211 qp: -1
2025-11-28T14:51:42.112095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789051602461534:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:42.113524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:42.113537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789046234735032:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.114424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789047307494138:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.114559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052016335812:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14683
2025-11-28T14:51:42.114415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789052184641707:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.120060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789047889674333:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.121477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789046234735032:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.122169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789046234735032:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:42.122284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789052184641724:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10093
2025-11-28T14:51:42.122522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789052184641707:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:42.122694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789046234735023:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.122766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:42.122991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789046234735023:2108] [node 5] ICP22 created new session: [2:7577789050529702353:2108]
2025-11-28T14:51:42.123022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789050529702353:2108] [node 5] ICS09 handshake done sender: [2:7577789046234735032:2112] self: [2:7577789046234735030:0] peer: [5:7577789052184641723:0] socket: 215 qp: -1
2025-11-28T14:51:42.123046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789050529702353:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:42.123024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789047889674333:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.123160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:42.123146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:42.123207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789047889674333:2049] [node 2] ICP22 created new session: [5:7577789052184641725:2049]
2025-11-28T14:51:42.123238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789052184641725:2049] [node 2] ICS09 handshake done sender: [5:7577789052184641707:2116] self: [5:7577789052184641723:0] peer: [2:7577789046234735030:0] socket: 216 qp: -1
2025-11-28T14:51:42.123296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789052184641725:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:42.123332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.124794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238917:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.137281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789046234735029:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.145758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789053898649093:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.145927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789049603681714:2048] [node 2] ICP18 handshake (actor [1:7577789053898649093:2118]) is too old
2025-11-28T14:51:42.155806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789049603681790:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.164955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789051765238905:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.165156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789047470271531:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.165398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789049603681790:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.165898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789049603681790:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:42.166106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789049603681778:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.166163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:42.166203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789049603681778:2109] [node 6] ICP22 created new session: [1:7577789053898649112:2109]
2025-11-28T14:51:42.166286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238919:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.166266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789053898649112:2109] [node 6] ICS09 handshake done sender: [1:7577789049603681790:2113] self: [1:7577789049603681785:0] peer: [6:7577789051765238918:0] socket: 220 qp: -1
2025-11-28T14:51:42.166290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789053898649112:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:42.166345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:42.166756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789047470271531:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789051765238905:2115] held: no
2025-11-28T14:51:42.166758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789051765238905:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:42.166945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789047470271531:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.167467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:42.167527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789047470271531:2048] [node 1] ICP22 created new session: [6:7577789051765238920:2048]
2025-11-28T14:51:42.167650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789051765238920:2048] [node 1] ICS09 handshake done sender: [6:7577789051765238905:2115] self: [6:7577789051765238918:0] peer: [1:7577789049603681785:0] socket: 221 qp: -1
2025-11-28T14:51:42.167687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789051765238920:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:42.167747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:42.182352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789047307494210:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.204850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789052016335809:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.205044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789047721368429:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.205301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789047307494210:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.205688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789047307494210:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:42.205918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789047307494201:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.205996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:42.206055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789047307494201:2107] [node 4] ICP22 created new session: [3:7577789051602461537:2107]
2025-11-28T14:51:42.206083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789051602461537:2107] [node 4] ICS09 handshake done sender: [3:7577789047307494210:2113] self: [3:7577789047307494209:0] peer: [4:7577789052016335813:0] socket: 206 qp: -1
2025-11-28T14:51:42.206105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789051602461537:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:42.206084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052016335814:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14683
2025-11-28T14:51:42.206131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:42.206415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789052016335809:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:42.207203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789047721368429:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789052016335809:2118] held: no
2025-11-28T14:51:42.207318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789047721368429:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.207436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:42.207498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789047721368429:2050] [node 3] ICP22 created new session: [4:7577789052016335815:2050]
2025-11-28T14:51:42.207569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789052016335815:2050] [node 3] ICS09 handshake done sender: [4:7577789052016335809:2118] self: [4:7577789052016335813:0] peer: [3:7577789047307494209:0] socket: 224 qp: -1
2025-11-28T14:51:42.207612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789052016335815:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:42.207650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.207992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238922:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.209166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789051765238922:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.209335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789047470271572:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.209571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789047721368503:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.209602Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789047721368503:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.211219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789047721368503:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.211376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238924:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.211401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789047721368491:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.211516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:42.211551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789047721368491:2109] [node 6] ICP22 created new session: [4:7577789052016335819:2109]
2025-11-28T14:51:42.211684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789051765238922:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:42.211650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789052016335819:2109] [node 6] ICS09 handshake done sender: [4:7577789047721368503:2116] self: [4:7577789047721368500:0] peer: [6:7577789051765238923:0] socket: 192 qp: -1
2025-11-28T14:51:42.211690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789052016335819:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:42.211725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:42.213271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789047470271572:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.213319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:42.213442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789047721368499:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.213455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789047470271572:2086] [node 4] ICP22 created new session: [6:7577789051765238925:2086]
2025-11-28T14:51:42.214241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789051765238925:2086] [node 4] ICS09 handshake done sender: [6:7577789051765238922:2122] self: [6:7577789051765238923:0] peer: [4:7577789047721368500:0] socket: 199 qp: -1
2025-11-28T14:51:42.221128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789052184641710:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.221019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789051765238925:2086] [node 4] ICS10 traffic start
2025-11-28T14:51:42.221064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:42.221441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789047470271572:2086] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:42.221451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789047889674396:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.221691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789047721368499:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.222152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789047721368499:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.222365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789047721368490:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.222338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789052184641728:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10093
2025-11-28T14:51:42.222474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:42.222548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789047721368490:2108] [node 5] ICP22 created new session: [4:7577789052016335822:2108]
2025-11-28T14:51:42.222594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789052016335822:2108] [node 5] ICS09 handshake done sender: [4:7577789047721368499:2113] self: [4:7577789047721368497:0] peer: [5:7577789052184641727:0] socket: 223 qp: -1
2025-11-28T14:51:42.222686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789052184641710:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:42.222628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789052016335822:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:42.222697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:42.222835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789047889674396:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.222872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:42.222900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789047889674396:2108] [node 4] ICP22 created new session: [5:7577789052184641729:2108]
2025-11-28T14:51:42.223043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789052184641729:2108] [node 4] ICS09 handshake done sender: [5:7577789052184641710:2118] self: [5:7577789052184641727:0] peer: [4:7577789047721368497:0] socket: 204 qp: -1
2025-11-28T14:51:42.223244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789052184641729:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:42.224059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:42.224433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789047889674396:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:42.227635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789046234735033:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.235328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789052016335812:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.235530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789047721368428:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.235746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789046234735033:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.236208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789046234735033:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.236443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789046234735024:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.236472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052016335825:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14683
2025-11-28T14:51:42.236531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:42.236580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789046234735024:2107] [node 4] ICP22 created new session: [2:7577789050529702357:2107]
2025-11-28T14:51:42.236653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789050529702357:2107] [node 4] ICS09 handshake done sender: [2:7577789046234735033:2113] self: [2:7577789046234735031:0] peer: [4:7577789052016335824:0] socket: 194 qp: -1
2025-11-28T14:51:42.236697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789050529702357:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:42.236733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:42.236738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789052016335812:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:42.236956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789047721368428:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.237246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:42.237409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789047721368428:2049] [node 2] ICP22 created new session: [4:7577789052016335826:2049]
2025-11-28T14:51:42.237493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789052016335826:2049] [node 2] ICS09 handshake done sender: [4:7577789052016335812:2120] self: [4:7577789052016335824:0] peer: [2:7577789046234735031:0] socket: 238 qp: -1
2025-11-28T14:51:42.237541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789052016335826:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:42.237624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:42.238349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789046234735038:2117] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.238449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789047721368428:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:42.239035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789051765238917:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.239181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789047470271532:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.239338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789046234735038:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.239618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789046234735038:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.239747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789046234735025:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.239846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:42.239882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789046234735025:2109] [node 6] ICP22 created new session: [2:7577789050529702360:2109]
2025-11-28T14:51:42.239908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789050529702360:2109] [node 6] ICS09 handshake done sender: [2:7577789046234735038:2117] self: [2:7577789046234735037:0] peer: [6:7577789051765238927:0] socket: 198 qp: -1
2025-11-28T14:51:42.239873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789051765238928:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13158
2025-11-28T14:51:42.239932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789050529702360:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:42.239976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:42.240055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789051765238917:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:42.240312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789047470271532:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789051765238917:2120] held: no
2025-11-28T14:51:42.240505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789047470271532:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.240527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:42.240553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789047470271532:2049] [node 2] ICP22 created new session: [6:7577789051765238929:2049]
2025-11-28T14:51:42.240579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789051765238929:2049] [node 2] ICS09 handshake done sender: [6:7577789051765238917:2120] self: [6:7577789051765238927:0] peer: [2:7577789046234735037:0] socket: 200 qp: -1
2025-11-28T14:51:42.240598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789051765238929:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:42.240628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:42.273614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:42.273798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789050529702345:2048] [node 1] ICS07 socket disconnect 185 reason# EndOfStream
2025-11-28T14:51:42.273824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789050529702345:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.273926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:42.273989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789050529702345:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:42.274506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:42.274704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789051602461526:2048] [node 1] ICS01 socket: 181 reason# EndOfStream
2025-11-28T14:51:42.274754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789047307494137:2048] [node 1] ICP30 unregister session Session# [3:7577789051602461526:2048] VirtualId# [3:7577789051602461517:0]
2025-11-28T14:51:42.274790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789051602461526:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.274882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:42.274934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:42.275805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789046234734960:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:42.275829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789050529702345:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:42.276754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789046234734960:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:42.276794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789050529702345:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:42.277295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789046234734960:2048] [node 1] ICP30 unregister session Session# [2:7577789050529702345:2048] VirtualId# [2:7577789050529702343:0]
2025-11-28T14:51:42.277332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:42.277355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789046234734960:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.277365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.277709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:42.277879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789052016335806:2048] [node 1] ICS07 socket disconnect 179 reason# EndOfStream
2025-11-28T14:51:42.277927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789052016335806:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.277996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:42.278042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789052016335806:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:42.278389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:42.278566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789052184641721:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-11-28T14:51:42.278604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789047889674332:2048] [node 1] ICP30 unregister session Session# [5:7577789052184641721:2048] VirtualId# [5:7577789052184641718:0]
2025-11-28T14:51:42.278633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789052184641721:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.278716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:42.278770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:42.278782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789047721368427:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:42.278944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789052016335806:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:42.278991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789047721368427:2048] [node 1] ICP30 unregister session Session# [4:7577789052016335806:2048] VirtualId# [4:7577789047721368507:0]
2025-11-28T14:51:42.279038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:42.279067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789047721368427:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.279080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.279289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:42.279543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789051765238920:2048] [node 1] ICS01 socket: 221 reason# EndOfStream
2025-11-28T14:51:42.279589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789047470271531:2048] [node 1] ICP30 unregister session Session# [6:7577789051765238920:2048] VirtualId# [6:7577789051765238918:0]
2025-11-28T14:51:42.279620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789051765238920:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.279708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:42.279765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:42.285984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:42.286206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789051602461534:2049] [node 2] ICS07 socket disconnect 211 reason# EndOfStream
2025-11-28T14:51:42.286243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789051602461534:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.286351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:42.286399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789051602461534:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:42.287092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789047307494138:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:42.287191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789051602461534:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:42.287239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789047307494138:2049] [node 2] ICP30 unregister session Session# [3:7577789051602461534:2049] VirtualId# [3:7577789051602461532:0]
2025-11-28T14:51:42.287285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:42.287337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789047307494138:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.287354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.288226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:42.288452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789052016335826:2049] [node 2] ICS07 socket disconnect 238 reason# EndOfStream
2025-11-28T14:51:42.288482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789052016335826:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.288525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789047721368427:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:42.288553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:42.288594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789052016335826:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:42.289279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:42.289415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789052184641725:2049] [node 2] ICS07 socket disconnect 216 reason# EndOfStream
2025-11-28T14:51:42.289311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789047721368428:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:42.289436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789052184641725:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.289501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:42.289540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789052184641725:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:42.289361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789052016335826:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:42.289448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789047721368428:2049] [node 2] ICP30 unregister session Session# [4:7577789052016335826:2049] VirtualId# [4:7577789052016335824:0]
2025-11-28T14:51:42.289494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:42.289529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789047721368428:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.289542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.290117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789047889674333:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:42.290205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789052184641725:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:42.290239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789047889674333:2049] [node 2] ICP30 unregister session Session# [5:7577789052184641725:2049] VirtualId# [5:7577789052184641723:0]
2025-11-28T14:51:42.290279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:42.290302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789047889674333:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.290330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25156) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.290524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:42.290666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789051765238929:2049] [node 2] ICS01 socket: 200 reason# EndOfStream
2025-11-28T14:51:42.290705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789047470271532:2049] [node 2] ICP30 unregister session Session# [6:7577789051765238929:2049] VirtualId# [6:7577789051765238927:0]
2025-11-28T14:51:42.290725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789051765238929:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.290774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:42.290808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:42.296751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789047307494138:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:42.299812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789047889674333:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:42.299814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789047721368428:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:42.318249Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:42.318481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789052016335815:2050] [node 3] ICS01 socket: 224 reason# EndOfStream
2025-11-28T14:51:42.318528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789047721368429:2050] [node 3] ICP30 unregister session Session# [4:7577789052016335815:2050] VirtualId# [4:7577789052016335813:0]
2025-11-28T14:51:42.318564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789052016335815:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.318668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:42.318736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:42.319290Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:42.319430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789052184641711:2050] [node 3] ICS01 socket: 201 reason# EndOfStream
2025-11-28T14:51:42.319454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789047889674334:2050] [node 3] ICP30 unregister session Session# [5:7577789052184641711:2050] VirtualId# [5:7577789052184641708:0]
2025-11-28T14:51:42.319471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789052184641711:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.319529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:42.319561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:42.320232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:42.320445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789051765238915:2050] [node 3] ICS01 socket: 227 reason# EndOfStream
2025-11-28T14:51:42.320486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789047470271533:2050] [node 3] ICP30 unregister session Session# [6:7577789051765238915:2050] VirtualId# [6:7577789051765238910:0]
2025-11-28T14:51:42.320517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789051765238915:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.320591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:42.320665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:42.345774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:42.346022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789052184641729:2108] [node 4] ICS07 socket disconnect 204 reason# EndOfStream
2025-11-28T14:51:42.346068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789052184641729:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.346155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:42.346193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789052184641729:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:42.346577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:42.346802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789051765238925:2086] [node 4] ICS07 socket disconnect 199 reason# EndOfStream
2025-11-28T14:51:42.346830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789051765238925:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.346819Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789047889674396:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23353) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:42.346867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789052184641729:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:42.347007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789047889674396:2108] [node 4] ICP30 unregister session Session# [5:7577789052184641729:2108] VirtualId# [5:7577789052184641727:0]
2025-11-28T14:51:42.346906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:42.347119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:42.346961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789051765238925:2086] [node 4] ICS15 start handshake
2025-11-28T14:51:42.347224Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789047889674396:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23353) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.347233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23353) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.347890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789047470271572:2086] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23353) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:42.347938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789051765238925:2086] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:42.348009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789047470271572:2086] [node 4] ICP30 unregister session Session# [6:7577789051765238925:2086] VirtualId# [6:7577789051765238923:0]
2025-11-28T14:51:42.348062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:42.348093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789047470271572:2086] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23353) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.348106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23353) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.358182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789047470271572:2086] [node 4] ICP33 wake up from error state
2025-11-28T14:51:42.367396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:42.367657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789051765238911:2092] [node 5] ICS07 socket disconnect 225 reason# EndOfStream
2025-11-28T14:51:42.367699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789051765238911:2092] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:42.367921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:42.367989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789051765238911:2092] [node 5] ICS15 start handshake
2025-11-28T14:51:42.368853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789047470271582:2092] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27943) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:42.368998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789051765238911:2092] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:42.369040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789047470271582:2092] [node 5] ICP30 unregister session Session# [6:7577789051765238911:2092] VirtualId# [6:7577789051765238908:0]
2025-11-28T14:51:42.369088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:42.369135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789047470271582:2092] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27943) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:42.369154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27943) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:42.434088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789052326848090:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:42.434171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789052326848089:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:42.459487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789051298256106:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:42.459499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789051298256107:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:42.484126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789050091923421:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:42.484202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789050091923422:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:42.507493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789052556805872:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:42.507551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789052556805873:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:42.507752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789052556805874:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:42.532537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789053628893385:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:42.532592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789053628893386:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:42.533092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789053628893387:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:42.559750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789050763317420:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:42.559877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789050763317421:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:42.559936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789050763317422:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:42.621043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789050763317479:2105] [node 5] ICP01 ready to work
2025-11-28T14:51:42.640101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789052326848152:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:42.700521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789052326848153:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:42.709825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789050091923484:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:42.710110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789053628893449:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:42.709224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789052326848154:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:42.709801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789050091923485:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:42.710476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789052556805935:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:42.710344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789050763317493:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:42.710620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789051298256188:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:42.710644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789053628893450:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:42.710838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789050091923486:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:42.711356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789052556805936:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:42.710951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789051298256187:2115] [node 6] ICP01 ready to work
2025-11-28T14:51:42.712353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789051298256189:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:42.729424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789051298256187:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16041148ns) {Configure:104(16036222ns) {ProcessConfigured:135(218425ns) {StartInitialHandshake:157(215065ns) {DropHandshakes:470(3401ns) {DropIncomingHandshake:442(899ns) DropOutgoingHandshake:457(610ns)} GenerateSessionVirtualId:988(1235ns) PrepareNewSessionHandshake:477(2608ns) {SwitchToState:179(1173ns)}}}}}
2025-11-28T14:51:42.731618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789050763317422:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(117481748ns) {Configure:104(109101830ns) {ProcessConfigured:135(722279ns) {StartInitialHandshake:157(720059ns) {DropHandshakes:470(1943ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(327ns)} GenerateSessionVirtualId:988(651ns) PrepareNewSessionHandshake:477(829ns) {SwitchToState:179(294ns)}}}}}
2025-11-28T14:51:42.731302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789051298256106:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(124275344ns) {Configure:104(124269778ns) {ProcessConfigured:135(960704ns) {StartInitialHandshake:157(958009ns) {DropHandshakes:470(2515ns) {DropIncomingHandshake:442(504ns) DropOutgoingHandshake:457(493ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:477(1845ns) {SwitchToState:179(1053ns)}}}}}
2025-11-28T14:51:42.731851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789050091923484:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20792912ns) {Configure:104(20503670ns) {ProcessConfigured:135(1190539ns) {StartInitialHandshake:157(1187732ns) {DropHandshakes:470(1181ns) {DropIncomingHandshake:442(308ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(533ns) PrepareNewSessionHandshake:477(1602ns) {SwitchToState:179(380ns)}}}}}
2025-11-28T14:51:42.731618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789051298256188:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18688645ns) {Configure:104(18685520ns) {ProcessConfigured:135(969582ns) {StartInitialHandshake:157(966277ns) {DropHandshakes:470(5440ns) {DropIncomingHandshake:442(591ns) DropOutgoingHandshake:457(1102ns)} GenerateSessionVirtualId:988(1117ns) PrepareNewSessionHandshake:477(2573ns) {SwitchToState:179(842ns)}}}}}
2025-11-28T14:51:42.731678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789053628893386:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30180430ns) {Configure:104(30172237ns) {ProcessConfigured:135(865934ns) {StartInitialHandshake:157(863413ns) {DropHandshakes:470(671ns) {DropIncomingHandshake:442(227ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(836ns) PrepareNewSessionHandshake:477(884ns) {SwitchToState:179(138ns)}}}}}
2025-11-28T14:51:42.731670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789052326848090:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(125689265ns) {Configure:104(125678405ns) {ProcessConfigured:135(568626ns) {StartInitialHandshake:157(566001ns) {DropHandshakes:470(1814ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(537ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:477(1792ns) {SwitchToState:179(652ns)}}}}}
2025-11-28T14:51:42.731771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789053628893387:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28994155ns) {Configure:104(28988632ns) {ProcessConfigured:135(1265311ns) {StartInitialHandshake:157(1263780ns) {DropHandshakes:470(817ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(212ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:477(1533ns) {SwitchToState:179(586ns)}}}}}
2025-11-28T14:51:42.732041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789052326848089:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(126945984ns) {Configure:104(126940579ns) {ProcessConfigured:135(539691ns) {StartInitialHandshake:157(537158ns) {DropHandshakes:470(1231ns) {DropIncomingHandshake:442(372ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(536ns) PrepareNewSessionHandshake:477(2142ns) {SwitchToState:179(1184ns)}}}}}
2025-11-28T14:51:42.732571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789053628893450:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19442630ns) {Configure:104(19438737ns) {ProcessConfigured:135(1281243ns) {StartInitialHandshake:157(1278402ns) {DropHandshakes:470(1573ns) {DropIncomingHandshake:442(449ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:477(1164ns) {SwitchToState:179(118ns)}}}}}
2025-11-28T14:51:42.733171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789051298256189:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18496555ns) {Configure:104(18493516ns) {ProcessConfigured:135(948856ns) {StartInitialHandshake:157(947781ns) {DropHandshakes:470(988ns) {DropIncomingHandshake:442(174ns) DropOutgoingHandshake:457(152ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(1072ns) {SwitchToState:179(150ns)}}}}}
2025-11-28T14:51:42.732918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789050091923422:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(120438802ns) {Configure:104(120433071ns) {ProcessConfigured:135(1184341ns) {StartInitialHandshake:157(1183812ns) {DropHandshakes:470(773ns) {DropIncomingHandshake:442(251ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(342ns) PrepareNewSessionHandshake:477(1279ns) {SwitchToState:179(163ns)}}}}}
2025-11-28T14:51:42.732453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789052326848152:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28242424ns) {Configure:104(28236681ns) {ProcessConfigured:135(864274ns) {StartInitialHandshake:157(720806ns) {DropHandshakes:470(1174ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(619ns) PrepareNewSessionHandshake:477(1359ns) {SwitchToState:179(375ns)}}}}}
2025-11-28T14:51:42.733379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789053628893385:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29423391ns) {Configure:104(29417632ns) {ProcessConfigured:135(1269345ns) {StartInitialHandshake:157(1267748ns) {DropHandshakes:470(948ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(1583ns) {SwitchToState:179(698ns)}}}}}
2025-11-28T14:51:42.733157Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789050091923421:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29020212ns) {Configure:104(29013565ns) {ProcessConfigured:135(635376ns) {StartInitialHandshake:157(632552ns) {DropHandshakes:470(1426ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(1728ns) PrepareNewSessionHandshake:477(1422ns) {SwitchToState:179(571ns)}}}}}
2025-11-28T14:51:42.732741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789052326848153:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21914149ns) {Configure:104(21901730ns) {ProcessConfigured:135(1503113ns) {StartInitialHandshake:157(1500356ns) {DropHandshakes:470(1401ns) {DropIncomingHandshake:442(465ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(576ns) PrepareNewSessionHandshake:477(1456ns) {SwitchToState:179(643ns)}}}}}
2025-11-28T14:51:42.733171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789050763317421:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(90568520ns) {Configure:104(90561322ns) {ProcessConfigured:135(786597ns) {StartInitialHandshake:157(785040ns) {DropHandshakes:470(756ns) {DropIncomingHandshake:442(252ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:477(998ns) {SwitchToState:179(135ns)}}}}}
2025-11-28T14:51:42.733329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789052326848154:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21006522ns) {Configure:104(21000076ns) {ProcessConfigured:135(577277ns) {StartInitialHandshake:157(574954ns) {DropHandshakes:470(1201ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(1713ns) {SwitchToState:179(574ns)}}}}}
2025-11-28T14:51:42.733187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789050091923485:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20304672ns) {Configure:104(20299100ns) {ProcessConfigured:135(1478314ns) {StartInitialHandshake:157(1476283ns) {DropHandshakes:470(1163ns) {DropIncomingHandshake:442(366ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(520ns) PrepareNewSessionHandshake:477(1563ns) {SwitchToState:179(440ns)}}}}}
2025-11-28T14:51:42.733352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789050763317479:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25173835ns) {Configure:104(25167812ns) {ProcessConfigured:135(1222614ns) {StartInitialHandshake:157(1220003ns) {DropHandshakes:470(1199ns) {DropIncomingHandshake:442(338ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:477(1457ns) {SwitchToState:179(582ns)}}}}}
2025-11-28T14:51:42.733849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789050091923486:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19910271ns) {Configure:104(19906305ns) {ProcessConfigured:135(1121390ns) {StartInitialHandshake:157(1119125ns) {DropHandshakes:470(1192ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:477(1481ns) {SwitchToState:179(625ns)}}}}}
2025-11-28T14:51:42.734143Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789053628893449:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21245141ns) {Configure:104(21237881ns) {ProcessConfigured:135(1947446ns) {StartInitialHandshake:157(1945881ns) {DropHandshakes:470(639ns) {DropIncomingHandshake:442(296ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:477(971ns) {SwitchToState:179(299ns)}}}}}
2025-11-28T14:51:42.733395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789050763317420:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(118809130ns) {Configure:104(118801800ns) {ProcessConfigured:135(1401545ns) {StartInitialHandshake:157(1400169ns) {DropHandshakes:470(893ns) {DropIncomingHandshake:442(277ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(959ns) {SwitchToState:179(136ns)}}}}}
2025-11-28T14:51:42.733420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789050763317493:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19416339ns) {Configure:104(19412945ns) {ProcessConfigured:135(911660ns) {StartInitialHandshake:157(910469ns) {DropHandshakes:470(600ns) {DropIncomingHandshake:442(118ns) DropOutgoingHandshake:457(153ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:477(928ns) {SwitchToState:179(63ns)}}}}}
2025-11-28T14:51:42.734078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789052556805935:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20809719ns) {Configure:104(20806007ns) {ProcessConfigured:135(630810ns) {StartInitialHandshake:157(629880ns) {DropHandshakes:470(945ns) {DropIncomingHandshake:442(182ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(463ns) {SwitchToState:179(54ns)}}}}}
2025-11-28T14:51:42.734392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789052556805872:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(92752292ns) {Configure:104(47940526ns) {ProcessConfigured:135(397334ns) {StartInitialHandshake:157(395542ns) {DropHandshakes:470(1022ns) {DropIncomingHandshake:442(285ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:477(899ns) {SwitchToState:179(347ns)}}}}}
2025-11-28T14:51:42.734873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789051298256107:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21109822ns) {Configure:104(21104487ns) {ProcessConfigured:135(1709893ns) {StartInitialHandshake:157(1707798ns) {DropHandshakes:470(291273ns) {DropIncomingHandshake:442(1188ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(1319ns) PrepareNewSessionHandshake:477(1201ns) {SwitchToState:179(522ns)}}}}}
2025-11-28T14:51:42.734394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789052556805873:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34369604ns) {Configure:104(34364408ns) {ProcessConfigured:135(757250ns) {StartInitialHandshake:157(756289ns) {DropHandshakes:470(743ns) {DropIncomingHandshake:442(209ns) DropOutgoingHandshake:457(168ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:477(438ns) {SwitchToState:179(111ns)}}}}}
2025-11-28T14:51:42.734696Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789052556805874:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31085598ns) {Configure:104(31079715ns) {ProcessConfigured:135(1196145ns) {StartInitialHandshake:157(1193919ns) {DropHandshakes:470(1129ns) {DropIncomingHandshake:442(360ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(937ns) PrepareNewSessionHandshake:477(871ns) {SwitchToState:179(272ns)}}}}}
2025-11-28T14:51:42.734758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789052556805936:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20323031ns) {Configure:104(20319771ns) {ProcessConfigured:135(1141908ns) {StartInitialHandshake:157(1140584ns) {DropHandshakes:470(845ns) {DropIncomingHandshake:442(254ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(254ns) PrepareNewSessionHandshake:477(771ns) {SwitchToState:179(100ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.739860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789050763317507:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.739913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789050763317508:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:51:42.749876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789050091923500:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10823
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.750223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052556805950:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34091
No RDMA context found for GID: 2025-11-28T14:51:42.751058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789050091923501:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10823
2025-11-28T14:51:42.750422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789050763317508:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.750930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789050763317493:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.750849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052556805951:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34091
2025-11-28T14:51:42.751604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789051298256203:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46680
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.751980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789050091923500:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.751919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789051298256204:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46680
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.751639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789052556805941:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.751591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789050763317507:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.751607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789052326848168:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-28T14:51:42.751987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789050091923501:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.752570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789050763317421:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.751674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789052556805941:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.752825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789051298256205:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46680
2025-11-28T14:51:42.751690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789052326848169:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-28T14:51:42.752404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789050091923502:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10823
2025-11-28T14:51:42.752825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789051298256206:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46680
2025-11-28T14:51:42.752425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789052326848170:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-28T14:51:42.763088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789050091923503:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10823
2025-11-28T14:51:42.751984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789052556805950:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.762960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789051298256195:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.753046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848170:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.752517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789052556805873:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.774640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789050091923490:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.763004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789051298256195:2129] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.774678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789050091923490:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.770612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789051298256191:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.753095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789052326848171:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-28T14:51:42.752810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052556805952:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34091
2025-11-28T14:51:42.770654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789051298256191:2128] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.770543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848169:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.775939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789051298256203:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.752826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052556805953:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34091
2025-11-28T14:51:42.776686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789051298256195:2129] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:42.763005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789052556805951:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.776717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789051298256191:2128] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:42.763226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789052556805874:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.777307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789051298256188:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.776907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789052556805941:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:42.777354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789051298256187:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.777276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789050091923490:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:42.778026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789052556805935:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.778171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789050763317511:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
2025-11-28T14:51:42.778025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052556805957:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34091
2025-11-28T14:51:42.778049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:42.778510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789050763317512:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
2025-11-28T14:51:42.778196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789052556805935:2109] [node 6] ICP22 created new session: [4:7577789052556805958:2109]
2025-11-28T14:51:42.777505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:42.778704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789050091923484:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.778518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789052556805958:2109] [node 6] ICS09 handshake done sender: [4:7577789052556805941:2110] self: [4:7577789052556805937:0] peer: [6:7577789050763317509:0] socket: 196 qp: -1
2025-11-28T14:51:42.777553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789051298256187:2115] [node 6] ICP22 created new session: [2:7577789051298256210:2115]
2025-11-28T14:51:42.778744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:42.777780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:42.778545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789052556805958:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:42.778580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.778853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789050091923484:2107] [node 4] ICP22 created new session: [3:7577789050091923507:2107]
2025-11-28T14:51:42.779584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789050763317507:2124] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:42.778018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789051298256210:2115] [node 6] ICS09 handshake done sender: [2:7577789051298256191:2128] self: [2:7577789051298256190:0] peer: [6:7577789050763317510:0] socket: 180 qp: -1
2025-11-28T14:51:42.779291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789050091923507:2107] [node 4] ICS09 handshake done sender: [3:7577789050091923490:2110] self: [3:7577789050091923487:0] peer: [4:7577789052556805955:0] socket: 181 qp: -1
2025-11-28T14:51:42.781129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789050763317508:2125] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:42.781225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789052556805950:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:42.778054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789051298256210:2115] [node 6] ICS10 traffic start
2025-11-28T14:51:42.779316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789050091923507:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:42.783479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789050763317421:2049] [node 2] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:51:42.783821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789052556805959:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34091
2025-11-28T14:51:42.779345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:42.784067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789050763317493:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.778167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789051298256188:2107] [node 4] ICP22 created new session: [2:7577789051298256211:2107]
2025-11-28T14:51:42.784105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:42.784113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789052556805873:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.778425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:42.784349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789050091923509:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10823
2025-11-28T14:51:42.784235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789050763317421:2049] [node 2] ICP22 created new session: [6:7577789050763317514:2049]
2025-11-28T14:51:42.784354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:42.778502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789051298256211:2107] [node 4] ICS09 handshake done sender: [2:7577789051298256195:2129] self: [2:7577789051298256192:0] peer: [4:7577789052556805954:0] socket: 182 qp: -1
2025-11-28T14:51:42.784306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789053628893464:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4124
2025-11-28T14:51:42.784652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789052556805873:2049] [node 2] ICP22 created new session: [4:7577789052556805962:2049]
2025-11-28T14:51:42.778574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789051298256211:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:42.784334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789050763317514:2049] [node 2] ICS09 handshake done sender: [6:7577789050763317507:2124] self: [6:7577789050763317510:0] peer: [2:7577789051298256190:0] socket: 188 qp: -1
2025-11-28T14:51:42.784969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789052556805962:2049] [node 2] ICS09 handshake done sender: [4:7577789052556805950:2115] self: [4:7577789052556805954:0] peer: [2:7577789051298256192:0] socket: 197 qp: -1
2025-11-28T14:51:42.784355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789050763317513:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
2025-11-28T14:51:42.785003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789052556805962:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:42.778612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:42.785174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789052556805951:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:42.784392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789050763317514:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.784397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:42.785190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:42.785558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789050091923509:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.786221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789050091923422:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.784459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789050763317493:2107] [node 4] ICP22 created new session: [6:7577789050763317515:2107]
2025-11-28T14:51:42.784847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789050763317515:2107] [node 4] ICS09 handshake done sender: [6:7577789050763317508:2125] self: [6:7577789050763317509:0] peer: [4:7577789052556805937:0] socket: 200 qp: -1
2025-11-28T14:51:42.787420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789052556805874:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.787475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:42.786338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789053628893465:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4124
2025-11-28T14:51:42.787870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789052556805874:2050] [node 3] ICP22 created new session: [4:7577789052556805964:2050]
2025-11-28T14:51:42.784948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789050763317515:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:42.788033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789052556805964:2050] [node 3] ICS09 handshake done sender: [4:7577789052556805951:2116] self: [4:7577789052556805955:0] peer: [3:7577789050091923487:0] socket: 206 qp: -1
2025-11-28T14:51:42.788282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789051298256198:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.785135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:42.785500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:42.788187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789052556805964:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:42.788352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789051298256198:2131] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:51:42.788231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0
2025-11-28T14:51:42.789005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789053628893466:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4124
2025-11-28T14:51:42.789274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789051298256198:2131] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:42.789173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789053628893465:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.789340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789053628893450:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.789648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789052556805944:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.789637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789051298256107:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.789678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789052556805944:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.790189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:42.790229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789051298256107:2049] [node 3] ICP22 created new session: [2:7577789051298256217:2049]
2025-11-28T14:51:42.789847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789050091923512:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10823
2025-11-28T14:51:42.790422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789051298256217:2049] [node 3] ICS09 handshake done sender: [2:7577789051298256198:2131] self: [2:7577789051298256194:0] peer: [3:7577789050091923511:0] socket: 231 qp: -1
2025-11-28T14:51:42.790453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789051298256217:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:42.790498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:42.790521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789050091923509:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:42.790677Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789052556805944:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:42.790948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789050091923422:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.791000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:42.791087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789050091923422:2049] [node 2] ICP22 created new session: [3:7577789050091923513:2049]
2025-11-28T14:51:42.791113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789050091923513:2049] [node 2] ICS09 handshake done sender: [3:7577789050091923509:2119] self: [3:7577789050091923511:0] peer: [2:7577789051298256194:0] socket: 233 qp: -1
2025-11-28T14:51:42.791137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789050091923513:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:42.791162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789052556805936:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.791169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789053628893468:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4124
2025-11-28T14:51:42.791316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:42.791167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:42.791512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789052556805936:2108] [node 5] ICP22 created new session: [4:7577789052556805967:2108]
2025-11-28T14:51:42.791645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789052556805967:2108] [node 5] ICS09 handshake done sender: [4:7577789052556805944:2112] self: [4:7577789052556805938:0] peer: [5:7577789053628893467:0] socket: 236 qp: -1
2025-11-28T14:51:42.791888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789053628893465:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:42.791774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789052556805967:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:42.791891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:42.792083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789053628893450:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.792145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:42.792323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789053628893450:2108] [node 4] ICP22 created new session: [5:7577789053628893469:2108]
2025-11-28T14:51:42.792444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789053628893469:2108] [node 4] ICS09 handshake done sender: [5:7577789053628893465:2117] self: [5:7577789053628893467:0] peer: [4:7577789052556805938:0] socket: 237 qp: -1
2025-11-28T14:51:42.792513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789053628893469:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:42.792554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:42.796011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789050091923494:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.806179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848168:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.809961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789052326848160:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.831722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789050091923502:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.831928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789050091923421:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.832216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848160:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.832697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789052326848160:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.832920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789052326848090:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.833076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:42.833014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789050091923516:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10823
2025-11-28T14:51:42.833805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789052326848090:2049] [node 3] ICP22 created new session: [1:7577789052326848176:2049]
2025-11-28T14:51:42.833872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789052326848176:2049] [node 3] ICS09 handshake done sender: [1:7577789052326848160:2111] self: [1:7577789052326848155:0] peer: [3:7577789050091923515:0] socket: 194 qp: -1
2025-11-28T14:51:42.833930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789052326848176:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:42.833982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:42.834166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789050091923502:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:42.834428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789050091923421:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.834478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:42.834766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789050091923421:2048] [node 1] ICP22 created new session: [3:7577789050091923517:2048]
2025-11-28T14:51:42.834849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789050091923517:2048] [node 1] ICS09 handshake done sender: [3:7577789050091923502:2117] self: [3:7577789050091923515:0] peer: [1:7577789052326848155:0] socket: 219 qp: -1
2025-11-28T14:51:42.834892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789050091923517:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:42.834963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:42.835667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789050091923421:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.837793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789053628893471:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4124
2025-11-28T14:51:42.837802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789053628893472:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4124
2025-11-28T14:51:42.838674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789053628893472:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.838787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789050091923493:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.838907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789053628893387:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.861421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789050091923496:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.861472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789050091923496:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.862464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789050091923496:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.862990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789050091923486:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.863064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:42.863079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789051298256204:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.863187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789050091923486:2108] [node 5] ICP22 created new session: [3:7577789050091923521:2108]
2025-11-28T14:51:42.863279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789051298256107:2049] [node 3] ICP18 handshake (actor [2:7577789051298256204:2134]) is too old
2025-11-28T14:51:42.863301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789050091923521:2108] [node 5] ICS09 handshake done sender: [3:7577789050091923496:2114] self: [3:7577789050091923492:0] peer: [5:7577789053628893473:0] socket: 186 qp: -1
2025-11-28T14:51:42.863388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789050091923521:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:42.863371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789053628893474:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4124
2025-11-28T14:51:42.863434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:42.864043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789053628893387:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18218) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789053628893472:2121] held: no
2025-11-28T14:51:42.864799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789053628893472:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:42.865255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789053628893387:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.865304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:42.865355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789053628893387:2050] [node 3] ICP22 created new session: [5:7577789053628893476:2050]
2025-11-28T14:51:42.865473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789053628893476:2050] [node 3] ICS09 handshake done sender: [5:7577789053628893472:2121] self: [5:7577789053628893473:0] peer: [3:7577789050091923492:0] socket: 247 qp: -1
2025-11-28T14:51:42.865506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789053628893476:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:42.865536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.865922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789050763317519:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
2025-11-28T14:51:42.867229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789050763317519:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.867375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789050763317479:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.867539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789053628893460:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.867560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789053628893460:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:42.870116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789052326848159:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.886493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789053628893460:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:42.893342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789053628893449:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.893633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:42.893699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789050763317521:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
2025-11-28T14:51:42.893721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789051298256206:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.893692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789053628893449:2109] [node 6] ICP22 created new session: [5:7577789053628893480:2109]
2025-11-28T14:51:42.893774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789053628893480:2109] [node 6] ICS09 handshake done sender: [5:7577789053628893460:2115] self: [5:7577789053628893456:0] peer: [6:7577789050763317520:0] socket: 184 qp: -1
2025-11-28T14:51:42.893821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789053628893480:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:42.893864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:42.893948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789051298256106:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:42.893986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789050763317519:2129] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.894387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789050763317479:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.894240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848159:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.894550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:42.894673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789050763317479:2105] [node 5] ICP22 created new session: [6:7577789050763317522:2105]
2025-11-28T14:51:42.894740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789050763317522:2105] [node 5] ICS09 handshake done sender: [6:7577789050763317519:2129] self: [6:7577789050763317520:0] peer: [5:7577789053628893456:0] socket: 191 qp: -1
2025-11-28T14:51:42.894548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789052326848179:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-28T14:51:42.894789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789050763317522:2105] [node 5] ICS10 traffic start
2025-11-28T14:51:42.894682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789052326848159:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:42.894828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:42.896916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789052326848089:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:42.896981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.897020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789052326848089:2048] [node 2] ICP22 created new session: [1:7577789052326848182:2048]
2025-11-28T14:51:42.902001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789052326848182:2048] [node 2] ICS09 handshake done sender: [1:7577789052326848159:2110] self: [1:7577789052326848156:0] peer: [2:7577789051298256219:0] socket: 195 qp: -1
2025-11-28T14:51:42.902276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789052326848182:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:42.902188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789051298256106:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789051298256206:2136] held: no
2025-11-28T14:51:42.902324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:42.903552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789051298256220:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46680
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:42.903987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848179:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.903865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789050763317524:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
2025-11-28T14:51:42.904212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789051298256206:2136] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:42.904556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789051298256106:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:42.904707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:42.904749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789051298256106:2048] [node 1] ICP22 created new session: [2:7577789051298256221:2048]
2025-11-28T14:51:42.904812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789051298256221:2048] [node 1] ICS09 handshake done sender: [2:7577789051298256206:2136] self: [2:7577789051298256219:0] peer: [1:7577789052326848156:0] socket: 216 qp: -1
2025-11-28T14:51:42.904846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789051298256221:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:42.904893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:42.925391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789050763317502:2122] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.926918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789051298256205:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.927096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789051298256187:2115] [node 6] ICP18 handshake (actor [2:7577789051298256205:2135]) is too old
2025-11-28T14:51:42.944381Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789053628893458:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:42.967615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789052556805952:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:42.967777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789052556805936:2108] [node 5] ICP18 handshake (actor [4:7577789052556805952:2117]) is too old
2025-11-28T14:51:42.978395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789052326848161:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:43.006133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789052556805953:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.006387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789052556805872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.006704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848161:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:51:43.007239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789052326848161:2112] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:51:43.007369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789056851773266:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34091
2025-11-28T14:51:43.007629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789052556805953:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:43.007626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789052326848152:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.007694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:43.007762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789052326848152:2107] [node 4] ICP22 created new session: [1:7577789056621815482:2107]
2025-11-28T14:51:43.007811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789056621815482:2107] [node 4] ICS09 handshake done sender: [1:7577789052326848161:2112] self: [1:7577789052326848157:0] peer: [4:7577789056851773265:0] socket: 193 qp: -1
2025-11-28T14:51:43.007881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789052556805872:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.007981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:43.007866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789056621815482:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:43.008079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789052556805872:2048] [node 1] ICP22 created new session: [4:7577789056851773267:2048]
2025-11-28T14:51:43.007985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:43.008131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789056851773267:2048] [node 1] ICS09 handshake done sender: [4:7577789052556805953:2118] self: [4:7577789056851773265:0] peer: [1:7577789052326848157:0] socket: 217 qp: -1
2025-11-28T14:51:43.008241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789056851773267:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:43.008475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:43.012002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789052556805946:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:43.021917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848171:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.022106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789052326848152:2107] [node 4] ICP18 handshake (actor [1:7577789052326848171:2118]) is too old
2025-11-28T14:51:43.024623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789052556805945:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:43.025352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789050091923503:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.025498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789050091923484:2107] [node 4] ICP18 handshake (actor [3:7577789050091923503:2118]) is too old
2025-11-28T14:51:43.038562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789051298256199:2132] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:43.046418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789053628893464:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.046717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789053628893386:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.047046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789051298256199:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.047594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789051298256199:2132] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.047819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789051298256189:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.047913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:43.047979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789051298256189:2108] [node 5] ICP22 created new session: [2:7577789055593223520:2108]
2025-11-28T14:51:43.047951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789057923860779:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4124
2025-11-28T14:51:43.048021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789055593223520:2108] [node 5] ICS09 handshake done sender: [2:7577789051298256199:2132] self: [2:7577789051298256197:0] peer: [5:7577789057923860778:0] socket: 230 qp: -1
2025-11-28T14:51:43.048046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789055593223520:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:43.048091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:43.048270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789053628893464:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:43.049327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789053628893386:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789053628893464:2116] held: no
2025-11-28T14:51:43.049796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789053628893386:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.049748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789050091923495:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:43.049843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:43.050103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789053628893386:2049] [node 2] ICP22 created new session: [5:7577789057923860780:2049]
2025-11-28T14:51:43.057865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789057923860780:2049] [node 2] ICS09 handshake done sender: [5:7577789053628893464:2116] self: [5:7577789057923860778:0] peer: [2:7577789051298256197:0] socket: 232 qp: -1
2025-11-28T14:51:43.057931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789057923860780:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:43.058010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:43.058125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789050763317513:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.058346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789050763317422:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.058622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789050091923495:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.059045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789050091923495:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:43.059202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789050091923485:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.059294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:43.059341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789050091923485:2109] [node 6] ICP22 created new session: [3:7577789054386890820:2109]
2025-11-28T14:51:43.059442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789054386890820:2109] [node 6] ICS09 handshake done sender: [3:7577789050091923495:2113] self: [3:7577789050091923488:0] peer: [6:7577789055058284821:0] socket: 234 qp: -1
2025-11-28T14:51:43.059472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789054386890820:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:43.059441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055058284822:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
2025-11-28T14:51:43.059518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:43.059795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789050763317422:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18218) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789050763317513:2128] held: no
2025-11-28T14:51:43.059807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789050763317513:2128] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:43.060284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789050763317422:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.060324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:43.060638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789050763317422:2050] [node 3] ICP22 created new session: [6:7577789055058284823:2050]
2025-11-28T14:51:43.060699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789055058284823:2050] [node 3] ICS09 handshake done sender: [6:7577789050763317513:2128] self: [6:7577789055058284821:0] peer: [3:7577789050091923488:0] socket: 235 qp: -1
2025-11-28T14:51:43.060729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789055058284823:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:43.060794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:43.063618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789050763317500:2120] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:43.070718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789053628893466:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.070926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789053628893449:2109] [node 6] ICP18 handshake (actor [5:7577789053628893466:2118]) is too old
2025-11-28T14:51:43.073305Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789052326848163:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:43.081211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789053628893471:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.081431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789053628893385:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.081680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848163:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.082320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789052326848163:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.082511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789057923860783:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4124
2025-11-28T14:51:43.082578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789052326848153:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.082691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:43.082763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789052326848153:2108] [node 5] ICP22 created new session: [1:7577789056621815485:2108]
2025-11-28T14:51:43.082820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789056621815485:2108] [node 5] ICS09 handshake done sender: [1:7577789052326848163:2113] self: [1:7577789052326848158:0] peer: [5:7577789057923860782:0] socket: 246 qp: -1
2025-11-28T14:51:43.082797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789053628893471:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:43.083108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789056621815485:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:43.083155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:43.083245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789053628893385:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.083304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:43.083451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789053628893385:2048] [node 1] ICP22 created new session: [5:7577789057923860784:2048]
2025-11-28T14:51:43.083803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789057923860784:2048] [node 1] ICS09 handshake done sender: [5:7577789053628893471:2120] self: [5:7577789057923860782:0] peer: [1:7577789052326848158:0] socket: 204 qp: -1
2025-11-28T14:51:43.083846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789057923860784:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:43.083911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:43.084684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789053628893385:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:43.086429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789052326848164:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:43.087056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789050763317524:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.087263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789050763317420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.087516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789052326848164:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.087990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789052326848164:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.088260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055058284826:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24376
2025-11-28T14:51:43.088286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789052326848154:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.088393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:43.088442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789052326848154:2109] [node 6] ICP22 created new session: [1:7577789056621815488:2109]
2025-11-28T14:51:43.088500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789056621815488:2109] [node 6] ICS09 handshake done sender: [1:7577789052326848164:2114] self: [1:7577789052326848162:0] peer: [6:7577789055058284825:0] socket: 254 qp: -1
2025-11-28T14:51:43.088547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789056621815488:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:43.088547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789050763317524:2131] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:43.088596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:43.088736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789050763317420:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.088845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:43.089024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789050763317420:2048] [node 1] ICP22 created new session: [6:7577789055058284827:2048]
2025-11-28T14:51:43.089101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789055058284827:2048] [node 1] ICS09 handshake done sender: [6:7577789050763317524:2131] self: [6:7577789055058284825:0] peer: [1:7577789052326848162:0] socket: 215 qp: -1
2025-11-28T14:51:43.089134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789055058284827:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:43.089177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:43.089847Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789050763317420:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:43.092579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:43.092757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789051298256221:2048] [node 1] ICS01 socket: 216 reason# EndOfStream
2025-11-28T14:51:43.092803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789051298256106:2048] [node 1] ICP30 unregister session Session# [2:7577789051298256221:2048] VirtualId# [2:7577789051298256219:0]
2025-11-28T14:51:43.092832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789051298256221:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.092914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.092972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.093502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:43.093651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789050091923517:2048] [node 1] ICS07 socket disconnect 219 reason# EndOfStream
2025-11-28T14:51:43.093676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789050091923517:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.093740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.093800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789050091923517:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:43.094398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789050091923421:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.094451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789050091923517:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.094486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789050091923421:2048] [node 1] ICP30 unregister session Session# [3:7577789050091923517:2048] VirtualId# [3:7577789050091923515:0]
2025-11-28T14:51:43.094517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.094565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789050091923421:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.094597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.095686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:43.095887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789056851773267:2048] [node 1] ICS07 socket disconnect 217 reason# EndOfStream
2025-11-28T14:51:43.095926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789056851773267:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.096008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.096065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789056851773267:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:43.096344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:43.096555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789057923860784:2048] [node 1] ICS07 socket disconnect 204 reason# EndOfStream
2025-11-28T14:51:43.096589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789057923860784:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.096685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.096714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789052556805872:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.096728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789057923860784:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:43.096760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789056851773267:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.096783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789052556805872:2048] [node 1] ICP30 unregister session Session# [4:7577789056851773267:2048] VirtualId# [4:7577789056851773265:0]
2025-11-28T14:51:43.096900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.096926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789052556805872:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.096965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.097096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:43.097347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789055058284827:2048] [node 1] ICS07 socket disconnect 215 reason# EndOfStream
2025-11-28T14:51:43.097384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789055058284827:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.097449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.097493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789055058284827:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:43.097538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789053628893385:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.097659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789057923860784:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.097712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789053628893385:2048] [node 1] ICP30 unregister session Session# [5:7577789057923860784:2048] VirtualId# [5:7577789057923860782:0]
2025-11-28T14:51:43.097758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.097788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789053628893385:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.097799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.098208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789050763317420:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.098313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789055058284827:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.098354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789050763317420:2048] [node 1] ICP30 unregister session Session# [6:7577789055058284827:2048] VirtualId# [6:7577789055058284825:0]
2025-11-28T14:51:43.098390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.098419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789050763317420:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.098435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.104721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789050091923421:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:43.106731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789052556805872:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:43.107765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789053628893385:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:43.108748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789050763317420:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:43.138937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:43.139183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789050091923513:2049] [node 2] ICS07 socket disconnect 233 reason# EndOfStream
2025-11-28T14:51:43.139218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789050091923513:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.139335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:43.139388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789050091923513:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:43.142010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:43.142141Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789050091923422:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:43.142178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789050091923513:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:43.142249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789052556805962:2049] [node 2] ICS07 socket disconnect 197 reason# EndOfStream
2025-11-28T14:51:43.142284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789052556805962:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.142370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:43.142416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789052556805962:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:43.143257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789050091923422:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.143364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789050091923513:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.143410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789050091923422:2049] [node 2] ICP30 unregister session Session# [3:7577789050091923513:2049] VirtualId# [3:7577789050091923511:0]
2025-11-28T14:51:43.143458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:43.143535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789050091923422:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.143573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.143587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:43.143645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789052556805873:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.143748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789052556805962:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.143788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789052556805873:2049] [node 2] ICP30 unregister session Session# [4:7577789052556805962:2049] VirtualId# [4:7577789052556805954:0]
2025-11-28T14:51:43.143839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789057923860780:2049] [node 2] ICS01 socket: 232 reason# EndOfStream
2025-11-28T14:51:43.143879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:43.143885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789053628893386:2049] [node 2] ICP30 unregister session Session# [5:7577789057923860780:2049] VirtualId# [5:7577789057923860778:0]
2025-11-28T14:51:43.143931Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789052556805873:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.143915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789057923860780:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.143945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.143995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:43.144047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:43.144679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:43.144872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789050763317514:2049] [node 2] ICS07 socket disconnect 188 reason# EndOfStream
2025-11-28T14:51:43.144901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789050763317514:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.144969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:43.145014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789050763317514:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:43.145719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789050763317421:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.145806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789050763317514:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.145843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789050763317421:2049] [node 2] ICP30 unregister session Session# [6:7577789050763317514:2049] VirtualId# [6:7577789050763317510:0]
2025-11-28T14:51:43.145900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:43.145930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789050763317421:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.145944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22710) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.153843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789052556805873:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:43.155882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789050763317421:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:43.166386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:43.166595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789052556805964:2050] [node 3] ICS07 socket disconnect 206 reason# EndOfStream
2025-11-28T14:51:43.166633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789052556805964:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.166734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:43.166788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789052556805964:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:43.167432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:43.167483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789052556805874:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18218) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.167531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789052556805964:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.167637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789052556805874:2050] [node 3] ICP30 unregister session Session# [4:7577789052556805964:2050] VirtualId# [4:7577789052556805955:0]
2025-11-28T14:51:43.167680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:43.167707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789052556805874:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18218) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.167719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18218) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.167711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789053628893476:2050] [node 3] ICS01 socket: 247 reason# EndOfStream
2025-11-28T14:51:43.167754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789053628893387:2050] [node 3] ICP30 unregister session Session# [5:7577789053628893476:2050] VirtualId# [5:7577789053628893473:0]
2025-11-28T14:51:43.167791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789053628893476:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.167912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:43.167989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:43.168420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:43.168620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789055058284823:2050] [node 3] ICS01 socket: 235 reason# EndOfStream
2025-11-28T14:51:43.168668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789050763317422:2050] [node 3] ICP30 unregister session Session# [6:7577789055058284823:2050] VirtualId# [6:7577789055058284821:0]
2025-11-28T14:51:43.168703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789055058284823:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.168791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:43.168857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:43.189687Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:43.189916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789053628893469:2108] [node 4] ICS07 socket disconnect 237 reason# EndOfStream
2025-11-28T14:51:43.189971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789053628893469:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.190081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:43.190130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789053628893469:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:43.190311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:43.190509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789050763317515:2107] [node 4] ICS07 socket disconnect 200 reason# EndOfStream
2025-11-28T14:51:43.190545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789050763317515:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.190632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:43.190677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789050763317515:2107] [node 4] ICS15 start handshake
2025-11-28T14:51:43.190886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789053628893450:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11141) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.190970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789053628893469:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.191030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789053628893450:2108] [node 4] ICP30 unregister session Session# [5:7577789053628893469:2108] VirtualId# [5:7577789053628893467:0]
2025-11-28T14:51:43.191074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:43.191105Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789053628893450:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11141) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.191119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11141) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.191329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789050763317493:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11141) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.191420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789050763317515:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.191459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789050763317493:2107] [node 4] ICP30 unregister session Session# [6:7577789050763317515:2107] VirtualId# [6:7577789050763317509:0]
2025-11-28T14:51:43.191513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:43.191545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789050763317493:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11141) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.191563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11141) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.200932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789050763317493:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:51:43.215039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:43.215305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789050763317522:2105] [node 5] ICS07 socket disconnect 191 reason# EndOfStream
2025-11-28T14:51:43.215353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789050763317522:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.215454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:43.215504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789050763317522:2105] [node 5] ICS15 start handshake
2025-11-28T14:51:43.216251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789050763317479:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7184) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:43.216377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789050763317522:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:43.216421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789050763317479:2105] [node 5] ICP30 unregister session Session# [6:7577789050763317522:2105] VirtualId# [6:7577789050763317520:0]
2025-11-28T14:51:43.216470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:43.216502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789050763317479:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7184) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:43.216520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7184) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:43.282079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789056483423481:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:43.282131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789056483423482:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:43.300840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789056709564449:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:43.300886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789056709564450:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:43.318887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789056399830542:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:43.318919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789056399830543:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:43.335912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789054907040236:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:43.335941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789054907040237:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:43.335958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789054907040238:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:43.355606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789055509551605:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:43.355661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789055509551606:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:43.355697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789055509551607:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:43.373857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789055887733983:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:43.373900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789055887733984:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:43.373923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789055887733985:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:43.432002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789055887734043:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:43.533241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789056709564512:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:43.540505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789055509551670:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:43.540630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789056709564513:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:43.540065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789054907040299:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:43.540865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789054907040300:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:43.541897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789055509551671:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:43.542195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789056483423481:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(111195178ns) {Configure:104(110566753ns) {ProcessConfigured:135(80689ns) {StartInitialHandshake:157(77531ns) {DropHandshakes:470(8353ns) {DropIncomingHandshake:442(656ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(511ns) PrepareNewSessionHandshake:477(922ns) {SwitchToState:179(355ns)}}}}}
2025-11-28T14:51:43.542583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789056709564514:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:43.545047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789056483423563:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:43.545585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789056483423564:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:43.545896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789056483423565:2117] [node 6] ICP01 ready to work
2025-11-28T14:51:43.546398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789055887734043:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(112702217ns) {Configure:104(112697586ns) {ProcessConfigured:135(423916ns) {StartInitialHandshake:157(418721ns) {DropHandshakes:470(2247ns) {DropIncomingHandshake:442(505ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(762ns) PrepareNewSessionHandshake:477(2188ns) {SwitchToState:179(1078ns)}}}}}
2025-11-28T14:51:43.547317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789055887734049:2109] [node 4] ICP01 ready to work
2025-11-28T14:51:43.547472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789056399830542:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(80540281ns) {Configure:104(37888024ns) {ProcessConfigured:135(393569ns) {StartInitialHandshake:157(389921ns) {DropHandshakes:470(2547ns) {DropIncomingHandshake:442(729ns) DropOutgoingHandshake:457(383ns)} GenerateSessionVirtualId:988(3573ns) PrepareNewSessionHandshake:477(2211ns) {SwitchToState:179(1102ns)}}}}}
2025-11-28T14:51:43.548007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789056399830616:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:43.548457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789056399830617:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:43.549014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789056399830618:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.566936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789055509551682:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20330
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.575323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789054907040311:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29431
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.576766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789056483423574:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.576933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789056483423575:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.578343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789056399830625:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1540
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.578792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789056399830626:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1540
2025-11-28T14:51:43.578927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789056483423577:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44599
2025-11-28T14:51:43.582371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789054907040311:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.582793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789056483423580:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44599
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.582918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789056483423581:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44599
2025-11-28T14:51:43.582784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789056399830625:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.582897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789055509551682:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.583200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423574:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.583157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789056399830632:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1540
2025-11-28T14:51:43.584534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789056709564528:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29984
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.583287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423575:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.586051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789056709564529:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29984
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.586390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789056709564528:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.586671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789055509551686:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20330
2025-11-28T14:51:43.586582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789056709564530:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29984
2025-11-28T14:51:43.586656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789056709564531:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29984
2025-11-28T14:51:43.588684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789056709564532:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29984
2025-11-28T14:51:43.588949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789056709564529:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.589205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789054907040315:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29431
2025-11-28T14:51:43.589219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789055509551686:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.589373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789054907040316:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29431
2025-11-28T14:51:43.589449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789054907040237:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.589465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789055509551670:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.589835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789056709564531:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.590031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789056399830626:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.590163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789054907040304:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.590204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789056709564516:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.590191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789054907040304:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.590229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789056709564516:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.592006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789054907040304:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:43.592343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789056399830543:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.592765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789056709564516:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:43.592895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789055509551689:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20330
2025-11-28T14:51:43.592913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423577:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.593224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789054907040299:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.593324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789056709564532:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.593356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:43.593417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789054907040299:2108] [node 5] ICP22 created new session: [4:7577789054907040320:2108]
2025-11-28T14:51:43.593437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789056709564520:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.593461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789054907040320:2108] [node 5] ICS09 handshake done sender: [4:7577789054907040304:2111] self: [4:7577789054907040303:0] peer: [5:7577789055509551688:0] socket: 209 qp: -1
2025-11-28T14:51:43.593462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789056709564520:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.593501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789054907040320:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:43.593531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:43.593548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423581:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.593492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789055509551686:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:43.594187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789056399830632:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.594281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789056709564512:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.594297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789056399830542:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.594328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:43.594369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789056709564512:2107] [node 4] ICP22 created new session: [2:7577789056709564537:2107]
2025-11-28T14:51:43.594414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789056709564537:2107] [node 4] ICS09 handshake done sender: [2:7577789056709564516:2110] self: [2:7577789056709564515:0] peer: [4:7577789054907040317:0] socket: 186 qp: -1
2025-11-28T14:51:43.594255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789055509551670:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.594440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789056709564537:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:43.594478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:43.594294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:43.594477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423567:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.594501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789056483423567:2128] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.594331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789055509551670:2108] [node 4] ICP22 created new session: [5:7577789055509551690:2108]
2025-11-28T14:51:43.594380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789055509551690:2108] [node 4] ICS09 handshake done sender: [5:7577789055509551686:2118] self: [5:7577789055509551688:0] peer: [4:7577789054907040303:0] socket: 226 qp: -1
2025-11-28T14:51:43.594406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789055509551690:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:43.594444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:43.595093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789056709564530:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.595364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423580:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.595130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789054907040316:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.595771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789054907040319:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29431
2025-11-28T14:51:43.595998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789056709564449:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.595828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789056483423567:2128] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:43.596376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789056709564520:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:43.596437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789056483423482:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.596550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.596594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789056483423482:2049] [node 3] ICP22 created new session: [1:7577789056483423588:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.596715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789056483423588:2049] [node 3] ICS09 handshake done sender: [1:7577789056483423567:2128] self: [1:7577789056483423566:0] peer: [3:7577789056399830635:0] socket: 214 qp: -1
2025-11-28T14:51:43.596674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789054907040315:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.596743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789056483423588:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:43.596768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734061:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-28T14:51:43.596773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:43.596845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789056399830636:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1540
2025-11-28T14:51:43.596970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789056709564450:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.597016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.597858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423562:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.597883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789056483423562:2127] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.599159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789056483423562:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:43.597188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789056709564450:2049] [node 3] ICP22 created new session: [2:7577789056709564543:2049]
2025-11-28T14:51:43.598601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789056399830637:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1540
2025-11-28T14:51:43.597226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789056709564543:2049] [node 3] ICS09 handshake done sender: [2:7577789056709564520:2112] self: [2:7577789056709564519:0] peer: [3:7577789056399830633:0] socket: 188 qp: -1
2025-11-28T14:51:43.597255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789056709564543:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:43.597293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.600396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789054907040237:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789054907040311:2115] held: no
2025-11-28T14:51:43.599916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789055509551693:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20330
2025-11-28T14:51:43.602737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789054907040238:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.600283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789055509551694:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20330
2025-11-28T14:51:43.602856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789054907040311:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:43.602616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789055887734061:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.602790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789055887734043:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.603079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789056483423481:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.603141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:43.603181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789056483423481:2048] [node 2] ICP22 created new session: [1:7577789056483423592:2048]
2025-11-28T14:51:43.603214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789056483423592:2048] [node 2] ICS09 handshake done sender: [1:7577789056483423562:2127] self: [1:7577789056483423561:0] peer: [2:7577789056709564542:0] socket: 206 qp: -1
2025-11-28T14:51:43.603675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789056483423592:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:43.603781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789055509551681:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.603708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:43.603750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789054907040237:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.603819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:43.603817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789055509551681:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.604535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789055509551681:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:43.604673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789054907040237:2049] [node 2] ICP22 created new session: [4:7577789054907040324:2049]
2025-11-28T14:51:43.605107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789055509551671:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.605150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789054907040324:2049] [node 2] ICS09 handshake done sender: [4:7577789054907040311:2115] self: [4:7577789054907040317:0] peer: [2:7577789056709564515:0] socket: 202 qp: -1
2025-11-28T14:51:43.605169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:43.605181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789054907040324:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:43.606669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789055509551671:2110] [node 6] ICP22 created new session: [5:7577789055509551696:2110]
2025-11-28T14:51:43.605613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:43.606717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789055509551696:2110] [node 6] ICS09 handshake done sender: [5:7577789055509551681:2116] self: [5:7577789055509551680:0] peer: [6:7577789055887734062:0] socket: 234 qp: -1
2025-11-28T14:51:43.606756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789055509551696:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:43.606726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789056399830632:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:43.606789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:43.608633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734063:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-28T14:51:43.608627Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789056399830542:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789056399830632:2126] held: no
2025-11-28T14:51:43.610307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789056399830542:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.610353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:43.610735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789054907040236:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.610426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789056399830542:2048] [node 1] ICP22 created new session: [3:7577789056399830639:2048]
2025-11-28T14:51:43.610469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789056399830639:2048] [node 1] ICS09 handshake done sender: [3:7577789056399830632:2126] self: [3:7577789056399830635:0] peer: [1:7577789056483423566:0] socket: 218 qp: -1
2025-11-28T14:51:43.610501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789056399830639:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:43.610550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:43.611332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789056709564545:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29984
2025-11-28T14:51:43.612019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789056709564449:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789056709564530:2117] held: no
2025-11-28T14:51:43.623286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789056709564530:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:43.622063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789055509551694:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.624005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789056399830622:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.624239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789056399830622:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.622462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789055887734061:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:43.625551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423569:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.625729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789056483423569:2129] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.623539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789055887734043:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27215) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789055887734061:2115] held: no
2025-11-28T14:51:43.623985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789055509551605:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.623656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789055887734043:2106] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.634250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789056399830625:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:43.624135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789055509551693:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.634556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789056709564522:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.624346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789055509551606:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.623694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:43.624639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789055887734043:2106] [node 5] ICP22 created new session: [6:7577789055887734065:2106]
2025-11-28T14:51:43.624706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789055887734065:2106] [node 5] ICS09 handshake done sender: [6:7577789055887734061:2115] self: [6:7577789055887734062:0] peer: [5:7577789055509551680:0] socket: 237 qp: -1
2025-11-28T14:51:43.624742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789055887734065:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:43.624796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:43.627229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734064:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-28T14:51:43.633723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734067:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-28T14:51:43.634593Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789056709564522:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.636210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789056399830638:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1540
2025-11-28T14:51:43.634600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423571:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.634629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789056483423571:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.634924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734068:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-28T14:51:43.640475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789055887734068:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.640692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789055887734067:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.642167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789055887734049:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.642248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789055887733983:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.642576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789056399830543:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789056399830625:2123] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.644714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789055509551700:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20330
2025-11-28T14:51:43.644726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789056399830543:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.644782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:43.644830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789056399830543:2049] [node 2] ICP22 created new session: [3:7577789056399830641:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.644767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789054907040310:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.643529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789056483423569:2129] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:43.644806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789054907040310:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.646649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789054907040328:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29431
2025-11-28T14:51:43.647478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789054907040310:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:43.644879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789056399830641:2049] [node 2] ICS09 handshake done sender: [3:7577789056399830625:2123] self: [3:7577789056399830633:0] peer: [2:7577789056709564519:0] socket: 203 qp: -1
2025-11-28T14:51:43.646882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789056399830641:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:43.647282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:43.648344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789056483423571:2130] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:43.648504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734072:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-28T14:51:43.649071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789056483423573:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.647271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789056709564449:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.649102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789056483423573:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.649602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789056483423563:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.647624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:43.649627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789056399830638:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.649655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:43.649697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789056483423563:2107] [node 4] ICP22 created new session: [1:7577789056483423597:2107]
2025-11-28T14:51:43.649928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789056483423597:2107] [node 4] ICS09 handshake done sender: [1:7577789056483423569:2129] self: [1:7577789056483423568:0] peer: [4:7577789054907040326:0] socket: 217 qp: -1
2025-11-28T14:51:43.650356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789056483423597:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:43.650420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:43.647708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789056709564449:2048] [node 1] ICP22 created new session: [2:7577789056709564547:2048]
2025-11-28T14:51:43.650550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789055887734068:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:43.647745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789056709564547:2048] [node 1] ICS09 handshake done sender: [2:7577789056709564530:2117] self: [2:7577789056709564542:0] peer: [1:7577789056483423561:0] socket: 219 qp: -1
2025-11-28T14:51:43.650475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789054907040315:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:43.650983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789054907040300:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.647768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789056709564547:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:43.647794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:43.651351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:43.651370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789055887734049:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.651411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789054907040300:2109] [node 6] ICP22 created new session: [4:7577789054907040330:2109]
2025-11-28T14:51:43.651463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:43.651610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789056709564522:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:43.651673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789056483423564:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.651498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789054907040330:2109] [node 6] ICS09 handshake done sender: [4:7577789054907040310:2114] self: [4:7577789054907040309:0] peer: [6:7577789055887734070:0] socket: 189 qp: -1
2025-11-28T14:51:43.651913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:43.651533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789055887734049:2109] [node 4] ICP22 created new session: [6:7577789055887734073:2109]
2025-11-28T14:51:43.651548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789054907040330:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:43.651945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789056399830622:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:43.650475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789055509551694:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:43.651582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:43.651321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789055509551605:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.652108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789056483423564:2108] [node 5] ICP22 created new session: [1:7577789056483423599:2108]
2025-11-28T14:51:43.651582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789055887734073:2109] [node 4] ICS09 handshake done sender: [6:7577789055887734068:2119] self: [6:7577789055887734070:0] peer: [4:7577789054907040309:0] socket: 243 qp: -1
2025-11-28T14:51:43.652175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789056483423599:2108] [node 5] ICS09 handshake done sender: [1:7577789056483423571:2130] self: [1:7577789056483423570:0] peer: [5:7577789055509551698:0] socket: 213 qp: -1
2025-11-28T14:51:43.651609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789055887734073:2109] [node 4] ICS10 traffic start
2025-11-28T14:51:43.651363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:43.652379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789056483423599:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:43.651637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:43.651403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789055509551605:2048] [node 1] ICP22 created new session: [5:7577789055509551701:2048]
2025-11-28T14:51:43.652416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:43.651459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789055509551701:2048] [node 1] ICS09 handshake done sender: [5:7577789055509551694:2121] self: [5:7577789055509551698:0] peer: [1:7577789056483423570:0] socket: 241 qp: -1
2025-11-28T14:51:43.651492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789055509551701:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.651527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:43.652572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789056709564513:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.652629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:43.652609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789056399830644:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1540
2025-11-28T14:51:43.652677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789056709564513:2108] [node 5] ICP22 created new session: [2:7577789056709564550:2108]
2025-11-28T14:51:43.652719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789056709564550:2108] [node 5] ICS09 handshake done sender: [2:7577789056709564522:2113] self: [2:7577789056709564521:0] peer: [5:7577789055509551699:0] socket: 236 qp: -1
2025-11-28T14:51:43.652721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789055887734064:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.652749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789056709564550:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:43.652731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789054907040236:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.652789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:43.652786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789056399830616:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.652812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:43.652838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:43.652848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789054907040236:2048] [node 1] ICP22 created new session: [4:7577789054907040332:2048]
2025-11-28T14:51:43.652957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789054907040332:2048] [node 1] ICS09 handshake done sender: [4:7577789054907040315:2116] self: [4:7577789054907040326:0] peer: [1:7577789056483423568:0] socket: 223 qp: -1
2025-11-28T14:51:43.652993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789054907040332:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:43.652871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789056399830616:2107] [node 4] ICP22 created new session: [3:7577789056399830648:2107]
2025-11-28T14:51:43.653069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:43.652957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789056399830648:2107] [node 4] ICS09 handshake done sender: [3:7577789056399830622:2121] self: [3:7577789056399830621:0] peer: [4:7577789054907040323:0] socket: 222 qp: -1
2025-11-28T14:51:43.653049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789056399830648:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:43.653111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:43.654772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789056483423573:2131] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:43.655183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789056483423565:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.655448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:43.655492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789056483423565:2117] [node 6] ICP22 created new session: [1:7577789056483423602:2117]
2025-11-28T14:51:43.655526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789056483423602:2117] [node 6] ICS09 handshake done sender: [1:7577789056483423573:2131] self: [1:7577789056483423572:0] peer: [6:7577789055887734071:0] socket: 196 qp: -1
2025-11-28T14:51:43.655553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789056483423602:2117] [node 6] ICS10 traffic start
2025-11-28T14:51:43.655583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:43.657890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789054907040238:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789054907040316:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.658114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789054907040334:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29431
2025-11-28T14:51:43.658542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789054907040316:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:43.658791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789054907040238:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.658837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:43.658879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789054907040238:2050] [node 3] ICP22 created new session: [4:7577789054907040335:2050]
2025-11-28T14:51:43.658942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789054907040335:2050] [node 3] ICS09 handshake done sender: [4:7577789054907040316:2117] self: [4:7577789054907040323:0] peer: [3:7577789056399830621:0] socket: 225 qp: -1
2025-11-28T14:51:43.658982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789054907040335:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:43.659017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:43.745202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789056399830644:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.748445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789055887733983:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789055887734067:2118] held: no
2025-11-28T14:51:43.748756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789055887733984:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.748931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734075:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.749170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734076:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-28T14:51:43.749717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789055509551606:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789055509551693:2120] held: no
2025-11-28T14:51:43.749808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789055887734075:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:43.749906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789055887734067:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:43.750013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789055509551703:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20330
2025-11-28T14:51:43.750088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789056709564524:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.750151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789056709564524:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.750211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789055887733983:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.750268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:43.750455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789055887733983:2048] [node 1] ICP22 created new session: [6:7577789055887734078:2048]
2025-11-28T14:51:43.750495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789055887734078:2048] [node 1] ICS09 handshake done sender: [6:7577789055887734067:2118] self: [6:7577789055887734071:0] peer: [1:7577789056483423572:0] socket: 215 qp: -1
2025-11-28T14:51:43.750463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789055509551704:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20330
2025-11-28T14:51:43.750542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789055887734078:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:43.750596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:43.750608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789055509551693:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:43.751202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789055509551606:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.751241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:43.751436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789055887733985:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.751274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789055509551606:2049] [node 2] ICP22 created new session: [5:7577789055509551705:2049]
2025-11-28T14:51:43.751305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789055509551705:2049] [node 2] ICS09 handshake done sender: [5:7577789055509551693:2120] self: [5:7577789055509551699:0] peer: [2:7577789056709564521:0] socket: 238 qp: -1
2025-11-28T14:51:43.751531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789056709564524:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:43.751337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789055509551705:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:43.751360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:43.751826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789055509551704:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.751991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789056399830628:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.752067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789055509551607:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:43.752032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789056399830628:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.752116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734081:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-28T14:51:43.752190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789056709564514:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.752300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:43.752354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789056709564514:2109] [node 6] ICP22 created new session: [2:7577789056709564553:2109]
2025-11-28T14:51:43.752746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789056709564553:2109] [node 6] ICS09 handshake done sender: [2:7577789056709564524:2114] self: [2:7577789056709564523:0] peer: [6:7577789055887734077:0] socket: 194 qp: -1
2025-11-28T14:51:43.752803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789056709564553:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:43.752804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789055887734064:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:43.752839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:43.752839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789056399830624:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:43.752869Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789056399830624:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:43.753930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789055887733984:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.753980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:43.754024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789055887733984:2049] [node 2] ICP22 created new session: [6:7577789055887734082:2049]
2025-11-28T14:51:43.754066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789055887734082:2049] [node 2] ICS09 handshake done sender: [6:7577789055887734064:2117] self: [6:7577789055887734077:0] peer: [2:7577789056709564523:0] socket: 187 qp: -1
2025-11-28T14:51:43.754102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789055887734082:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:43.754131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:43.754191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789056399830628:2125] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:43.754455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789055509551708:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20330
2025-11-28T14:51:43.754633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789056399830624:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:43.754969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789056399830618:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.755025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:43.755073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789056399830618:2109] [node 6] ICP22 created new session: [3:7577789056399830653:2109]
2025-11-28T14:51:43.755116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789055509551704:2124] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:43.755115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789056399830653:2109] [node 6] ICS09 handshake done sender: [3:7577789056399830628:2125] self: [3:7577789056399830627:0] peer: [6:7577789055887734080:0] socket: 255 qp: -1
2025-11-28T14:51:43.755142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789056399830653:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:43.755182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:43.756261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789055509551607:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.756319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:43.756356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789055509551607:2050] [node 3] ICP22 created new session: [5:7577789055509551709:2050]
2025-11-28T14:51:43.756414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789055509551709:2050] [node 3] ICS09 handshake done sender: [5:7577789055509551704:2124] self: [5:7577789055509551707:0] peer: [3:7577789056399830623:0] socket: 212 qp: -1
2025-11-28T14:51:43.756446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789055509551709:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:43.756472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:43.756700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789056399830617:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:43.756757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:43.756805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789056399830617:2108] [node 5] ICP22 created new session: [3:7577789056399830655:2108]
2025-11-28T14:51:43.756855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789056399830655:2108] [node 5] ICS09 handshake done sender: [3:7577789056399830624:2122] self: [3:7577789056399830623:0] peer: [5:7577789055509551707:0] socket: 254 qp: -1
2025-11-28T14:51:43.756885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789056399830655:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:43.756915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:43.756915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789055887734084:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12349
2025-11-28T14:51:43.758179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789055887734075:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:43.758462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789055887733985:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:43.758514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:43.758553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789055887733985:2050] [node 3] ICP22 created new session: [6:7577789055887734085:2050]
2025-11-28T14:51:43.758588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789055887734085:2050] [node 3] ICS09 handshake done sender: [6:7577789055887734075:2121] self: [6:7577789055887734080:0] peer: [3:7577789056399830627:0] socket: 208 qp: -1
2025-11-28T14:51:43.758621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789055887734085:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:43.758648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:43.773495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:43.773735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789056709564547:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-28T14:51:43.773788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789056709564449:2048] [node 1] ICP30 unregister session Session# [2:7577789056709564547:2048] VirtualId# [2:7577789056709564542:0]
2025-11-28T14:51:43.773820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789056709564547:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.773919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.774009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.774553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:43.774749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789056399830639:2048] [node 1] ICS01 socket: 218 reason# EndOfStream
2025-11-28T14:51:43.774800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789056399830542:2048] [node 1] ICP30 unregister session Session# [3:7577789056399830639:2048] VirtualId# [3:7577789056399830635:0]
2025-11-28T14:51:43.774836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789056399830639:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.774922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.774990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.777180Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:43.777327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789054907040332:2048] [node 1] ICS01 socket: 223 reason# EndOfStream
2025-11-28T14:51:43.777360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789054907040236:2048] [node 1] ICP30 unregister session Session# [4:7577789054907040332:2048] VirtualId# [4:7577789054907040326:0]
2025-11-28T14:51:43.777387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789054907040332:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.777438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.777475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.778091Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:43.778327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789055509551701:2048] [node 1] ICS01 socket: 241 reason# EndOfStream
2025-11-28T14:51:43.778403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789055509551605:2048] [node 1] ICP30 unregister session Session# [5:7577789055509551701:2048] VirtualId# [5:7577789055509551698:0]
2025-11-28T14:51:43.778445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789055509551701:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.778562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.778617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.778871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:43.779187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789055887734078:2048] [node 1] ICS01 socket: 215 reason# EndOfStream
2025-11-28T14:51:43.779233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789055887733983:2048] [node 1] ICP30 unregister session Session# [6:7577789055887734078:2048] VirtualId# [6:7577789055887734071:0]
2025-11-28T14:51:43.779259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789055887734078:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.779325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:43.779367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:43.800443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:43.800696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789056399830641:2049] [node 2] ICS01 socket: 203 reason# EndOfStream
2025-11-28T14:51:43.800747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789056399830543:2049] [node 2] ICP30 unregister session Session# [3:7577789056399830641:2049] VirtualId# [3:7577789056399830633:0]
2025-11-28T14:51:43.800787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789056399830641:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.800872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:43.800921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:43.803715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:43.803868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789054907040324:2049] [node 2] ICS01 socket: 202 reason# EndOfStream
2025-11-28T14:51:43.803901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789054907040237:2049] [node 2] ICP30 unregister session Session# [4:7577789054907040324:2049] VirtualId# [4:7577789054907040317:0]
2025-11-28T14:51:43.803921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789054907040324:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.803984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:43.804028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:43.804741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:43.804958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789055509551705:2049] [node 2] ICS01 socket: 238 reason# EndOfStream
2025-11-28T14:51:43.805000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789055509551606:2049] [node 2] ICP30 unregister session Session# [5:7577789055509551705:2049] VirtualId# [5:7577789055509551699:0]
2025-11-28T14:51:43.805039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789055509551705:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.805107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:43.805156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:43.805847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:43.806080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789055887734082:2049] [node 2] ICS01 socket: 187 reason# EndOfStream
2025-11-28T14:51:43.806132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789055887733984:2049] [node 2] ICP30 unregister session Session# [6:7577789055887734082:2049] VirtualId# [6:7577789055887734077:0]
2025-11-28T14:51:43.806175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789055887734082:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.806272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:43.806332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:43.830861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:43.831085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789054907040335:2050] [node 3] ICS01 socket: 225 reason# EndOfStream
2025-11-28T14:51:43.831134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789054907040238:2050] [node 3] ICP30 unregister session Session# [4:7577789054907040335:2050] VirtualId# [4:7577789054907040323:0]
2025-11-28T14:51:43.831170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789054907040335:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.831296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:43.831368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:43.831883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:43.832104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789055509551709:2050] [node 3] ICS01 socket: 212 reason# EndOfStream
2025-11-28T14:51:43.832141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789055509551607:2050] [node 3] ICP30 unregister session Session# [5:7577789055509551709:2050] VirtualId# [5:7577789055509551707:0]
2025-11-28T14:51:43.832171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789055509551709:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.832279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:43.832340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:43.833018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:43.833222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789055887734085:2050] [node 3] ICS01 socket: 208 reason# EndOfStream
2025-11-28T14:51:43.833255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789055887733985:2050] [node 3] ICP30 unregister session Session# [6:7577789055887734085:2050] VirtualId# [6:7577789055887734080:0]
2025-11-28T14:51:43.833281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789055887734085:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.833358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:43.833418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:43.861563Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:43.861802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789055509551690:2108] [node 4] ICS01 socket: 226 reason# EndOfStream
2025-11-28T14:51:43.861845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789055509551670:2108] [node 4] ICP30 unregister session Session# [5:7577789055509551690:2108] VirtualId# [5:7577789055509551688:0]
2025-11-28T14:51:43.861874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789055509551690:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.861958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:43.862026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:43.862522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:43.862713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789055887734073:2109] [node 4] ICS01 socket: 243 reason# EndOfStream
2025-11-28T14:51:43.862751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789055887734049:2109] [node 4] ICP30 unregister session Session# [6:7577789055887734073:2109] VirtualId# [6:7577789055887734070:0]
2025-11-28T14:51:43.862770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789055887734073:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.862840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:43.862879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:43.886014Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:43.886257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789055887734065:2106] [node 5] ICS01 socket: 237 reason# EndOfStream
2025-11-28T14:51:43.886296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789055887734043:2106] [node 5] ICP30 unregister session Session# [6:7577789055887734065:2106] VirtualId# [6:7577789055887734062:0]
2025-11-28T14:51:43.886330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789055887734065:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:43.886432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:43.886491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:43.947908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789058448586408:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:43.947999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789058448586409:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:43.969169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789056635680160:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:43.969200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789056635680161:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:43.988136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789056430069594:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:43.988245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789056430069595:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:44.012600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789059115726965:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:44.012650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789059115726967:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:44.012713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789059115726966:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:44.037396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789061213471611:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:44.037568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789061213471612:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:44.037939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789061213471613:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:44.063542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789058781375686:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:44.063569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789058781375687:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:44.063615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789058781375688:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:44.174362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789059115727029:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:44.174720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789059115727030:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:44.168859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789058781375749:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:44.171252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789060930647520:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:44.172687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789061213471675:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:44.175163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789060930647519:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:44.183952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789061213471676:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:44.184539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789060725036953:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:44.181119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789060930647521:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:44.186211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789060725036954:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:44.186495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789060725036955:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:44.185499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789058781375751:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:44.189757Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789058448586408:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(71575067ns) {Configure:104(71570104ns) {ProcessConfigured:135(277315ns) {StartInitialHandshake:157(274568ns) {DropHandshakes:470(2331ns) {DropIncomingHandshake:442(420ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(635ns) PrepareNewSessionHandshake:477(1121ns) {SwitchToState:179(476ns)}}}}}
2025-11-28T14:51:44.189985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789058448586409:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(70405960ns) {Configure:104(70399023ns) {ProcessConfigured:135(119472ns) {StartInitialHandshake:157(117022ns) {DropHandshakes:470(1197ns) {DropIncomingHandshake:442(340ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(497ns) PrepareNewSessionHandshake:477(873ns) {SwitchToState:179(348ns)}}}}}
2025-11-28T14:51:44.190225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789062743553771:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:44.190776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789062743553773:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:44.190792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789062743553772:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:44.191554Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789058781375686:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50193745ns) {Configure:104(50189046ns) {ProcessConfigured:135(317397ns) {StartInitialHandshake:157(315342ns) {DropHandshakes:470(668ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(283ns) PrepareNewSessionHandshake:477(111390ns) {SwitchToState:179(1963ns)}}}}}
2025-11-28T14:51:44.192020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789056430069595:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56914760ns) {Configure:104(56905980ns) {ProcessConfigured:135(335505ns) {StartInitialHandshake:157(333246ns) {DropHandshakes:470(4460ns) {DropIncomingHandshake:442(398ns) DropOutgoingHandshake:457(588ns)} GenerateSessionVirtualId:988(587ns) PrepareNewSessionHandshake:477(854ns) {SwitchToState:179(418ns)}}}}}
2025-11-28T14:51:44.190916Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789059115726966:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55472545ns) {Configure:104(49117442ns) {ProcessConfigured:135(336554ns) {StartInitialHandshake:157(334400ns) {DropHandshakes:470(1183ns) {DropIncomingHandshake:442(357ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(622ns) PrepareNewSessionHandshake:477(1380ns) {SwitchToState:179(540ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.212147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789062743553778:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8545
2025-11-28T14:51:44.212460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789061213471690:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8989
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.213483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789060725036964:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62229
2025-11-28T14:51:44.213498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789061213471691:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8989
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.213833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789060725036969:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62229
2025-11-28T14:51:44.214244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789060725036964:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.221272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789060930647530:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.216753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553778:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.221740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789060930647534:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63759
2025-11-28T14:51:44.223520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789060930647533:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63759
2025-11-28T14:51:44.217334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789062743553783:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8545
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.224215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789061213471690:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.224456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789060930647530:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.226193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789061213471675:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.226591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789062743553784:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8545
2025-11-28T14:51:44.224811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789060930647534:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.225195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789060930647535:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.227695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789060930647536:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63759
2025-11-28T14:51:44.228697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375765:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.227960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789059115727036:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.229653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789061213471691:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.229925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789060930647536:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.228007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789059115727036:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:44.230128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553783:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.230472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789060725036972:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62229
2025-11-28T14:51:44.229892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789059115727044:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5969
2025-11-28T14:51:44.230243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789060930647535:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.230606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553784:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.230638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789059115727045:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5969
2025-11-28T14:51:44.230512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789060930647533:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.230757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789059115727036:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:44.233476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789062743553786:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8545
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.240237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789056635680160:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.239772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789060725036969:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.240192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553786:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.240738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789059115727029:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.240219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789062743553787:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8545
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.241416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553768:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.241289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:44.241440Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789062743553768:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:44.241515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789056430069594:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.241470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789059115727029:2108] [node 5] ICP22 created new session: [4:7577789059115727047:2108]
2025-11-28T14:51:44.241834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789061213471694:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8989
2025-11-28T14:51:44.241805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789059115727047:2108] [node 5] ICS09 handshake done sender: [4:7577789059115727036:2113] self: [4:7577789059115727035:0] peer: [5:7577789061213471692:0] socket: 202 qp: -1
2025-11-28T14:51:44.241911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789059115727047:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:44.242083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:44.242787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789061213471690:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.244398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789060725036974:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62229
2025-11-28T14:51:44.244400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553770:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.244282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789061213471675:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.244343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:44.244454Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789062743553770:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:44.244400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789061213471675:2108] [node 4] ICP22 created new session: [5:7577789061213471695:2108]
2025-11-28T14:51:44.244509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789061213471695:2108] [node 4] ICS09 handshake done sender: [5:7577789061213471690:2116] self: [5:7577789061213471692:0] peer: [4:7577789059115727035:0] socket: 203 qp: -1
2025-11-28T14:51:44.244592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789061213471695:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:44.252587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789062743553768:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:44.244814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.258390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789060725036974:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.258685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789060725036975:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62229
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.258905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789062743553770:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:44.258825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375766:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
2025-11-28T14:51:44.259573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789058448586408:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.260452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789058781375766:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.260674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789058781375749:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.260724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789060725036969:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:44.259617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:44.259657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789058448586408:2048] [node 2] ICP22 created new session: [1:7577789062743553794:2048]
2025-11-28T14:51:44.259697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789062743553794:2048] [node 2] ICS09 handshake done sender: [1:7577789062743553768:2110] self: [1:7577789062743553767:0] peer: [2:7577789060930647544:0] socket: 199 qp: -1
2025-11-28T14:51:44.259724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789062743553794:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:44.259754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:44.260183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789058448586409:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.260221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:44.261191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789056430069594:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.261162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789061213471689:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.260258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789058448586409:2049] [node 3] ICP22 created new session: [1:7577789062743553796:2049]
2025-11-28T14:51:44.261241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:44.260297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789062743553796:2049] [node 3] ICS09 handshake done sender: [1:7577789062743553770:2111] self: [1:7577789062743553769:0] peer: [3:7577789060725036973:0] socket: 207 qp: -1
2025-11-28T14:51:44.261197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789061213471689:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:44.260316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789062743553796:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:44.261282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789056430069594:2048] [node 1] ICP22 created new session: [3:7577789060725036977:2048]
2025-11-28T14:51:44.260337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:44.261556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789060725036977:2048] [node 1] ICS09 handshake done sender: [3:7577789060725036969:2116] self: [3:7577789060725036973:0] peer: [1:7577789062743553769:0] socket: 209 qp: -1
2025-11-28T14:51:44.261587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789060725036977:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:44.261631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:44.262298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789061213471689:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.262681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789061213471676:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.262964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:44.264117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789061213471676:2110] [node 6] ICP22 created new session: [5:7577789061213471700:2110]
2025-11-28T14:51:44.264383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789061213471700:2110] [node 6] ICS09 handshake done sender: [5:7577789061213471689:2115] self: [5:7577789061213471688:0] peer: [6:7577789058781375767:0] socket: 239 qp: -1
2025-11-28T14:51:44.264411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789061213471700:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:44.264564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789059115727050:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5969
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.264593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:44.265267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375768:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.266054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789061213471699:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8989
2025-11-28T14:51:44.267929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789058781375749:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7459) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789058781375766:2117] held: no
2025-11-28T14:51:44.272782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789058781375766:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:44.273755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789058781375749:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.273801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:44.273840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789058781375749:2106] [node 5] ICP22 created new session: [6:7577789058781375770:2106]
2025-11-28T14:51:44.274602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789059115727050:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.273897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789058781375770:2106] [node 5] ICS09 handshake done sender: [6:7577789058781375766:2117] self: [6:7577789058781375767:0] peer: [5:7577789061213471688:0] socket: 242 qp: -1
2025-11-28T14:51:44.274618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789061213471699:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.273930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789058781375770:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:44.274780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789061213471611:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.273963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: 2025-11-28T14:51:44.274997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789059115726965:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-28T14:51:44.274987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553781:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.275016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789062743553781:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:44.275503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553777:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.275518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789062743553777:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:44.276876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789060725036979:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62229
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.277920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789059115727052:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5969
2025-11-28T14:51:44.278216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789062743553781:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.278871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789060725036979:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.279037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789061213471703:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8989
2025-11-28T14:51:44.279177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789061213471704:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8989
2025-11-28T14:51:44.279240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789062743553777:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:44.279458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789061213471703:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.279410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789059115727050:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:44.279533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789062743553772:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.279584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789061213471699:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:44.279621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:44.279652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789062743553772:2108] [node 5] ICP22 created new session: [1:7577789062743553802:2108]
2025-11-28T14:51:44.279724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789062743553802:2108] [node 5] ICS09 handshake done sender: [1:7577789062743553781:2115] self: [1:7577789062743553780:0] peer: [5:7577789061213471702:0] socket: 244 qp: -1
2025-11-28T14:51:44.279847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789059115726965:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.282384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789061213471611:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.282596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:44.281354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:44.279752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789062743553802:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:44.286330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789060725036961:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:44.284589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789061213471613:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.286350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789061213471611:2048] [node 1] ICP22 created new session: [5:7577789061213471705:2048]
2025-11-28T14:51:44.281419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789059115726965:2048] [node 1] ICP22 created new session: [4:7577789059115727053:2048]
2025-11-28T14:51:44.280171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:44.287961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789061213471705:2048] [node 1] ICS09 handshake done sender: [5:7577789061213471699:2119] self: [5:7577789061213471702:0] peer: [1:7577789062743553780:0] socket: 246 qp: -1
2025-11-28T14:51:44.281479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789059115727053:2048] [node 1] ICS09 handshake done sender: [4:7577789059115727050:2118] self: [4:7577789059115727051:0] peer: [1:7577789062743553776:0] socket: 243 qp: -1
2025-11-28T14:51:44.288002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789061213471705:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:44.294697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:44.281515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789059115727053:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:44.280633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789062743553771:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.295941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789060725036963:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.281551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:44.280692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:44.295993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789060725036963:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:44.280727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789062743553771:2107] [node 4] ICP22 created new session: [1:7577789062743553804:2107]
2025-11-28T14:51:44.296893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789059115727044:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.282145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789062743553804:2107] [node 4] ICS09 handshake done sender: [1:7577789062743553777:2113] self: [1:7577789062743553776:0] peer: [4:7577789059115727051:0] socket: 220 qp: -1
2025-11-28T14:51:44.282211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789062743553804:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:44.299032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789059115726967:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.282277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.299262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375772:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
2025-11-28T14:51:44.299432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789060725036961:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.299887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789060725036961:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:44.300085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789060725036963:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:44.300649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789060725036953:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.300810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.300681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789059115727057:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5969
2025-11-28T14:51:44.301291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789060725036953:2107] [node 4] ICP22 created new session: [3:7577789060725036984:2107]
2025-11-28T14:51:44.301550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789060725036984:2107] [node 4] ICS09 handshake done sender: [3:7577789060725036961:2112] self: [3:7577789060725036960:0] peer: [4:7577789059115727056:0] socket: 221 qp: -1
2025-11-28T14:51:44.301584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789060725036984:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:44.301626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:44.301764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789060725036954:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.301817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789061213471708:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8989
2025-11-28T14:51:44.301818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:44.301852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789060725036954:2108] [node 5] ICP22 created new session: [3:7577789060725036986:2108]
2025-11-28T14:51:44.302430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789061213471703:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:44.301908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789060725036986:2108] [node 5] ICS09 handshake done sender: [3:7577789060725036963:2113] self: [3:7577789060725036962:0] peer: [5:7577789061213471706:0] socket: 218 qp: -1
2025-11-28T14:51:44.302806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789061213471613:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.301936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789060725036986:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:44.302285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:44.304091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:44.304647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789061213471613:2050] [node 3] ICP22 created new session: [5:7577789061213471709:2050]
2025-11-28T14:51:44.304398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789059115726967:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5619) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789059115727044:2116] held: no
2025-11-28T14:51:44.304735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789061213471709:2050] [node 3] ICS09 handshake done sender: [5:7577789061213471703:2120] self: [5:7577789061213471706:0] peer: [3:7577789060725036962:0] socket: 251 qp: -1
2025-11-28T14:51:44.304701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789059115727044:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:44.304786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789061213471709:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:44.304835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:44.305224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789058781375772:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.305206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789059115726967:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.306914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789058781375751:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.305352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:44.307192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789059115726967:2050] [node 3] ICP22 created new session: [4:7577789059115727058:2050]
2025-11-28T14:51:44.307289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789059115727058:2050] [node 3] ICS09 handshake done sender: [4:7577789059115727044:2116] self: [4:7577789059115727056:0] peer: [3:7577789060725036960:0] socket: 229 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.307323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789059115727058:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:44.307369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:44.307456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375773:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
2025-11-28T14:51:44.307737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789059115727040:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.307773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789059115727040:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.309145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789059115727040:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:44.309602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375775:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
2025-11-28T14:51:44.309732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789059115727030:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.309811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789058781375772:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:44.309789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:44.309829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789059115727030:2109] [node 6] ICP22 created new session: [4:7577789059115727061:2109]
2025-11-28T14:51:44.309867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789059115727061:2109] [node 6] ICS09 handshake done sender: [4:7577789059115727040:2115] self: [4:7577789059115727039:0] peer: [6:7577789058781375774:0] socket: 192 qp: -1
2025-11-28T14:51:44.309904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789059115727061:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:44.309995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:44.310236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789058781375773:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.310282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789058781375751:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.310333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:44.310373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789058781375751:2108] [node 4] ICP22 created new session: [6:7577789058781375776:2108]
2025-11-28T14:51:44.310420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789058781375776:2108] [node 4] ICS09 handshake done sender: [6:7577789058781375772:2119] self: [6:7577789058781375774:0] peer: [4:7577789059115727039:0] socket: 196 qp: -1
2025-11-28T14:51:44.310448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789058781375776:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:44.310479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:44.310846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789058781375688:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.311143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789060725036966:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.311174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789060725036966:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:44.312226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789060725036966:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.312378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375779:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
2025-11-28T14:51:44.312395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789060725036955:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.312444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:44.312468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789060725036955:2109] [node 6] ICP22 created new session: [3:7577789060725036990:2109]
2025-11-28T14:51:44.312567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789060725036990:2109] [node 6] ICS09 handshake done sender: [3:7577789060725036966:2115] self: [3:7577789060725036965:0] peer: [6:7577789058781375778:0] socket: 236 qp: -1
2025-11-28T14:51:44.312600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789060725036990:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:44.312623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:44.314022Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789058781375688:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5619) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789058781375773:2120] held: no
2025-11-28T14:51:44.314198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789058781375773:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:44.314439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789058781375688:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.314468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:44.314499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789058781375688:2050] [node 3] ICP22 created new session: [6:7577789058781375780:2050]
2025-11-28T14:51:44.314528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789058781375780:2050] [node 3] ICS09 handshake done sender: [6:7577789058781375773:2120] self: [6:7577789058781375778:0] peer: [3:7577789060725036965:0] socket: 197 qp: -1
2025-11-28T14:51:44.314554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789058781375780:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:44.314585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:44.320846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789062743553775:2112] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:44.328801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789058781375765:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.328976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789058781375686:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.329321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553775:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.330309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789062743553775:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.330518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375783:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
2025-11-28T14:51:44.330810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789062743553773:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.330903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:44.330977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789062743553773:2109] [node 6] ICP22 created new session: [1:7577789062743553807:2109]
2025-11-28T14:51:44.330981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789058781375765:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:44.331033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789062743553807:2109] [node 6] ICS09 handshake done sender: [1:7577789062743553775:2112] self: [1:7577789062743553774:0] peer: [6:7577789058781375782:0] socket: 222 qp: -1
2025-11-28T14:51:44.331075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789062743553807:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:44.331241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:44.331402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789058781375686:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.331751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:44.336447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789058781375686:2048] [node 1] ICP22 created new session: [6:7577789058781375784:2048]
2025-11-28T14:51:44.340469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789058781375784:2048] [node 1] ICS09 handshake done sender: [6:7577789058781375765:2116] self: [6:7577789058781375782:0] peer: [1:7577789062743553774:0] socket: 223 qp: -1
2025-11-28T14:51:44.340841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789058781375784:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:44.340908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:44.341964Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789058781375686:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:44.355446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789060930647524:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:44.363556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789062743553787:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.363698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789058448586408:2048] [node 2] ICP18 handshake (actor [1:7577789062743553787:2119]) is too old
2025-11-28T14:51:44.374319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789060930647525:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:44.378355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789060725036972:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.378520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789056430069595:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.381948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789056635680160:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789060930647533:2116] held: no
2025-11-28T14:51:44.382106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789060930647525:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.382375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789060930647545:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63759
2025-11-28T14:51:44.383025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789060930647533:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.383409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789060930647525:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:44.383418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789060725036993:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62229
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.383620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789061213471711:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8989
2025-11-28T14:51:44.383732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789060725036972:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:44.383921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789056635680160:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.384113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789056430069595:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.383964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:44.384164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:44.384114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789056635680160:2048] [node 1] ICP22 created new session: [2:7577789060930647548:2048]
2025-11-28T14:51:44.384189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789056430069595:2049] [node 2] ICP22 created new session: [3:7577789060725036994:2049]
2025-11-28T14:51:44.384165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789060930647548:2048] [node 1] ICS09 handshake done sender: [2:7577789060930647533:2116] self: [2:7577789060930647544:0] peer: [1:7577789062743553767:0] socket: 213 qp: -1
2025-11-28T14:51:44.384217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789060725036994:2049] [node 2] ICS09 handshake done sender: [3:7577789060725036972:2117] self: [3:7577789060725036992:0] peer: [2:7577789060930647523:0] socket: 232 qp: -1
2025-11-28T14:51:44.384237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789060725036994:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:44.384189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789060930647548:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:44.384215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:44.384254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:44.385404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789056635680161:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.385450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:44.385651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789056635680161:2049] [node 3] ICP22 created new session: [2:7577789060930647550:2049]
2025-11-28T14:51:44.385719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789060930647550:2049] [node 3] ICS09 handshake done sender: [2:7577789060930647525:2111] self: [2:7577789060930647523:0] peer: [3:7577789060725036992:0] socket: 227 qp: -1
2025-11-28T14:51:44.385745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789060930647550:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:44.385770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.388814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375786:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
2025-11-28T14:51:44.404084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789060930647528:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:44.411554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789059115727045:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.411739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789059115726966:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.425301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789060930647529:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:44.432581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789060930647528:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.432738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789061213471711:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.433051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789060930647528:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:44.433020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789061213471612:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.433391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789059115727064:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5969
2025-11-28T14:51:44.433549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789060930647529:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.433633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789060930647520:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.433694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:44.433726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789059115727045:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:44.433738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789060930647520:2107] [node 4] ICP22 created new session: [2:7577789060930647554:2107]
2025-11-28T14:51:44.433797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789060930647554:2107] [node 4] ICS09 handshake done sender: [2:7577789060930647528:2112] self: [2:7577789060930647526:0] peer: [4:7577789059115727063:0] socket: 228 qp: -1
2025-11-28T14:51:44.433888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789060930647554:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:44.433942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:44.434005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789059115726966:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.436663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:44.436752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789059115726966:2049] [node 2] ICP22 created new session: [4:7577789059115727065:2049]
2025-11-28T14:51:44.436785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789059115727065:2049] [node 2] ICS09 handshake done sender: [4:7577789059115727045:2117] self: [4:7577789059115727063:0] peer: [2:7577789060930647526:0] socket: 230 qp: -1
2025-11-28T14:51:44.436825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789059115727065:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:44.436855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.438328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789060930647529:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:44.438409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789061213471713:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8989
2025-11-28T14:51:44.438638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789060930647519:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.438677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:44.438706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789060930647519:2108] [node 5] ICP22 created new session: [2:7577789060930647557:2108]
2025-11-28T14:51:44.438691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789061213471711:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:44.438729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789060930647557:2108] [node 5] ICS09 handshake done sender: [2:7577789060930647529:2113] self: [2:7577789060930647527:0] peer: [5:7577789061213471712:0] socket: 231 qp: -1
2025-11-28T14:51:44.438745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789060930647557:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:44.439189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789061213471612:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.438843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:44.439248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:44.439493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789061213471612:2049] [node 2] ICP22 created new session: [5:7577789061213471714:2049]
2025-11-28T14:51:44.439534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789061213471714:2049] [node 2] ICS09 handshake done sender: [5:7577789061213471711:2123] self: [5:7577789061213471712:0] peer: [2:7577789060930647527:0] socket: 257 qp: -1
2025-11-28T14:51:44.439621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789061213471714:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:44.439660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:44.453512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789060930647532:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:44.454542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789058781375786:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.454750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789058781375687:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.454989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789060930647532:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.455455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789060930647532:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.455652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789058781375788:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33559
2025-11-28T14:51:44.455746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789060930647521:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.455803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:44.455834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789060930647521:2109] [node 6] ICP22 created new session: [2:7577789060930647560:2109]
2025-11-28T14:51:44.455871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789060930647560:2109] [node 6] ICS09 handshake done sender: [2:7577789060930647532:2115] self: [2:7577789060930647531:0] peer: [6:7577789058781375787:0] socket: 215 qp: -1
2025-11-28T14:51:44.455918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789060930647560:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:44.455957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:44.455949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789058781375786:2124] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:44.456355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789058781375687:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:44.456499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:44.456565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789058781375687:2049] [node 2] ICP22 created new session: [6:7577789058781375789:2049]
2025-11-28T14:51:44.456616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789058781375789:2049] [node 2] ICS09 handshake done sender: [6:7577789058781375786:2124] self: [6:7577789058781375787:0] peer: [2:7577789060930647531:0] socket: 219 qp: -1
2025-11-28T14:51:44.456663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789058781375789:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:44.456699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:44.476786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:44.477030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789060930647548:2048] [node 1] ICS01 socket: 213 reason# EndOfStream
2025-11-28T14:51:44.477074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789056635680160:2048] [node 1] ICP30 unregister session Session# [2:7577789060930647548:2048] VirtualId# [2:7577789060930647544:0]
2025-11-28T14:51:44.477109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789060930647548:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.477210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:44.477258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:44.477890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:44.478138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789060725036977:2048] [node 1] ICS07 socket disconnect 209 reason# EndOfStream
2025-11-28T14:51:44.478189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789060725036977:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.478268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:44.478317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789060725036977:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:44.479057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789056430069594:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:44.479169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789060725036977:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:44.479228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789056430069594:2048] [node 1] ICP30 unregister session Session# [3:7577789060725036977:2048] VirtualId# [3:7577789060725036973:0]
2025-11-28T14:51:44.479276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:44.479338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789056430069594:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.479352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.480062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:44.480231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789059115727053:2048] [node 1] ICS01 socket: 243 reason# EndOfStream
2025-11-28T14:51:44.480263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789059115726965:2048] [node 1] ICP30 unregister session Session# [4:7577789059115727053:2048] VirtualId# [4:7577789059115727051:0]
2025-11-28T14:51:44.480287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789059115727053:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.480375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:44.480428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:44.480992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:44.481211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789061213471705:2048] [node 1] ICS07 socket disconnect 246 reason# EndOfStream
2025-11-28T14:51:44.481242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789061213471705:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.481316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:44.481388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789061213471705:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:44.481886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:44.482131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789061213471611:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:44.482138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789058781375784:2048] [node 1] ICS07 socket disconnect 223 reason# EndOfStream
2025-11-28T14:51:44.482176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789061213471705:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:44.482262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789061213471611:2048] [node 1] ICP30 unregister session Session# [5:7577789061213471705:2048] VirtualId# [5:7577789061213471702:0]
2025-11-28T14:51:44.482173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789058781375784:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.482307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:44.482287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:44.482355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789061213471611:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.482349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789058781375784:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:44.482364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.483092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789058781375686:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:44.483176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789058781375784:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:44.483221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789058781375686:2048] [node 1] ICP30 unregister session Session# [6:7577789058781375784:2048] VirtualId# [6:7577789058781375782:0]
2025-11-28T14:51:44.483262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:44.483292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789058781375686:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.483302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24865) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.489072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789056430069594:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:44.489752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:44.489926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789060725036994:2049] [node 2] ICS01 socket: 232 reason# EndOfStream
2025-11-28T14:51:44.489968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789056430069595:2049] [node 2] ICP30 unregister session Session# [3:7577789060725036994:2049] VirtualId# [3:7577789060725036992:0]
2025-11-28T14:51:44.490006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789060725036994:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.490097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:44.490155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:44.492187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789061213471611:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:44.492531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:44.492783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789059115727065:2049] [node 2] ICS07 socket disconnect 230 reason# EndOfStream
2025-11-28T14:51:44.492817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789059115727065:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.492916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:44.492968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789059115727065:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:44.493175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789058781375686:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:44.493528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:44.493692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789059115726966:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4089) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:44.493787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789059115727065:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:44.493900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789059115726966:2049] [node 2] ICP30 unregister session Session# [4:7577789059115727065:2049] VirtualId# [4:7577789059115727063:0]
2025-11-28T14:51:44.493814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789061213471714:2049] [node 2] ICS07 socket disconnect 257 reason# EndOfStream
2025-11-28T14:51:44.493845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789061213471714:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.493918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:44.493969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:44.493979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789061213471714:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:44.494005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789059115726966:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.494018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.494589Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:44.494743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789058781375789:2049] [node 2] ICS07 socket disconnect 219 reason# EndOfStream
2025-11-28T14:51:44.494777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789058781375789:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.494855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:44.494895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789058781375789:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:44.494858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789061213471612:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4089) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:44.494924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789061213471714:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:44.494963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789061213471612:2049] [node 2] ICP30 unregister session Session# [5:7577789061213471714:2049] VirtualId# [5:7577789061213471712:0]
2025-11-28T14:51:44.495007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:44.495039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789061213471612:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.495057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.495684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789058781375687:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4089) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:44.495790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789058781375789:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:44.495823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789058781375687:2049] [node 2] ICP30 unregister session Session# [6:7577789058781375789:2049] VirtualId# [6:7577789058781375787:0]
2025-11-28T14:51:44.495860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:44.495889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789058781375687:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.495910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.503472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789059115726966:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:44.504465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789061213471612:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:44.505495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789058781375687:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:44.517925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:44.518093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789059115727058:2050] [node 3] ICS01 socket: 229 reason# EndOfStream
2025-11-28T14:51:44.518125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789059115726967:2050] [node 3] ICP30 unregister session Session# [4:7577789059115727058:2050] VirtualId# [4:7577789059115727056:0]
2025-11-28T14:51:44.518150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789059115727058:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.518234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:44.518287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:44.518994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:44.519187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789061213471709:2050] [node 3] ICS07 socket disconnect 251 reason# EndOfStream
2025-11-28T14:51:44.519226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789061213471709:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.519314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:44.519384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789061213471709:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:44.519948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:44.520165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789058781375780:2050] [node 3] ICS01 socket: 197 reason# EndOfStream
2025-11-28T14:51:44.520220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789061213471613:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5619) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:44.520228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789058781375688:2050] [node 3] ICP30 unregister session Session# [6:7577789058781375780:2050] VirtualId# [6:7577789058781375778:0]
2025-11-28T14:51:44.520289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789058781375780:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.520356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789061213471709:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:44.520401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:44.520467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789061213471613:2050] [node 3] ICP30 unregister session Session# [5:7577789061213471709:2050] VirtualId# [5:7577789061213471706:0]
2025-11-28T14:51:44.520457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:44.520524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:44.520570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789061213471613:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5619) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.520586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5619) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.530086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789061213471613:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:44.555355Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:44.555579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789061213471695:2108] [node 4] ICS07 socket disconnect 203 reason# EndOfStream
2025-11-28T14:51:44.555612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789061213471695:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.555683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:44.555739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789061213471695:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:44.556275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:44.556471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789058781375776:2108] [node 4] ICS07 socket disconnect 196 reason# EndOfStream
2025-11-28T14:51:44.556500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789058781375776:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.556506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789061213471675:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20759) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:44.556572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:44.556613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789058781375776:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:44.556604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789061213471695:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:44.556836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789061213471675:2108] [node 4] ICP30 unregister session Session# [5:7577789061213471695:2108] VirtualId# [5:7577789061213471692:0]
2025-11-28T14:51:44.556876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:44.556909Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789061213471675:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20759) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.556926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20759) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.557474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789058781375751:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20759) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:44.557523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789058781375776:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:44.557592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789058781375751:2108] [node 4] ICP30 unregister session Session# [6:7577789058781375776:2108] VirtualId# [6:7577789058781375774:0]
2025-11-28T14:51:44.557639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:44.557667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789058781375751:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20759) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.557768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20759) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:44.567972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789058781375751:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:44.583839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:44.584008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789058781375770:2106] [node 5] ICS01 socket: 242 reason# EndOfStream
2025-11-28T14:51:44.584038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789058781375749:2106] [node 5] ICP30 unregister session Session# [6:7577789058781375770:2106] VirtualId# [6:7577789058781375767:0]
2025-11-28T14:51:44.584062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789058781375770:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:44.584138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:44.584180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:44.658579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789061169972164:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:44.658609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789061169972165:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:44.677900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789062357188402:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:44.677977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789062357188403:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:44.698606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789059880421519:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:44.698609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789059880421520:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:44.718333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789058801945411:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:44.718415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789058801945412:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:44.718454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789058801945413:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:44.737403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789059625827538:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:44.737403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789059625827539:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:44.737439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789059625827540:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:44.760096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789059924685237:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:44.760162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789059924685238:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:44.760873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789059924685239:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:44.860758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789059924685294:2101] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.863589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789062357188462:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57677
2025-11-28T14:51:44.860077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789059924685293:2098] [node 5] ICP01 ready to work
2025-11-28T14:51:44.943910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789059625827601:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:44.931937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789061169972230:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:44.945512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789059880421582:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:44.946418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789061169972231:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:44.947749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789058801945484:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:44.948033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789062357188467:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:44.947523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789059625827602:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:44.948796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789059880421583:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:44.946935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789061169972232:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:44.948079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789058801945483:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:44.949085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789062357188468:2110] [node 5] ICP01 ready to work
2025-11-28T14:51:44.950417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789059880421584:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:44.951413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789062357188469:2111] [node 6] ICP01 ready to work
2025-11-28T14:51:44.957260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789061169972165:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(98057368ns) {Configure:104(98049866ns) {ProcessConfigured:135(237910ns) {StartInitialHandshake:157(235299ns) {DropHandshakes:470(2568ns) {DropIncomingHandshake:442(1076ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(1220ns) PrepareNewSessionHandshake:477(1457ns) {SwitchToState:179(514ns)}}}}}
2025-11-28T14:51:44.959382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789059924685237:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(66235218ns) {Configure:104(27148632ns) {ProcessConfigured:135(255841ns) {StartInitialHandshake:157(254508ns) {DropHandshakes:470(1010ns) {DropIncomingHandshake:442(294ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(388ns) PrepareNewSessionHandshake:477(846ns) {SwitchToState:179(239ns)}}}}}
2025-11-28T14:51:44.959401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789062357188403:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(78356897ns) {Configure:104(78352440ns) {ProcessConfigured:135(282932ns) {StartInitialHandshake:157(276035ns) {DropHandshakes:470(1233ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(1409ns) {SwitchToState:179(468ns)}}}}}
2025-11-28T14:51:44.959771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789062357188402:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(90197463ns) {Configure:104(90181694ns) {ProcessConfigured:135(271621ns) {StartInitialHandshake:157(269855ns) {DropHandshakes:470(3810ns) {DropIncomingHandshake:442(460ns) DropOutgoingHandshake:457(2456ns)} GenerateSessionVirtualId:988(945ns) PrepareNewSessionHandshake:477(1109ns) {SwitchToState:179(556ns)}}}}}
2025-11-28T14:51:44.960857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789059880421520:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(86822394ns) {Configure:104(86819428ns) {ProcessConfigured:135(418371ns) {StartInitialHandshake:157(416889ns) {DropHandshakes:470(3955ns) {DropIncomingHandshake:442(386ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(1050ns) {SwitchToState:179(335ns)}}}}}
2025-11-28T14:51:44.960560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789059924685294:2101] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(98206623ns) {Configure:104(98202649ns) {ProcessConfigured:135(452931ns) {StartInitialHandshake:157(451916ns) {DropHandshakes:470(3798ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(404ns) PrepareNewSessionHandshake:477(817ns) {SwitchToState:179(156ns)}}}}}
2025-11-28T14:51:44.960898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789059880421519:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(90016157ns) {Configure:104(90009572ns) {ProcessConfigured:135(1058996ns) {StartInitialHandshake:157(1056892ns) {DropHandshakes:470(1328ns) {DropIncomingHandshake:442(236ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(143ns) PrepareNewSessionHandshake:477(943ns) {SwitchToState:179(194ns)}}}}}
2025-11-28T14:51:44.960966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789059924685293:2098] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(84969184ns) {Configure:104(84961447ns) {ProcessConfigured:135(501687ns) {StartInitialHandshake:157(500630ns) {DropHandshakes:470(687ns) {DropIncomingHandshake:442(199ns) DropOutgoingHandshake:457(146ns)} GenerateSessionVirtualId:988(176ns) PrepareNewSessionHandshake:477(1012ns) {SwitchToState:179(228ns)}}}}}
2025-11-28T14:51:44.961826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789059625827539:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(68831041ns) {Configure:104(49503352ns) {ProcessConfigured:135(348844ns) {StartInitialHandshake:157(347429ns) {DropHandshakes:470(1282ns) {DropIncomingHandshake:442(225ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(681ns) PrepareNewSessionHandshake:477(631ns) {SwitchToState:179(141ns)}}}}}
2025-11-28T14:51:44.965006Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789058801945413:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16981352ns) {Configure:104(16968663ns) {ProcessConfigured:135(598687ns) {StartInitialHandshake:157(596734ns) {DropHandshakes:470(1408ns) {DropIncomingHandshake:442(434ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(568ns) PrepareNewSessionHandshake:477(1150ns) {SwitchToState:179(409ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.973416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789059625827612:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57113
2025-11-28T14:51:44.975488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789061169972243:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55364
2025-11-28T14:51:44.975700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789062357188462:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.979396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789062357188402:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:44.977963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972225:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.977999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789061169972225:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.981408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789058801945483:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16708579ns) {Configure:104(16701919ns) {ProcessConfigured:135(2197224ns) {StartInitialHandshake:157(2194893ns) {DropHandshakes:470(1569ns) {DropIncomingHandshake:442(616ns) DropOutgoingHandshake:457(308ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:477(11422ns) {SwitchToState:179(929ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.984884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789058801945493:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27201
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.984999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789058801945495:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27201
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.985247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789058801945498:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27201
2025-11-28T14:51:44.983123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789061169972225:2107] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:44.985375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789059880421593:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1037
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.986961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789061169972244:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.986987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789058801945499:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27201
2025-11-28T14:51:44.987506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789059880421599:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1037
2025-11-28T14:51:44.987771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789061169972245:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55364
2025-11-28T14:51:44.988442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789059625827612:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.988464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789058801945494:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27201
2025-11-28T14:51:44.990027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972243:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.990088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789061169972164:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:44.988482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789058801945495:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:44.990464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789059880421593:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.990100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789058801945493:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.990346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972244:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:44.992989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789062357188484:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57677
2025-11-28T14:51:44.991549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:44.990380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789058801945411:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.001463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789062357188486:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57677
2025-11-28T14:51:44.991987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789061169972164:2048] [node 2] ICP22 created new session: [1:7577789061169972249:2048]
2025-11-28T14:51:44.990509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789058801945494:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.000832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789061169972247:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55364
2025-11-28T14:51:45.001840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789062357188485:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57677
2025-11-28T14:51:45.002174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789059880421600:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1037
2025-11-28T14:51:45.000850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789061169972249:2048] [node 2] ICS09 handshake done sender: [1:7577789061169972225:2107] self: [1:7577789061169972224:0] peer: [2:7577789062357188483:0] socket: 198 qp: -1
2025-11-28T14:51:45.001532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789061169972249:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:45.002404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789059880421519:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.001783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: 2025-11-28T14:51:45.002144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972236:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:51:45.002168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789061169972236:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.004229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972234:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.004255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789061169972234:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.004783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789061169972248:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55364
2025-11-28T14:51:45.005293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789062357188484:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.005348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789061169972236:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.005826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789061169972230:2108] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.006058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789063920794914:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57113
2025-11-28T14:51:45.005986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:45.006237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789063096912802:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27201
2025-11-28T14:51:45.006040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789061169972230:2108] [node 4] ICP22 created new session: [1:7577789065464939552:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.006263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789065464939552:2108] [node 4] ICS09 handshake done sender: [1:7577789061169972236:2112] self: [1:7577789061169972235:0] peer: [4:7577789058801945504:0] socket: 202 qp: -1
2025-11-28T14:51:45.006309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789065464939552:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:45.006361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.006483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789064175388898:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1037
2025-11-28T14:51:45.006528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789061169972234:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.007087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789058801945493:2123] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:45.007447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789061169972165:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.007496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.008271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789059880421593:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:45.008393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789063920794914:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.008904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789059625827538:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.008066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789061169972165:2049] [node 3] ICP22 created new session: [1:7577789065464939555:2049]
2025-11-28T14:51:45.008967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789058801945411:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.008299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789065464939555:2049] [node 3] ICS09 handshake done sender: [1:7577789061169972234:2111] self: [1:7577789061169972233:0] peer: [3:7577789064175388897:0] socket: 201 qp: -1
2025-11-28T14:51:45.009025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.008655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789065464939555:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:45.009128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789058801945411:2048] [node 1] ICP22 created new session: [4:7577789063096912803:2048]
2025-11-28T14:51:45.008706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:45.009722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789059880421519:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.009309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789063096912803:2048] [node 1] ICS09 handshake done sender: [4:7577789058801945493:2123] self: [4:7577789058801945504:0] peer: [1:7577789061169972235:0] socket: 215 qp: -1
2025-11-28T14:51:45.009554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972238:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.009993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:45.009579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789061169972238:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.009557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789063096912803:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:45.010056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652611:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.010115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789059880421519:2048] [node 1] ICP22 created new session: [3:7577789064175388899:2048]
2025-11-28T14:51:45.009661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:45.010866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789064175388899:2048] [node 1] ICS09 handshake done sender: [3:7577789059880421593:2114] self: [3:7577789064175388897:0] peer: [1:7577789061169972233:0] socket: 204 qp: -1
2025-11-28T14:51:45.011037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789064175388899:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:45.011070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:45.011375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064219652611:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.011588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789059924685237:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.011535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789061169972238:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:45.011922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972241:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.011951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789061169972241:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.011967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789061169972232:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.012004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.012170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789061169972232:2109] [node 5] ICP22 created new session: [1:7577789065464939558:2109]
2025-11-28T14:51:45.012834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789063920794916:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57113
2025-11-28T14:51:45.012769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789065464939558:2109] [node 5] ICS09 handshake done sender: [1:7577789061169972238:2113] self: [1:7577789061169972237:0] peer: [5:7577789063920794915:0] socket: 242 qp: -1
2025-11-28T14:51:45.012849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789065464939558:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:45.013459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789059625827538:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17624) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789063920794914:2116] held: no
2025-11-28T14:51:45.012893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.014476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789061169972241:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:45.014215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789063920794914:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:45.014634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789059625827538:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.014904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789061169972231:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.014868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:45.015017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:45.015018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789059625827538:2048] [node 1] ICP22 created new session: [5:7577789063920794917:2048]
2025-11-28T14:51:45.015063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789061169972231:2110] [node 6] ICP22 created new session: [1:7577789065464939561:2110]
2025-11-28T14:51:45.015250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789065464939561:2110] [node 6] ICS09 handshake done sender: [1:7577789061169972241:2114] self: [1:7577789061169972239:0] peer: [6:7577789064219652612:0] socket: 248 qp: -1
2025-11-28T14:51:45.015140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789063920794917:2048] [node 1] ICS09 handshake done sender: [5:7577789063920794914:2116] self: [5:7577789063920794915:0] peer: [1:7577789061169972237:0] socket: 243 qp: -1
2025-11-28T14:51:45.015286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789065464939561:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:45.015309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789063920794917:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:51:45.015335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
2025-11-28T14:51:45.015370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:45.015705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789059924685237:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17624) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789064219652611:2115] held: no
2025-11-28T14:51:45.016116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652613:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.016856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064219652611:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:45.017292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789059924685237:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.017333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:45.017576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789059924685237:2048] [node 1] ICP22 created new session: [6:7577789064219652615:2048]
2025-11-28T14:51:45.017724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064219652615:2048] [node 1] ICS09 handshake done sender: [6:7577789064219652611:2115] self: [6:7577789064219652612:0] peer: [1:7577789061169972239:0] socket: 249 qp: -1
2025-11-28T14:51:45.017925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064219652615:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:45.018165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.018616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789064175388901:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1037
2025-11-28T14:51:45.018826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789064175388902:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1037
2025-11-28T14:51:45.019171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789064175388901:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.019657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789064175388902:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.023388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789062357188472:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.030820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789059880421599:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.030853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789062357188487:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57677
2025-11-28T14:51:45.031006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789062357188462:2107] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:45.031447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789062357188402:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.031532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:45.031592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789062357188402:2048] [node 1] ICP22 created new session: [2:7577789066652155784:2048]
2025-11-28T14:51:45.031655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789066652155784:2048] [node 1] ICS09 handshake done sender: [2:7577789062357188462:2107] self: [2:7577789062357188483:0] peer: [1:7577789061169972224:0] socket: 199 qp: -1
2025-11-28T14:51:45.031705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789066652155784:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:45.031749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:45.032658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789062357188473:2113] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.033836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972245:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.033944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789061169972164:2048] [node 2] ICP18 handshake (actor [1:7577789061169972245:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.040672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652617:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.045094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789059880421589:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.052689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972247:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.052923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789061169972165:2049] [node 3] ICP18 handshake (actor [1:7577789061169972247:2118]) is too old
2025-11-28T14:51:45.052979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789059880421520:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.053339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789062357188472:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.053768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789062357188472:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.053957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789063920794920:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57113
2025-11-28T14:51:45.054223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789062357188403:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.054274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.054319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789062357188403:2049] [node 3] ICP22 created new session: [2:7577789066652155789:2049]
2025-11-28T14:51:45.054407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789066652155789:2049] [node 3] ICS09 handshake done sender: [2:7577789062357188472:2112] self: [2:7577789062357188470:0] peer: [3:7577789064175388905:0] socket: 218 qp: -1
2025-11-28T14:51:45.054442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789066652155789:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:45.054474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.055154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789064175388907:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1037
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.055886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789063920794921:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57113
2025-11-28T14:51:45.056774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789058801945488:2119] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.081446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789059880421590:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.088780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789059880421600:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.088836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789059880421599:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:45.088866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789058801945499:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.089032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789058801945413:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.089177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789059880421520:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.089220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:45.089259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789059880421520:2049] [node 2] ICP22 created new session: [3:7577789064175388908:2049]
2025-11-28T14:51:45.089293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789064175388908:2049] [node 2] ICS09 handshake done sender: [3:7577789059880421599:2116] self: [3:7577789064175388905:0] peer: [2:7577789062357188470:0] socket: 223 qp: -1
2025-11-28T14:51:45.089320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789064175388908:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:45.089352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:45.089905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789059880421590:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.090364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789059880421590:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.090610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789063096912806:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27201
2025-11-28T14:51:45.090646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789059880421582:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.090691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:45.090720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789059880421582:2107] [node 4] ICP22 created new session: [3:7577789064175388912:2107]
2025-11-28T14:51:45.090751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789064175388912:2107] [node 4] ICS09 handshake done sender: [3:7577789059880421590:2112] self: [3:7577789059880421586:0] peer: [4:7577789063096912805:0] socket: 231 qp: -1
2025-11-28T14:51:45.090770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789064175388912:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:45.090789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.091057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789058801945499:2128] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.092031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789058801945413:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.092084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.092142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789058801945413:2050] [node 3] ICP22 created new session: [4:7577789063096912807:2050]
2025-11-28T14:51:45.092180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789063096912807:2050] [node 3] ICS09 handshake done sender: [4:7577789058801945499:2128] self: [4:7577789063096912805:0] peer: [3:7577789059880421586:0] socket: 233 qp: -1
2025-11-28T14:51:45.092214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789063096912807:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:45.092241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.092869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652618:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.093663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789058801945413:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.095944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789063920794922:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57113
2025-11-28T14:51:45.101973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789058801945490:2121] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.110685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789061169972248:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.110941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789061169972230:2108] [node 4] ICP18 handshake (actor [1:7577789061169972248:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.111871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652619:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.128604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789062357188475:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.141938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789059880421588:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.150809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789062357188485:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.150947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789062357188403:2049] [node 3] ICP18 handshake (actor [2:7577789062357188485:2118]) is too old
2025-11-28T14:51:45.155294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789058801945489:2120] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.162298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789058801945498:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.162348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789062357188487:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.162464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789058801945412:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.162741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789062357188475:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.163120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789062357188475:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.163421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789063096912812:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27201
2025-11-28T14:51:45.163463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789062357188467:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.163522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:45.163568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789062357188467:2108] [node 4] ICP22 created new session: [2:7577789066652155794:2108]
2025-11-28T14:51:45.163619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789066652155794:2108] [node 4] ICS09 handshake done sender: [2:7577789062357188475:2114] self: [2:7577789062357188474:0] peer: [4:7577789063096912811:0] socket: 222 qp: -1
2025-11-28T14:51:45.163658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789066652155794:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:45.163693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.163861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789058801945498:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:45.164075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789058801945412:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.164149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:45.164196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789058801945412:2049] [node 2] ICP22 created new session: [4:7577789063096912813:2049]
2025-11-28T14:51:45.164255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789063096912813:2049] [node 2] ICS09 handshake done sender: [4:7577789058801945498:2127] self: [4:7577789063096912811:0] peer: [2:7577789062357188474:0] socket: 232 qp: -1
2025-11-28T14:51:45.164289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789063096912813:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:45.164319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:45.165269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789062357188479:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.171795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064219652617:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.172019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789059924685238:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.172312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789062357188479:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.172774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789062357188479:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.172946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789062357188469:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.172890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652621:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.173065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:45.173108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789062357188469:2111] [node 6] ICP22 created new session: [2:7577789066652155797:2111]
2025-11-28T14:51:45.173153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789066652155797:2111] [node 6] ICS09 handshake done sender: [2:7577789062357188479:2116] self: [2:7577789062357188477:0] peer: [6:7577789064219652620:0] socket: 216 qp: -1
2025-11-28T14:51:45.173181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789066652155797:2111] [node 6] ICS10 traffic start
2025-11-28T14:51:45.173209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:45.173217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064219652617:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:45.173352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789059924685238:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.173403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:45.173498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789059924685238:2049] [node 2] ICP22 created new session: [6:7577789064219652622:2049]
2025-11-28T14:51:45.173590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064219652622:2049] [node 2] ICS09 handshake done sender: [6:7577789064219652617:2117] self: [6:7577789064219652620:0] peer: [2:7577789062357188477:0] socket: 254 qp: -1
2025-11-28T14:51:45.173637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064219652622:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:45.173725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:45.175270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789059880421592:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.182779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789063920794920:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.183022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789059625827540:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.183347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789059880421592:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.183910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789059880421592:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.184148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789059880421583:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.184226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.184248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789063920794924:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57113
2025-11-28T14:51:45.184300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789059880421583:2108] [node 5] ICP22 created new session: [3:7577789064175388916:2108]
2025-11-28T14:51:45.184409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789064175388916:2108] [node 5] ICS09 handshake done sender: [3:7577789059880421592:2113] self: [3:7577789059880421591:0] peer: [5:7577789063920794923:0] socket: 219 qp: -1
2025-11-28T14:51:45.184462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789064175388916:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:45.184471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789063920794920:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.184504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.184763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789059625827540:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.184844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.184913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789059625827540:2050] [node 3] ICP22 created new session: [5:7577789063920794925:2050]
2025-11-28T14:51:45.184972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789063920794925:2050] [node 3] ICS09 handshake done sender: [5:7577789063920794920:2118] self: [5:7577789063920794923:0] peer: [3:7577789059880421591:0] socket: 226 qp: -1
2025-11-28T14:51:45.185014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789063920794925:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:45.185073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.186221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652624:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.187638Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789062357188478:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.195175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789063920794921:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.195443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789059625827539:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.195678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789062357188478:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.195754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064219652624:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.195964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789059924685293:2098] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.196196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789062357188478:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:45.196231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789059625827616:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.196264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789059625827616:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.196300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789063920794929:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57113
2025-11-28T14:51:45.196444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789062357188468:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.197500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.197561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789062357188468:2110] [node 5] ICP22 created new session: [2:7577789066652155800:2110]
2025-11-28T14:51:45.197520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789063920794921:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:45.197843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789059625827539:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.197840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789066652155800:2110] [node 5] ICS09 handshake done sender: [2:7577789062357188478:2115] self: [2:7577789062357188476:0] peer: [5:7577789063920794928:0] socket: 256 qp: -1
2025-11-28T14:51:45.197886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:45.197957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789066652155800:2110] [node 5] ICS10 traffic start
2025-11-28T14:51:45.197999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.197931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789059625827539:2049] [node 2] ICP22 created new session: [5:7577789063920794931:2049]
2025-11-28T14:51:45.197972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789063920794931:2049] [node 2] ICS09 handshake done sender: [5:7577789063920794921:2119] self: [5:7577789063920794928:0] peer: [2:7577789062357188476:0] socket: 257 qp: -1
2025-11-28T14:51:45.198008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789063920794931:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.198113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:45.199230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652626:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.198192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789059625827616:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:45.199424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789059625827539:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:45.199864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064219652624:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:45.199853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789059625827602:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.199918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:45.199959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789059625827602:2109] [node 6] ICP22 created new session: [5:7577789063920794933:2109]
2025-11-28T14:51:45.200036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789059924685293:2098] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.200136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789063920794933:2109] [node 6] ICS09 handshake done sender: [5:7577789059625827616:2115] self: [5:7577789059625827611:0] peer: [6:7577789064219652625:0] socket: 211 qp: -1
2025-11-28T14:51:45.200109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.200252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789063920794933:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:45.200311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789059924685293:2098] [node 5] ICP22 created new session: [6:7577789064219652627:2098]
2025-11-28T14:51:45.200313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:45.200402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064219652627:2098] [node 5] ICS09 handshake done sender: [6:7577789064219652624:2121] self: [6:7577789064219652625:0] peer: [5:7577789059625827611:0] socket: 205 qp: -1
2025-11-28T14:51:45.200438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064219652627:2098] [node 5] ICS10 traffic start
2025-11-28T14:51:45.200473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.203731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789059880421595:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.211122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064219652618:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.211327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789059924685239:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.211584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789059880421595:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.212080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789059880421595:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.212218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652630:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.212278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789059880421584:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.212353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:45.212424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789059880421584:2109] [node 6] ICP22 created new session: [3:7577789064175388919:2109]
2025-11-28T14:51:45.212525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064219652618:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.212480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789064175388919:2109] [node 6] ICS09 handshake done sender: [3:7577789059880421595:2115] self: [3:7577789059880421594:0] peer: [6:7577789064219652629:0] socket: 238 qp: -1
2025-11-28T14:51:45.212517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789064175388919:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:45.212711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789059924685239:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.212648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:45.212788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.212866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789059924685239:2050] [node 3] ICP22 created new session: [6:7577789064219652631:2050]
2025-11-28T14:51:45.213192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064219652631:2050] [node 3] ICS09 handshake done sender: [6:7577789064219652618:2118] self: [6:7577789064219652629:0] peer: [3:7577789059880421594:0] socket: 260 qp: -1
2025-11-28T14:51:45.213239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064219652631:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:45.213304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:45.213917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789059924685239:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:45.215666Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789058801945492:2122] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.224484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789063920794922:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.224696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789059625827601:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.224947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789058801945492:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.225436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789058801945492:2122] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.225616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789063920794936:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57113
2025-11-28T14:51:45.225665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789058801945484:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.225738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.225782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789058801945484:2108] [node 5] ICP22 created new session: [4:7577789063096912816:2108]
2025-11-28T14:51:45.225829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789063096912816:2108] [node 5] ICS09 handshake done sender: [4:7577789058801945492:2122] self: [4:7577789058801945491:0] peer: [5:7577789063920794935:0] socket: 237 qp: -1
2025-11-28T14:51:45.225861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789063096912816:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:45.225904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.225956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789063920794922:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:45.226200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789059625827601:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.226267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:45.226310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789059625827601:2108] [node 4] ICP22 created new session: [5:7577789063920794937:2108]
2025-11-28T14:51:45.226465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789063920794937:2108] [node 4] ICS09 handshake done sender: [5:7577789063920794922:2120] self: [5:7577789063920794935:0] peer: [4:7577789058801945491:0] socket: 261 qp: -1
2025-11-28T14:51:45.226967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789063920794937:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:45.227017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.229353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789058801945497:2126] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.230076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064219652619:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.230269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789059924685294:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.230487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789058801945497:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.230865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789058801945497:2126] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.231109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064219652634:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45856
2025-11-28T14:51:45.231138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789058801945483:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.231219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:45.231254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789058801945483:2109] [node 6] ICP22 created new session: [4:7577789063096912819:2109]
2025-11-28T14:51:45.231294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789063096912819:2109] [node 6] ICS09 handshake done sender: [4:7577789058801945497:2126] self: [4:7577789058801945496:0] peer: [6:7577789064219652633:0] socket: 235 qp: -1
2025-11-28T14:51:45.231324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789063096912819:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:45.231476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:45.231510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064219652619:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:45.231698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789059924685294:2101] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.231744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:45.232404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789059924685294:2101] [node 4] ICP22 created new session: [6:7577789064219652635:2101]
2025-11-28T14:51:45.232502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064219652635:2101] [node 4] ICS09 handshake done sender: [6:7577789064219652619:2119] self: [6:7577789064219652633:0] peer: [4:7577789058801945496:0] socket: 240 qp: -1
2025-11-28T14:51:45.232537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064219652635:2101] [node 4] ICS10 traffic start
2025-11-28T14:51:45.232584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.265734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:45.265915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789066652155784:2048] [node 1] ICS01 socket: 199 reason# EndOfStream
2025-11-28T14:51:45.265946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789062357188402:2048] [node 1] ICP30 unregister session Session# [2:7577789066652155784:2048] VirtualId# [2:7577789062357188483:0]
2025-11-28T14:51:45.265968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789066652155784:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.266053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.266110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.266825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:45.267084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789064175388899:2048] [node 1] ICS07 socket disconnect 204 reason# EndOfStream
2025-11-28T14:51:45.267129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789064175388899:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.267240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.267297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789064175388899:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:45.268192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789059880421519:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17624) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.268400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789064175388899:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.268443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789059880421519:2048] [node 1] ICP30 unregister session Session# [3:7577789064175388899:2048] VirtualId# [3:7577789064175388897:0]
2025-11-28T14:51:45.268501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.268531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789059880421519:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.268545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.269353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:45.269585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789063096912803:2048] [node 1] ICS07 socket disconnect 215 reason# EndOfStream
2025-11-28T14:51:45.269632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789063096912803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.269721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.269775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789063096912803:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:45.270036Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:45.270225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789063920794917:2048] [node 1] ICS01 socket: 243 reason# EndOfStream
2025-11-28T14:51:45.270268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789059625827538:2048] [node 1] ICP30 unregister session Session# [5:7577789063920794917:2048] VirtualId# [5:7577789063920794915:0]
2025-11-28T14:51:45.270304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789063920794917:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.270379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.270427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.270462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789058801945411:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17624) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.270561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789063096912803:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.270606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789058801945411:2048] [node 1] ICP30 unregister session Session# [4:7577789063096912803:2048] VirtualId# [4:7577789058801945504:0]
2025-11-28T14:51:45.270652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.270684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789058801945411:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.270699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.270745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:45.270867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064219652615:2048] [node 1] ICS01 socket: 249 reason# EndOfStream
2025-11-28T14:51:45.270891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789059924685237:2048] [node 1] ICP30 unregister session Session# [6:7577789064219652615:2048] VirtualId# [6:7577789064219652612:0]
2025-11-28T14:51:45.270912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064219652615:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.270967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.271008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.276842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:45.277021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789064175388908:2049] [node 2] ICS01 socket: 223 reason# EndOfStream
2025-11-28T14:51:45.277065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789059880421520:2049] [node 2] ICP30 unregister session Session# [3:7577789064175388908:2049] VirtualId# [3:7577789064175388905:0]
2025-11-28T14:51:45.277095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789064175388908:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.277201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:45.277271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:45.278593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789059880421519:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:45.279018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:45.279232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789063096912813:2049] [node 2] ICS07 socket disconnect 232 reason# EndOfStream
2025-11-28T14:51:45.279271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789063096912813:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.279365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:45.279413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789063096912813:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:45.279764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:45.279953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789063920794931:2049] [node 2] ICS01 socket: 257 reason# EndOfStream
2025-11-28T14:51:45.279997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789059625827539:2049] [node 2] ICP30 unregister session Session# [5:7577789063920794931:2049] VirtualId# [5:7577789063920794928:0]
2025-11-28T14:51:45.280024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789063920794931:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.280104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:45.280102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789058801945412:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19937) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.280148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:45.280198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789063096912813:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.280243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789058801945412:2049] [node 2] ICP30 unregister session Session# [4:7577789063096912813:2049] VirtualId# [4:7577789063096912811:0]
2025-11-28T14:51:45.280294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:45.280339Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789058801945412:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19937) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.280356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19937) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.280437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:45.280563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789064219652622:2049] [node 2] ICS07 socket disconnect 254 reason# EndOfStream
2025-11-28T14:51:45.280614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064219652622:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.280674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:45.280633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789058801945411:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:45.280708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789064219652622:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:45.281270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789059924685238:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19937) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.281340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064219652622:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.281379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789059924685238:2049] [node 2] ICP30 unregister session Session# [6:7577789064219652622:2049] VirtualId# [6:7577789064219652620:0]
2025-11-28T14:51:45.281414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:45.281452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789059924685238:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19937) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.281463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19937) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.289876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789058801945412:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:45.290923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789059924685238:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:45.291884Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:45.292133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789063096912807:2050] [node 3] ICS07 socket disconnect 233 reason# EndOfStream
2025-11-28T14:51:45.292177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789063096912807:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.292291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:45.292339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789063096912807:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:45.292675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:45.292867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789063920794925:2050] [node 3] ICS07 socket disconnect 226 reason# EndOfStream
2025-11-28T14:51:45.292903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789063920794925:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.292978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:45.293020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789063920794925:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:45.293072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789058801945413:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3332) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.293141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789063096912807:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.293175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789058801945413:2050] [node 3] ICP30 unregister session Session# [4:7577789063096912807:2050] VirtualId# [4:7577789063096912805:0]
2025-11-28T14:51:45.293228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:45.293261Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789058801945413:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3332) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.293276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3332) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.293459Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:45.293735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789059625827540:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3332) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.293694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789064219652631:2050] [node 3] ICS07 socket disconnect 260 reason# EndOfStream
2025-11-28T14:51:45.293734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064219652631:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.293793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789063920794925:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.293822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:45.293866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789064219652631:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:45.293823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789059625827540:2050] [node 3] ICP30 unregister session Session# [5:7577789063920794925:2050] VirtualId# [5:7577789063920794923:0]
2025-11-28T14:51:45.293945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:45.293970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789059625827540:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3332) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.293980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3332) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.294851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789059924685239:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3332) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.294966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064219652631:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.295003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789059924685239:2050] [node 3] ICP30 unregister session Session# [6:7577789064219652631:2050] VirtualId# [6:7577789064219652629:0]
2025-11-28T14:51:45.295050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:45.295075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789059924685239:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3332) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.295091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3332) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.302231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:45.302452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789063920794937:2108] [node 4] ICS07 socket disconnect 261 reason# EndOfStream
2025-11-28T14:51:45.302494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789063920794937:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.302573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:45.302619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789063920794937:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:45.302926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:45.303166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789064219652635:2101] [node 4] ICS07 socket disconnect 240 reason# EndOfStream
2025-11-28T14:51:45.303208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064219652635:2101] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.303315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:45.303279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789059625827601:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16746) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.303385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789064219652635:2101] [node 4] ICS15 start handshake
2025-11-28T14:51:45.303372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789063920794937:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.303432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789059625827601:2108] [node 4] ICP30 unregister session Session# [5:7577789063920794937:2108] VirtualId# [5:7577789063920794935:0]
2025-11-28T14:51:45.303482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:45.303509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789059625827601:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.303521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.304095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789059924685294:2101] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16746) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.304179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064219652635:2101] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.304208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789059625827540:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:45.304232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789059924685294:2101] [node 4] ICP30 unregister session Session# [6:7577789064219652635:2101] VirtualId# [6:7577789064219652633:0]
2025-11-28T14:51:45.304291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:45.304348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789059924685294:2101] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.304382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.305270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789059924685239:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:45.314492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789059924685294:2101] [node 4] ICP33 wake up from error state
2025-11-28T14:51:45.342775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:45.343049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789064219652627:2098] [node 5] ICS07 socket disconnect 205 reason# EndOfStream
2025-11-28T14:51:45.343090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064219652627:2098] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.343208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:45.343276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789064219652627:2098] [node 5] ICS15 start handshake
2025-11-28T14:51:45.344012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789059924685293:2098] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6623) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.344141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064219652627:2098] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.344202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789059924685293:2098] [node 5] ICP30 unregister session Session# [6:7577789064219652627:2098] VirtualId# [6:7577789064219652625:0]
2025-11-28T14:51:45.344285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:45.344353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789059924685293:2098] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6623) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.344395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6623) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:45.435799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789065639110750:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:45.435800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789065639110749:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:45.453971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789064451980083:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:45.454006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789064451980084:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:45.473621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067038954305:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:45.473632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067038954304:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:45.492586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789065788464135:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:45.492626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789065788464137:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:45.492805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789065788464136:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:45.512207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789066411110043:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:45.512253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789066411110045:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:45.512469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789066411110044:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:45.533501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789064585437917:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:45.533524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789064585437918:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:45.533568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789064585437916:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:45.628550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789064585437976:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:45.639372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067038954368:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:45.646548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789066411110107:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:45.646361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789065639110812:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:45.647608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789064451980147:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:45.646764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067038954370:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:45.647860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789064585437980:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:45.647802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789065639110813:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:45.648098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789064451980146:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:45.648747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789065788464198:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:45.648313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067038954369:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:45.648593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789066411110108:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:45.647886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789064451980148:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:45.648277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789065639110814:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:45.648788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789065788464199:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:45.655284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789065639110750:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(62928619ns) {Configure:104(62202557ns) {ProcessConfigured:135(86771ns) {StartInitialHandshake:157(84914ns) {DropHandshakes:470(1597ns) {DropIncomingHandshake:442(431ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(1012ns) PrepareNewSessionHandshake:477(815ns) {SwitchToState:179(318ns)}}}}}
2025-11-28T14:51:45.655325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789065639110749:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(63207424ns) {Configure:104(63202183ns) {ProcessConfigured:135(124348ns) {StartInitialHandshake:157(122423ns) {DropHandshakes:470(23957ns) {DropIncomingHandshake:442(366ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(460ns) PrepareNewSessionHandshake:477(876ns) {SwitchToState:179(421ns)}}}}}
2025-11-28T14:51:45.656684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789067038954304:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38290192ns) {Configure:104(38280301ns) {ProcessConfigured:135(195379ns) {StartInitialHandshake:157(193226ns) {DropHandshakes:470(1195ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:477(710ns) {SwitchToState:179(364ns)}}}}}
2025-11-28T14:51:45.658484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789065788464136:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28940974ns) {Configure:104(28935005ns) {ProcessConfigured:135(790098ns) {StartInitialHandshake:157(789232ns) {DropHandshakes:470(1354ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(173ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:477(1039ns) {SwitchToState:179(256ns)}}}}}
2025-11-28T14:51:45.659398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789064585437917:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18829739ns) {Configure:104(18824561ns) {ProcessConfigured:135(332945ns) {StartInitialHandshake:157(329068ns) {DropHandshakes:470(3237ns) {DropIncomingHandshake:442(742ns) DropOutgoingHandshake:457(528ns)} GenerateSessionVirtualId:988(1645ns) PrepareNewSessionHandshake:477(2263ns) {SwitchToState:179(894ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.660229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789064585437916:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(55300014ns) {Configure:104(55279611ns) {ProcessConfigured:135(455328ns) {StartInitialHandshake:157(452556ns) {DropHandshakes:470(1340ns) {DropIncomingHandshake:442(559ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:477(1261ns) {SwitchToState:179(517ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.662208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067038954378:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46961
2025-11-28T14:51:45.662493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789065788464208:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28486
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.663657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789065788464208:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.663854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789066411110044:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16985220ns) {Configure:104(16980495ns) {ProcessConfigured:135(317882ns) {StartInitialHandshake:157(315934ns) {DropHandshakes:470(1112ns) {DropIncomingHandshake:442(494ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(348ns) PrepareNewSessionHandshake:477(835ns) {SwitchToState:179(296ns)}}}}}
2025-11-28T14:51:45.664241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789065639110828:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56839
2025-11-28T14:51:45.665184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067038954378:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.663917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789065788464135:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.665252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789066411110043:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18915694ns) {Configure:104(18907742ns) {ProcessConfigured:135(670530ns) {StartInitialHandshake:157(668071ns) {DropHandshakes:470(1561ns) {DropIncomingHandshake:442(451ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(569ns) PrepareNewSessionHandshake:477(1211ns) {SwitchToState:179(533ns)}}}}}
2025-11-28T14:51:45.665648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110819:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.665670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789065639110819:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.667884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789065639110829:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56839
2025-11-28T14:51:45.668886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789064451980160:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36439
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.668505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789067038954304:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.668273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789065639110819:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:45.670531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585437993:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
2025-11-28T14:51:45.668818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067038954385:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46961
2025-11-28T14:51:45.669772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789064451980162:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36439
2025-11-28T14:51:45.668863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110821:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.671008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789064451980160:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.672382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585437994:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
2025-11-28T14:51:45.668897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789065639110821:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.671701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067038954386:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46961
2025-11-28T14:51:45.673037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789064451980162:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.673538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067038954385:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.675831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064585437993:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.668916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110828:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.675949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789064451980155:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.671258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789065639110830:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56839
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.674298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789067038954305:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.675986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789064451980155:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.681179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789064585437917:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.684865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789065788464215:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28486
2025-11-28T14:51:45.673851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789065639110821:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.682875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789064451980153:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.683824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789067038954304:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789067038954378:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.673979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789065639110812:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.682917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789064451980153:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.685828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067038954389:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46961
2025-11-28T14:51:45.686203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789064451980153:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:45.674286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789065639110750:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.687816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789066411110118:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11118
2025-11-28T14:51:45.686373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789064451980155:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.675442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.675796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.675834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789065639110750:2049] [node 3] ICP22 created new session: [1:7577789065639110834:2049]
2025-11-28T14:51:45.689017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789065788464208:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:45.687868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789066411110119:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11118
2025-11-28T14:51:45.689549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067038954378:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:45.689523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585437998:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
2025-11-28T14:51:45.690137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789064451980084:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.675856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789065639110812:2107] [node 4] ICP22 created new session: [1:7577789065639110835:2107]
2025-11-28T14:51:45.690379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789064451980147:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.687867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789066411110117:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11118
2025-11-28T14:51:45.690673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:45.676194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789065639110834:2049] [node 3] ICS09 handshake done sender: [1:7577789065639110821:2112] self: [1:7577789065639110815:0] peer: [3:7577789067038954387:0] socket: 206 qp: -1
2025-11-28T14:51:45.690559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789065788464135:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:45.690392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067038954390:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46961
::ffff:127.1.0.0
2025-11-28T14:51:45.690621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:45.687955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789066411110120:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11118
2025-11-28T14:51:45.691061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585437999:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
2025-11-28T14:51:45.676215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789065639110835:2107] [node 4] ICS09 handshake done sender: [1:7577789065639110819:2110] self: [1:7577789065639110817:0] peer: [4:7577789065788464214:0] socket: 205 qp: -1
2025-11-28T14:51:45.691126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789065788464135:2048] [node 1] ICP22 created new session: [4:7577789065788464216:2048]
2025-11-28T14:51:45.691086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789067038954304:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.690266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789066411110117:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.690737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.691119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:45.681202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789065639110835:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:45.691476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789064451980084:2049] [node 3] ICP22 created new session: [2:7577789064451980171:2049]
2025-11-28T14:51:45.690611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789066411110043:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.691573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789065788464216:2048] [node 1] ICS09 handshake done sender: [4:7577789065788464208:2114] self: [4:7577789065788464214:0] peer: [1:7577789065639110817:0] socket: 209 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.691457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067038954304:2048] [node 1] ICP22 created new session: [3:7577789067038954392:2048]
2025-11-28T14:51:45.681233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789065639110834:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:45.693965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064585437993:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:45.691615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789065788464216:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:45.682161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.691523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789064451980147:2109] [node 6] ICP22 created new session: [2:7577789064451980170:2109]
2025-11-28T14:51:45.693119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067038954385:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:45.691652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:45.694513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585438000:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
2025-11-28T14:51:45.693105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789064451980170:2109] [node 6] ICS09 handshake done sender: [2:7577789064451980153:2110] self: [2:7577789064451980149:0] peer: [6:7577789064585437997:0] socket: 212 qp: -1
2025-11-28T14:51:45.693761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067038954392:2048] [node 1] ICS09 handshake done sender: [3:7577789067038954378:2114] self: [3:7577789067038954387:0] peer: [1:7577789065639110815:0] socket: 208 qp: -1
2025-11-28T14:51:45.682230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:45.695546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789064585437917:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.693153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789064451980170:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:45.696074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:45.693957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067038954392:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:45.691494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110820:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.693168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789064451980171:2049] [node 3] ICS09 handshake done sender: [2:7577789064451980155:2112] self: [2:7577789064451980151:0] peer: [3:7577789067038954388:0] socket: 215 qp: -1
2025-11-28T14:51:45.694004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:45.696336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789064585437917:2049] [node 2] ICP22 created new session: [6:7577789064585438001:2049]
2025-11-28T14:51:45.691519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789065639110820:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.693214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789064451980171:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:45.696069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789067038954305:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.697312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064585438001:2049] [node 2] ICS09 handshake done sender: [6:7577789064585437993:2115] self: [6:7577789064585437997:0] peer: [2:7577789064451980149:0] socket: 226 qp: -1
2025-11-28T14:51:45.693350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:45.693461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789065639110820:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:45.696104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:45.693488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:45.698672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064585438001:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:45.697077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067038954305:2049] [node 2] ICP22 created new session: [3:7577789067038954394:2049]
2025-11-28T14:51:45.698715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:45.699559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789066411110127:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11118
2025-11-28T14:51:45.699532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789064451980168:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36439
2025-11-28T14:51:45.698201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067038954394:2049] [node 2] ICS09 handshake done sender: [3:7577789067038954385:2117] self: [3:7577789067038954388:0] peer: [2:7577789064451980151:0] socket: 217 qp: -1
2025-11-28T14:51:45.693472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789065639110840:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56839
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.698458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067038954394:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:45.694072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789065639110813:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.698495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:45.694271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.696001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789065639110813:2108] [node 5] ICP22 created new session: [1:7577789065639110842:2108]
2025-11-28T14:51:45.697151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789065639110842:2108] [node 5] ICS09 handshake done sender: [1:7577789065639110820:2111] self: [1:7577789065639110816:0] peer: [5:7577789066411110126:0] socket: 204 qp: -1
2025-11-28T14:51:45.697316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789065639110842:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:45.701258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789064451980169:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36439
2025-11-28T14:51:45.697362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.702191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789064451980168:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.701811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789067038954305:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.702721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789066411110117:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.703115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789064451980083:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.703082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789065639110844:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56839
2025-11-28T14:51:45.703264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789066411110043:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.703442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067038954396:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46961
2025-11-28T14:51:45.703422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:45.703469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789066411110043:2048] [node 1] ICP22 created new session: [5:7577789066411110128:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.703525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789066411110128:2048] [node 1] ICS09 handshake done sender: [5:7577789066411110117:2115] self: [5:7577789066411110126:0] peer: [1:7577789065639110816:0] socket: 229 qp: -1
2025-11-28T14:51:45.703554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789066411110128:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:45.703907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789065788464218:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28486
2025-11-28T14:51:45.703603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.704769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067038954396:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.704764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110822:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.705533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789065788464219:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28486
2025-11-28T14:51:45.705542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585438004:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
2025-11-28T14:51:45.705794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789065788464218:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.705908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789065788464137:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.704809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789065639110822:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.706028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064585438004:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.706205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789064585437918:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.706173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067038954379:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.706210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789067038954379:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.706231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789065788464219:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.706394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789065788464136:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.706345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110844:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.706431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789065639110822:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:45.706465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789064451980177:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36439
2025-11-28T14:51:45.706565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789064451980163:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.706543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067038954382:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.706583Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789064451980163:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.706619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789065639110749:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.706586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789067038954382:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:45.707874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:45.707868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789064451980168:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:45.707968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067038954379:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:45.707983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789065639110749:2048] [node 2] ICP22 created new session: [1:7577789065639110847:2048]
2025-11-28T14:51:45.708792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789064451980083:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.708031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789065639110847:2048] [node 2] ICS09 handshake done sender: [1:7577789065639110822:2113] self: [1:7577789065639110818:0] peer: [2:7577789064451980176:0] socket: 211 qp: -1
2025-11-28T14:51:45.708834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789067038954368:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.708878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789065639110847:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:45.708836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:45.708933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789064451980083:2048] [node 1] ICP22 created new session: [2:7577789064451980179:2048]
2025-11-28T14:51:45.708923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:45.708889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067038954382:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:45.708942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:45.709200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067038954368:2107] [node 4] ICP22 created new session: [3:7577789067038954401:2107]
2025-11-28T14:51:45.709265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789064451980163:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:45.709304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789064451980179:2048] [node 1] ICS09 handshake done sender: [2:7577789064451980168:2117] self: [2:7577789064451980176:0] peer: [1:7577789065639110818:0] socket: 243 qp: -1
2025-11-28T14:51:45.709250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067038954401:2107] [node 4] ICS09 handshake done sender: [3:7577789067038954379:2115] self: [3:7577789067038954377:0] peer: [4:7577789065788464220:0] socket: 256 qp: -1
2025-11-28T14:51:45.709653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789064451980179:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:51:45.709320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789067038954369:2109] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:51:45.709695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:45.710817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789064585437918:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29111) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789064585438004:2121] held: no
2025-11-28T14:51:45.709384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.709430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067038954369:2109] [node 6] ICP22 created new session: [3:7577789067038954402:2109]
2025-11-28T14:51:45.711343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789064451980146:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.709470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067038954402:2109] [node 6] ICS09 handshake done sender: [3:7577789067038954382:2116] self: [3:7577789067038954380:0] peer: [6:7577789064585438005:0] socket: 261 qp: -1
2025-11-28T14:51:45.711386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:45.709500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067038954402:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:45.709530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:45.711377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585438006:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
2025-11-28T14:51:45.709594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067038954401:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:45.711415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789064451980083:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:45.711442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789064451980146:2107] [node 4] ICP22 created new session: [2:7577789064451980181:2107]
2025-11-28T14:51:45.711413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789065788464222:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28486
2025-11-28T14:51:45.709632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.711483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789064451980181:2107] [node 4] ICS09 handshake done sender: [2:7577789064451980163:2116] self: [2:7577789064451980161:0] peer: [4:7577789065788464221:0] socket: 259 qp: -1
2025-11-28T14:51:45.711516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789064451980181:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:45.711545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.711641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064585438004:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.712511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789064585437918:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.712556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.712582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789064585437918:2050] [node 3] ICP22 created new session: [6:7577789064585438008:2050]
2025-11-28T14:51:45.712624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064585438008:2050] [node 3] ICS09 handshake done sender: [6:7577789064585438004:2121] self: [6:7577789064585438005:0] peer: [3:7577789067038954380:0] socket: 263 qp: -1
2025-11-28T14:51:45.712663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064585438008:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:45.712695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.712878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789065788464223:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28486
2025-11-28T14:51:45.715083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789065788464218:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.716289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789065788464137:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.716348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.716414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789065788464137:2050] [node 3] ICP22 created new session: [4:7577789065788464225:2050]
2025-11-28T14:51:45.716463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789065788464225:2050] [node 3] ICS09 handshake done sender: [4:7577789065788464218:2117] self: [4:7577789065788464220:0] peer: [3:7577789067038954377:0] socket: 216 qp: -1
2025-11-28T14:51:45.716498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789065788464225:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:45.716530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:45.716781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789065788464207:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.723716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789065788464219:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.723949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789065788464224:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28486
2025-11-28T14:51:45.723981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064585437994:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.724224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789064585437980:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.724249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789065788464136:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.724357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:45.724428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789065788464136:2049] [node 2] ICP22 created new session: [4:7577789065788464227:2049]
2025-11-28T14:51:45.724535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789065788464227:2049] [node 2] ICS09 handshake done sender: [4:7577789065788464219:2118] self: [4:7577789065788464221:0] peer: [2:7577789064451980161:0] socket: 262 qp: -1
2025-11-28T14:51:45.724580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789065788464227:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:45.724624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:45.724716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789065788464224:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.724827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789065788464207:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.725460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789065788464207:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.725625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585438011:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.732613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789065788464198:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.732941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:45.732903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789066411110130:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11118
2025-11-28T14:51:45.733015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789065788464198:2109] [node 6] ICP22 created new session: [4:7577789065788464232:2109]
2025-11-28T14:51:45.732976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064585437994:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:45.733070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789065788464232:2109] [node 6] ICS09 handshake done sender: [4:7577789065788464207:2113] self: [4:7577789065788464200:0] peer: [6:7577789064585438010:0] socket: 218 qp: -1
2025-11-28T14:51:45.733137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789065788464232:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:45.733184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:45.733232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789064585437980:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.733479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:45.733572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789064585437980:2108] [node 4] ICP22 created new session: [6:7577789064585438012:2108]
2025-11-28T14:51:45.733632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064585438012:2108] [node 4] ICS09 handshake done sender: [6:7577789064585437994:2116] self: [6:7577789064585438010:0] peer: [4:7577789065788464200:0] socket: 232 qp: -1
2025-11-28T14:51:45.733680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064585438012:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:45.733715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.734960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789065788464206:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.743094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067038954386:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.743424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789067038954368:2107] [node 4] ICP18 handshake (actor [3:7577789067038954386:2118]) is too old
2025-11-28T14:51:45.747545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789065788464205:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.754790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110829:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.755046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789065639110812:2107] [node 4] ICP18 handshake (actor [1:7577789065639110829:2116]) is too old
2025-11-28T14:51:45.756839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789064585437987:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.764498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110830:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.779583Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789067038954375:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.783495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789066411110118:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.783660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789066411110045:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.783972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067038954375:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.786725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067038954375:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.787060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789067038954370:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.787007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789066411110132:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11118
2025-11-28T14:51:45.787135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.787221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067038954370:2108] [node 5] ICP22 created new session: [3:7577789067038954406:2108]
2025-11-28T14:51:45.787286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067038954406:2108] [node 5] ICS09 handshake done sender: [3:7577789067038954375:2112] self: [3:7577789067038954373:0] peer: [5:7577789066411110131:0] socket: 213 qp: -1
2025-11-28T14:51:45.787342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067038954406:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:45.787422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.787975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789066411110118:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:45.788254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789066411110045:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.788303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:45.788346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789066411110045:2050] [node 3] ICP22 created new session: [5:7577789066411110133:2050]
2025-11-28T14:51:45.788418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789066411110133:2050] [node 3] ICS09 handshake done sender: [5:7577789066411110118:2116] self: [5:7577789066411110131:0] peer: [3:7577789067038954373:0] socket: 233 qp: -1
2025-11-28T14:51:45.788455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789066411110133:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:45.788487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.790099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789065788464234:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28486
2025-11-28T14:51:45.799925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789064451980154:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.820237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789064585437988:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.832218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789065639110827:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.840068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064585437998:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.840356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789064585437916:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.840689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110827:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.841298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789065639110827:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.841506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789065639110814:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.841587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:45.841581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585438015:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
2025-11-28T14:51:45.841657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789065639110814:2109] [node 6] ICP22 created new session: [1:7577789065639110851:2109]
2025-11-28T14:51:45.841771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789065639110851:2109] [node 6] ICS09 handshake done sender: [1:7577789065639110827:2114] self: [1:7577789065639110826:0] peer: [6:7577789064585438014:0] socket: 244 qp: -1
2025-11-28T14:51:45.841808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789065639110851:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:45.841856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:45.841917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064585437998:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:45.842160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789064585437916:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789064585437998:2118] held: no
2025-11-28T14:51:45.842439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789064585437916:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.842578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:45.843376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789064585437916:2048] [node 1] ICP22 created new session: [6:7577789064585438016:2048]
2025-11-28T14:51:45.843437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064585438016:2048] [node 1] ICS09 handshake done sender: [6:7577789064585437998:2118] self: [6:7577789064585438014:0] peer: [1:7577789065639110826:0] socket: 247 qp: -1
2025-11-28T14:51:45.843474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064585438016:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:45.843526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:45.843778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789066411110113:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.851460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789066411110119:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.851543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789066411110120:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.851491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789065639110840:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.851675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789065639110813:2108] [node 5] ICP18 handshake (actor [1:7577789065639110840:2118]) is too old
2025-11-28T14:51:45.852443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789066411110044:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.853881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789064451980154:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.854485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789064451980154:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.854748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789066411110138:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11118
2025-11-28T14:51:45.854755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789064451980148:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.854824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.854883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789064451980148:2108] [node 5] ICP22 created new session: [2:7577789064451980184:2108]
2025-11-28T14:51:45.854931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789064451980184:2108] [node 5] ICS09 handshake done sender: [2:7577789064451980154:2111] self: [2:7577789064451980150:0] peer: [5:7577789066411110136:0] socket: 214 qp: -1
2025-11-28T14:51:45.854973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789064451980184:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:45.855059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.855307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789066411110119:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:45.855550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789066411110044:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.855644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:45.855694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789066411110044:2049] [node 2] ICP22 created new session: [5:7577789066411110139:2049]
2025-11-28T14:51:45.855745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789066411110139:2049] [node 2] ICS09 handshake done sender: [5:7577789066411110119:2117] self: [5:7577789066411110136:0] peer: [2:7577789064451980150:0] socket: 236 qp: -1
2025-11-28T14:51:45.857280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789066411110139:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:45.857353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:45.857609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789066411110116:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.866038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789064585438000:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.866274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789064585437976:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.866602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789066411110116:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.867110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789066411110116:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.867355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789066411110107:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.867322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789064585438019:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52029
2025-11-28T14:51:45.867506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:45.867583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789066411110107:2109] [node 6] ICP22 created new session: [5:7577789066411110142:2109]
2025-11-28T14:51:45.867655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789064585438000:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:45.867670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789066411110142:2109] [node 6] ICS09 handshake done sender: [5:7577789066411110116:2114] self: [5:7577789066411110110:0] peer: [6:7577789064585438018:0] socket: 251 qp: -1
2025-11-28T14:51:45.867720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789066411110142:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:45.867797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789064585437976:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.867800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:45.867977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.868045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789064585437976:2106] [node 5] ICP22 created new session: [6:7577789064585438020:2106]
2025-11-28T14:51:45.868237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789064585438020:2106] [node 5] ICS09 handshake done sender: [6:7577789064585438000:2120] self: [6:7577789064585438018:0] peer: [5:7577789066411110110:0] socket: 253 qp: -1
2025-11-28T14:51:45.868291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789064585438020:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:45.868431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.870062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789066411110115:2113] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.878264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789064451980169:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.878517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789064451980148:2108] [node 5] ICP18 handshake (actor [2:7577789064451980169:2118]) is too old
2025-11-28T14:51:45.882781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789065788464213:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.891041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789066411110130:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.891290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789066411110108:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:45.891597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789065788464213:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.891992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789065788464213:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:45.892164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789065788464199:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:45.892330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:45.892386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789065788464199:2108] [node 5] ICP22 created new session: [4:7577789065788464236:2108]
2025-11-28T14:51:45.892430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789066411110145:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11118
2025-11-28T14:51:45.892462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789065788464236:2108] [node 5] ICS09 handshake done sender: [4:7577789065788464213:2115] self: [4:7577789065788464212:0] peer: [5:7577789066411110144:0] socket: 272 qp: -1
2025-11-28T14:51:45.892505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789065788464236:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:45.892558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:45.892699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789066411110130:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:45.892836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789066411110108:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:45.893089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:45.893147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789066411110108:2108] [node 4] ICP22 created new session: [5:7577789066411110146:2108]
2025-11-28T14:51:45.893211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789066411110146:2108] [node 4] ICS09 handshake done sender: [5:7577789066411110130:2121] self: [5:7577789066411110144:0] peer: [4:7577789065788464212:0] socket: 273 qp: -1
2025-11-28T14:51:45.893252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789066411110146:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:45.893294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:45.895892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789066411110122:2119] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:45.899445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789065788464234:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:45.900601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789065788464199:2108] [node 5] ICP18 handshake (actor [4:7577789065788464234:2122]) is too old
2025-11-28T14:51:45.936202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:45.936443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789064451980179:2048] [node 1] ICS07 socket disconnect 243 reason# EndOfStream
2025-11-28T14:51:45.936474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789064451980179:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.936562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.936613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789064451980179:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:45.937390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:45.937591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789067038954392:2048] [node 1] ICS01 socket: 208 reason# EndOfStream
2025-11-28T14:51:45.937639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789067038954304:2048] [node 1] ICP30 unregister session Session# [3:7577789067038954392:2048] VirtualId# [3:7577789067038954387:0]
2025-11-28T14:51:45.937676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789067038954392:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.937748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.937844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.938643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789064451980083:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Socket error# Connection reset by peer state# SendExRequestData processed# 0 remain# 34 incoming: [0:0:0] held: no
2025-11-28T14:51:45.938701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789064451980179:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:45.939728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789064451980083:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.940417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789064451980179:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.940468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789064451980083:2048] [node 1] ICP30 unregister session Session# [2:7577789064451980179:2048] VirtualId# [2:7577789064451980176:0]
2025-11-28T14:51:45.940534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.940564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789064451980083:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.940576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.940795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:45.941050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789065788464216:2048] [node 1] ICS07 socket disconnect 209 reason# EndOfStream
2025-11-28T14:51:45.941082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789065788464216:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.941156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.941216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789065788464216:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:45.941636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:45.941857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789066411110128:2048] [node 1] ICS07 socket disconnect 229 reason# EndOfStream
2025-11-28T14:51:45.941905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789066411110128:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.941984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.942043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789066411110128:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:45.942206Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789065788464135:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.942287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789065788464216:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.942332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789065788464135:2048] [node 1] ICP30 unregister session Session# [4:7577789065788464216:2048] VirtualId# [4:7577789065788464214:0]
2025-11-28T14:51:45.942376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.942415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789065788464135:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.942443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.942573Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:45.942792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064585438016:2048] [node 1] ICS01 socket: 247 reason# EndOfStream
2025-11-28T14:51:45.942869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789066411110043:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.942962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789066411110128:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.943014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789066411110043:2048] [node 1] ICP30 unregister session Session# [5:7577789066411110128:2048] VirtualId# [5:7577789066411110126:0]
2025-11-28T14:51:45.943061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.943151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789066411110043:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.943171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2014) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.942838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789064585437916:2048] [node 1] ICP30 unregister session Session# [6:7577789064585438016:2048] VirtualId# [6:7577789064585438014:0]
2025-11-28T14:51:45.943278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064585438016:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.943351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:45.943414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:45.952457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789065788464135:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:45.953450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789066411110043:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:45.966572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:45.966818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789067038954394:2049] [node 2] ICS07 socket disconnect 217 reason# EndOfStream
2025-11-28T14:51:45.966855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789067038954394:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.966973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:45.967031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789067038954394:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:45.967969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789067038954305:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22414) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.968153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789067038954394:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.968199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789067038954305:2049] [node 2] ICP30 unregister session Session# [3:7577789067038954394:2049] VirtualId# [3:7577789067038954388:0]
2025-11-28T14:51:45.968293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:45.968357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789067038954305:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.968399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.969718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:45.969865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789065788464227:2049] [node 2] ICS01 socket: 262 reason# EndOfStream
2025-11-28T14:51:45.969911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789065788464136:2049] [node 2] ICP30 unregister session Session# [4:7577789065788464227:2049] VirtualId# [4:7577789065788464221:0]
2025-11-28T14:51:45.969938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789065788464227:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.970005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:45.970042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:45.970805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:45.970945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789066411110139:2049] [node 2] ICS07 socket disconnect 236 reason# EndOfStream
2025-11-28T14:51:45.970973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789066411110139:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.971028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:45.971073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789066411110139:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:45.971699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:45.971764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789066411110044:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22414) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.971889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789066411110139:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.971927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789066411110044:2049] [node 2] ICP30 unregister session Session# [5:7577789066411110139:2049] VirtualId# [5:7577789066411110136:0]
2025-11-28T14:51:45.971983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:45.971948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789064585438001:2049] [node 2] ICS07 socket disconnect 226 reason# EndOfStream
2025-11-28T14:51:45.972057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789066411110044:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.971982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064585438001:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:45.972083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.972072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:45.972138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789064585438001:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:45.972946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789064585437917:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22414) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:45.972988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064585438001:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:45.973067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789064585437917:2049] [node 2] ICP30 unregister session Session# [6:7577789064585438001:2049] VirtualId# [6:7577789064585437997:0]
2025-11-28T14:51:45.973109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:45.973161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789064585437917:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.973193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:45.978056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789067038954305:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:45.982103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789066411110044:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:45.983112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789064585437917:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:46.019856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:46.020040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789065788464225:2050] [node 3] ICS07 socket disconnect 216 reason# EndOfStream
2025-11-28T14:51:46.020066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789065788464225:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.020136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:46.020171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789065788464225:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:46.020888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789065788464137:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29111) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.020904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:46.021018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789065788464225:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.021058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789065788464137:2050] [node 3] ICP30 unregister session Session# [4:7577789065788464225:2050] VirtualId# [4:7577789065788464220:0]
2025-11-28T14:51:46.021107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:46.021088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789066411110133:2050] [node 3] ICS07 socket disconnect 233 reason# EndOfStream
2025-11-28T14:51:46.021119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789066411110133:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.021153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789065788464137:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29111) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.021200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:46.021168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29111) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.021236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789066411110133:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:46.021813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:46.021984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064585438008:2050] [node 3] ICS01 socket: 263 reason# EndOfStream
2025-11-28T14:51:46.022019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789064585437918:2050] [node 3] ICP30 unregister session Session# [6:7577789064585438008:2050] VirtualId# [6:7577789064585438005:0]
2025-11-28T14:51:46.022041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064585438008:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.022093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:46.022062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789066411110045:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29111) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.022129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789066411110133:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.022192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:46.022300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789066411110045:2050] [node 3] ICP30 unregister session Session# [5:7577789066411110133:2050] VirtualId# [5:7577789066411110131:0]
2025-11-28T14:51:46.022366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:46.022407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789066411110045:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29111) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.022426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29111) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.032288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789066411110045:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:46.033339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:46.033488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789066411110146:2108] [node 4] ICS07 socket disconnect 273 reason# EndOfStream
2025-11-28T14:51:46.033518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789066411110146:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.033588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:46.033624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789066411110146:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:46.034358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789066411110108:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18031) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.034432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789066411110146:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.034478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789066411110108:2108] [node 4] ICP30 unregister session Session# [5:7577789066411110146:2108] VirtualId# [5:7577789066411110144:0]
2025-11-28T14:51:46.034529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:46.034477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:46.034565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789066411110108:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18031) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.034586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18031) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.034673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789064585438012:2108] [node 4] ICS07 socket disconnect 232 reason# EndOfStream
2025-11-28T14:51:46.034704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064585438012:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.034775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:46.034826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789064585438012:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:46.035555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789064585437980:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18031) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.035671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064585438012:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.035721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789064585437980:2108] [node 4] ICP30 unregister session Session# [6:7577789064585438012:2108] VirtualId# [6:7577789064585438010:0]
2025-11-28T14:51:46.035757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:46.035783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789064585437980:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18031) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.035795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18031) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.045606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789064585437980:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:46.047811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:46.047957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789064585438020:2106] [node 5] ICS07 socket disconnect 253 reason# EndOfStream
2025-11-28T14:51:46.047984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789064585438020:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.048079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:46.048164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789064585438020:2106] [node 5] ICS15 start handshake
2025-11-28T14:51:46.048987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789064585437976:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28203) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.049067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789064585438020:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.049134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789064585437976:2106] [node 5] ICP30 unregister session Session# [6:7577789064585438020:2106] VirtualId# [6:7577789064585438018:0]
2025-11-28T14:51:46.049185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:46.049217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789064585437976:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.049230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:46.099951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789070362307200:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:46.100059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789070362307201:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:46.120806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071278351163:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.121054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071278351164:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:46.140553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067998581220:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.140600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067998581221:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:46.161294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789069503386285:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.161434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789069503386286:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:46.161514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789069503386287:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:46.182142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789068811868483:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.182197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789068811868484:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:46.182550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789068811868485:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:46.200921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789071362589865:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.200959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789071362589867:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:46.201183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789071362589866:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:46.279960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789071362589924:2100] [node 4] ICP01 ready to work
2025-11-28T14:51:46.282039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789071362589929:2105] [node 5] ICP01 ready to work
2025-11-28T14:51:46.282486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789070362307261:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:46.317700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789070362307285:2116] [node 6] ICP01 ready to work
2025-11-28T14:51:46.331749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071278351253:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:46.335065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789069503386348:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:46.334720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789070362307286:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:46.335907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789070362307201:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(57516286ns) {Configure:104(57512167ns) {ProcessConfigured:135(382670ns) {StartInitialHandshake:157(379649ns) {DropHandshakes:470(2595ns) {DropIncomingHandshake:442(852ns) DropOutgoingHandshake:457(679ns)} GenerateSessionVirtualId:988(663ns) PrepareNewSessionHandshake:477(1264ns) {SwitchToState:179(294ns)}}}}}
2025-11-28T14:51:46.335638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071278351254:2111] [node 5] ICP01 ready to work
2025-11-28T14:51:46.335787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789071362589924:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36614145ns) {Configure:104(36609489ns) {ProcessConfigured:135(27578587ns) {StartInitialHandshake:157(27574781ns) {DropHandshakes:470(2745ns) {DropIncomingHandshake:442(653ns) DropOutgoingHandshake:457(492ns)} GenerateSessionVirtualId:988(689ns) PrepareNewSessionHandshake:477(4633ns) {SwitchToState:179(2137ns)}}}}}
2025-11-28T14:51:46.335971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071278351255:2136] [node 6] ICP01 ready to work
2025-11-28T14:51:46.336333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789068811868546:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:46.335311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067998581293:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:46.335537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067998581294:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:46.335556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789067998581295:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:46.336340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789070362307261:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53003391ns) {Configure:104(52995999ns) {ProcessConfigured:135(32813683ns) {StartInitialHandshake:157(32809823ns) {DropHandshakes:470(2784ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(390ns)} GenerateSessionVirtualId:988(969ns) PrepareNewSessionHandshake:477(1967ns) {SwitchToState:179(771ns)}}}}}
2025-11-28T14:51:46.336750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789069503386349:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:46.336834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789068811868547:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:46.351331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789068811868483:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40778527ns) {Configure:104(33275608ns) {ProcessConfigured:135(74778ns) {StartInitialHandshake:157(72677ns) {DropHandshakes:470(1581ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(300ns)} GenerateSessionVirtualId:988(827ns) PrepareNewSessionHandshake:477(1008ns) {SwitchToState:179(528ns)}}}}}
2025-11-28T14:51:46.351353Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789069503386286:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56184669ns) {Configure:104(56180921ns) {ProcessConfigured:135(649343ns) {StartInitialHandshake:157(646968ns) {DropHandshakes:470(1158ns) {DropIncomingHandshake:442(293ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:477(1128ns) {SwitchToState:179(418ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.351994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789069503386285:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47457012ns) {Configure:104(47449030ns) {ProcessConfigured:135(98526ns) {StartInitialHandshake:157(95985ns) {DropHandshakes:470(2067ns) {DropIncomingHandshake:442(528ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(503ns) PrepareNewSessionHandshake:477(1216ns) {SwitchToState:179(496ns)}}}}}
2025-11-28T14:51:46.352434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789070362307297:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8058
2025-11-28T14:51:46.352354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789068811868484:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(50705452ns) {Configure:104(50701172ns) {ProcessConfigured:135(1062160ns) {StartInitialHandshake:157(1059211ns) {DropHandshakes:470(1321ns) {DropIncomingHandshake:442(330ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(394ns) PrepareNewSessionHandshake:477(4886ns) {SwitchToState:179(3834ns)}}}}}
2025-11-28T14:51:46.352350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789069503386287:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33635586ns) {Configure:104(26308812ns) {ProcessConfigured:135(1097385ns) {StartInitialHandshake:157(1094809ns) {DropHandshakes:470(3532ns) {DropIncomingHandshake:442(1114ns) DropOutgoingHandshake:457(583ns)} GenerateSessionVirtualId:988(769ns) PrepareNewSessionHandshake:477(1849ns) {SwitchToState:179(506ns)}}}}}
2025-11-28T14:51:46.352529Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789067998581221:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56680693ns) {Configure:104(56671727ns) {ProcessConfigured:135(470322ns) {StartInitialHandshake:157(467589ns) {DropHandshakes:470(1241ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(277ns)} GenerateSessionVirtualId:988(507ns) PrepareNewSessionHandshake:477(2682ns) {SwitchToState:179(1300ns)}}}}}
2025-11-28T14:51:46.353451Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789071278351164:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(56801757ns) {Configure:104(56797900ns) {ProcessConfigured:135(168197ns) {StartInitialHandshake:157(166868ns) {DropHandshakes:470(761ns) {DropIncomingHandshake:442(244ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(665ns) {SwitchToState:179(302ns)}}}}}
2025-11-28T14:51:46.352515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789068811868485:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34461534ns) {Configure:104(29562070ns) {ProcessConfigured:135(539232ns) {StartInitialHandshake:157(537278ns) {DropHandshakes:470(1199ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(513ns) PrepareNewSessionHandshake:477(1206ns) {SwitchToState:179(421ns)}}}}}
2025-11-28T14:51:46.353345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789067998581295:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16632540ns) {Configure:104(16628420ns) {ProcessConfigured:135(1011633ns) {StartInitialHandshake:157(1009739ns) {DropHandshakes:470(985ns) {DropIncomingHandshake:442(502ns) DropOutgoingHandshake:457(161ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:477(1069ns) {SwitchToState:179(445ns)}}}}}
2025-11-28T14:51:46.354090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789070362307297:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.354020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789067998581294:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16616573ns) {Configure:104(16613790ns) {ProcessConfigured:135(1192522ns) {StartInitialHandshake:157(1187544ns) {DropHandshakes:470(1138ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(586ns) PrepareNewSessionHandshake:477(1391ns) {SwitchToState:179(817ns)}}}}}
2025-11-28T14:51:46.353531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789071278351254:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16083165ns) {Configure:104(16075725ns) {ProcessConfigured:135(1067951ns) {StartInitialHandshake:157(1065804ns) {DropHandshakes:470(3892ns) {DropIncomingHandshake:442(394ns) DropOutgoingHandshake:457(2794ns)} GenerateSessionVirtualId:988(766ns) PrepareNewSessionHandshake:477(1881ns) {SwitchToState:179(883ns)}}}}}
2025-11-28T14:51:46.354136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789071278351253:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16102681ns) {Configure:104(16098433ns) {ProcessConfigured:135(521055ns) {StartInitialHandshake:157(519186ns) {DropHandshakes:470(4003ns) {DropIncomingHandshake:442(3086ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(619ns) PrepareNewSessionHandshake:477(1359ns) {SwitchToState:179(432ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.354208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789071278351163:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(63801373ns) {Configure:104(63798103ns) {ProcessConfigured:135(738013ns) {StartInitialHandshake:157(736305ns) {DropHandshakes:470(1195ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(294ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:477(1329ns) {SwitchToState:179(566ns)}}}}}
2025-11-28T14:51:46.354374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789068811868557:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42003
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.354568Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789067998581220:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17915238ns) {Configure:104(17907917ns) {ProcessConfigured:135(1720006ns) {StartInitialHandshake:157(1540169ns) {DropHandshakes:470(1362ns) {DropIncomingHandshake:442(350ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(672ns) PrepareNewSessionHandshake:477(4980ns) {SwitchToState:179(3885ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.354639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071278351262:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35139
2025-11-28T14:51:46.354605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067998581303:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56088
2025-11-28T14:51:46.354783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789069503386361:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56179
2025-11-28T14:51:46.354953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789069503386360:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56179
2025-11-28T14:51:46.355449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071278351262:2139] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.354981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789068811868557:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.355790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067998581303:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.356545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789069503386360:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.356551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789067998581220:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.355746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789071278351163:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.357541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789069503386361:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.357750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789070362307283:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.357798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789070362307283:2128] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.357788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789069503386285:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.358759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789070362307289:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.362769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589944:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.370442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071278351270:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.369405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589943:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
2025-11-28T14:51:46.358791Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789070362307289:2130] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.377810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789068811868563:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42003
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:51:46.379547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789069503386365:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56179
2025-11-28T14:51:46.370220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589945:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
2025-11-28T14:51:46.379567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071278351272:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35139
2025-11-28T14:51:46.380264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789069503386368:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56179
2025-11-28T14:51:46.379753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071278351273:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35139
2025-11-28T14:51:46.379469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789071362589943:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.379463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067998581310:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56088
2025-11-28T14:51:46.359033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789070362307288:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.380312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789071362589929:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.380515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071278351274:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35139
2025-11-28T14:51:46.359065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789070362307288:2129] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:46.380594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067998581312:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56088
2025-11-28T14:51:46.380865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789068811868554:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.381504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789069503386360:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:46.380993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071278351272:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.359170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789070362307283:2128] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:46.380883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067998581313:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56088
2025-11-28T14:51:46.381308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071278351262:2139] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:46.380916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789068811868554:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:46.360419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789070362307300:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8058
2025-11-28T14:51:46.382668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789068811868554:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:46.382422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067998581303:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:46.382048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789069503386285:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.382495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789071278351163:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.382738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:46.362412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789070362307299:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8058
2025-11-28T14:51:46.383339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789067998581220:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.383267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.378634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789070362307288:2129] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:46.384404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071278351163:2048] [node 1] ICP22 created new session: [2:7577789071278351276:2048]
2025-11-28T14:51:46.383247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789069503386285:2048] [node 1] ICP22 created new session: [4:7577789069503386369:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.383384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.385113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071278351276:2048] [node 1] ICS09 handshake done sender: [2:7577789071278351262:2139] self: [2:7577789071278351271:0] peer: [1:7577789070362307262:0] socket: 222 qp: -1
2025-11-28T14:51:46.385099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789069503386369:2048] [node 1] ICS09 handshake done sender: [4:7577789069503386360:2115] self: [4:7577789069503386366:0] peer: [1:7577789070362307284:0] socket: 220 qp: -1
2025-11-28T14:51:46.385097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067998581220:2048] [node 1] ICP22 created new session: [3:7577789067998581314:2048]
2025-11-28T14:51:46.378795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789070362307289:2130] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:46.385965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589947:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
2025-11-28T14:51:46.385206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071278351276:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:46.385190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067998581314:2048] [node 1] ICS09 handshake done sender: [3:7577789067998581303:2122] self: [3:7577789067998581311:0] peer: [1:7577789070362307287:0] socket: 221 qp: -1
2025-11-28T14:51:46.385137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789069503386369:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:46.385212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067998581314:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:46.385244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:46.385243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:46.385208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789068811868565:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42003
2025-11-28T14:51:46.385352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:46.379133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789070362307200:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.386063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589948:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
2025-11-28T14:51:46.379136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789070362307299:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.385335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789068811868546:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.385771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:46.379200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:46.387168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789071362589947:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.385970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789068811868546:2109] [node 6] ICP22 created new session: [5:7577789068811868566:2109]
2025-11-28T14:51:46.387171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789071362589943:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:46.379510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789070362307200:2048] [node 2] ICP22 created new session: [1:7577789070362307304:2048]
2025-11-28T14:51:46.389300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789071362589929:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789071362589943:2115] held: no
2025-11-28T14:51:46.390524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789071362589865:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.380180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789070362307261:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.386085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789068811868565:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.380394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789070362307201:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.390745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789071362589929:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.386225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789068811868566:2109] [node 6] ICS09 handshake done sender: [5:7577789068811868554:2111] self: [5:7577789068811868551:0] peer: [6:7577789071362589946:0] socket: 228 qp: -1
2025-11-28T14:51:46.380592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:46.390794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:46.386539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789068811868566:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:46.391005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789071362589929:2105] [node 5] ICP22 created new session: [6:7577789071362589951:2105]
2025-11-28T14:51:46.380767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789070362307201:2049] [node 3] ICP22 created new session: [1:7577789070362307306:2049]
2025-11-28T14:51:46.390979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789069503386372:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56179
2025-11-28T14:51:46.391048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789071362589951:2105] [node 5] ICS09 handshake done sender: [6:7577789071362589943:2115] self: [6:7577789071362589946:0] peer: [5:7577789068811868551:0] socket: 233 qp: -1
2025-11-28T14:51:46.386570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789068811868483:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.391087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789071362589951:2105] [node 5] ICS10 traffic start
2025-11-28T14:51:46.380816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789070362307304:2048] [node 2] ICS09 handshake done sender: [1:7577789070362307283:2128] self: [1:7577789070362307262:0] peer: [2:7577789071278351271:0] socket: 210 qp: -1
2025-11-28T14:51:46.380826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:46.391563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589949:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
2025-11-28T14:51:46.386744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:46.380872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789070362307261:2107] [node 4] ICP22 created new session: [1:7577789070362307307:2107]
2025-11-28T14:51:46.391690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.392305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789068811868483:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789068811868565:2117] held: no
2025-11-28T14:51:46.380919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789070362307306:2049] [node 3] ICS09 handshake done sender: [1:7577789070362307289:2130] self: [1:7577789070362307287:0] peer: [3:7577789067998581311:0] socket: 214 qp: -1
2025-11-28T14:51:46.392402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789071362589949:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.393689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789071362589924:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.380957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789070362307304:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:46.393442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067998581316:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56088
2025-11-28T14:51:46.393651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789068811868570:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42003
2025-11-28T14:51:46.380959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789070362307306:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:46.380996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.393976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789069503386358:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.381234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789070362307300:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.381275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:46.395131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071278351279:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35139
2025-11-28T14:51:46.381352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789070362307307:2107] [node 4] ICS09 handshake done sender: [1:7577789070362307288:2129] self: [1:7577789070362307284:0] peer: [4:7577789069503386366:0] socket: 211 qp: -1
2025-11-28T14:51:46.395589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067998581316:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.395114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589954:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
2025-11-28T14:51:46.394995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789068811868565:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:46.394003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789069503386358:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:46.381526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789070362307307:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:46.395721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789069503386358:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:46.395100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789068811868571:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42003
2025-11-28T14:51:46.381562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:46.396271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789069503386348:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.383299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789070362307261:2107] [node 4] ICP18 handshake (actor [1:7577789070362307299:2134]) is too old
2025-11-28T14:51:46.396321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:46.396021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789068811868483:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.396070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:46.396789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789071362589865:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789071362589947:2118] held: no
2025-11-28T14:51:46.387533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789070362307296:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.397215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789069503386348:2109] [node 6] ICP22 created new session: [4:7577789069503386375:2109]
2025-11-28T14:51:46.396139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789068811868483:2048] [node 1] ICP22 created new session: [5:7577789068811868573:2048]
2025-11-28T14:51:46.387567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789070362307296:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:46.397361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789069503386375:2109] [node 6] ICS09 handshake done sender: [4:7577789069503386358:2113] self: [4:7577789069503386354:0] peer: [6:7577789071362589955:0] socket: 225 qp: -1
2025-11-28T14:51:46.396237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789068811868573:2048] [node 1] ICS09 handshake done sender: [5:7577789068811868565:2117] self: [5:7577789068811868568:0] peer: [1:7577789070362307295:0] socket: 262 qp: -1
2025-11-28T14:51:46.390409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789070362307296:2132] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:46.397332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789071362589947:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:46.397667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789069503386375:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:46.398363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789071362589865:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.390969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789070362307294:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.397690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:46.396384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789068811868573:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:46.390976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789070362307286:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.397734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789069503386374:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56179
2025-11-28T14:51:46.398406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:46.396429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:46.398450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789071362589924:2100] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789071362589949:2120] held: no
2025-11-28T14:51:46.390993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789070362307294:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:46.398512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789071362589865:2048] [node 1] ICP22 created new session: [6:7577789071362589957:2048]
No RDMA context found for GID: 2025-11-28T14:51:46.391568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
::ffff:127.1.0.0
2025-11-28T14:51:46.398786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789071362589957:2048] [node 1] ICS09 handshake done sender: [6:7577789071362589947:2118] self: [6:7577789071362589950:0] peer: [1:7577789070362307293:0] socket: 259 qp: -1
2025-11-28T14:51:46.391603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789070362307286:2108] [node 5] ICP22 created new session: [1:7577789070362307314:2108]
2025-11-28T14:51:46.391649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789070362307314:2108] [node 5] ICS09 handshake done sender: [1:7577789070362307296:2132] self: [1:7577789070362307295:0] peer: [5:7577789068811868568:0] socket: 260 qp: -1
2025-11-28T14:51:46.398891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789071362589957:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:46.391684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789070362307314:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:46.398923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589956:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
2025-11-28T14:51:46.398907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789069503386374:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.398938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:46.391871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:46.393661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789070362307294:2131] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:46.394543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789070362307285:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.394600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:46.394983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789070362307285:2116] [node 6] ICP22 created new session: [1:7577789070362307317:2116]
2025-11-28T14:51:46.395158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789070362307317:2116] [node 6] ICS09 handshake done sender: [1:7577789070362307294:2131] self: [1:7577789070362307293:0] peer: [6:7577789071362589950:0] socket: 258 qp: -1
2025-11-28T14:51:46.396029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789070362307317:2116] [node 6] ICS10 traffic start
2025-11-28T14:51:46.399697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789071362589949:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:46.396088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.399944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067998581319:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56088
2025-11-28T14:51:46.400092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789071362589924:2100] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.400189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789068811868575:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42003
2025-11-28T14:51:46.400244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:46.400302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789071362589924:2100] [node 4] ICP22 created new session: [6:7577789071362589960:2100]
2025-11-28T14:51:46.400514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789071362589960:2100] [node 4] ICS09 handshake done sender: [6:7577789071362589949:2120] self: [6:7577789071362589955:0] peer: [4:7577789069503386354:0] socket: 237 qp: -1
2025-11-28T14:51:46.400555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789071362589960:2100] [node 4] ICS10 traffic start
2025-11-28T14:51:46.400599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:46.401223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789068811868575:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.401430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789068811868547:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.401597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789069503386359:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.401628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789069503386359:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:46.401648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067998581319:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.403140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789069503386359:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.403811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789069503386349:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.404127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789068811868577:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42003
2025-11-28T14:51:46.404090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789069503386355:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.404152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:46.411337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071278351270:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.411290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789069503386349:2108] [node 5] ICP22 created new session: [4:7577789069503386380:2108]
2025-11-28T14:51:46.411483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789069503386380:2108] [node 5] ICS09 handshake done sender: [4:7577789069503386359:2114] self: [4:7577789069503386356:0] peer: [5:7577789068811868576:0] socket: 217 qp: -1
2025-11-28T14:51:46.411534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789069503386380:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:46.411671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:46.411649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789068811868575:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:46.411927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789068811868547:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789068811868575:2120] held: no
2025-11-28T14:51:46.412566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789068811868547:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.412613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:46.412684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789068811868547:2108] [node 4] ICP22 created new session: [5:7577789068811868578:2108]
2025-11-28T14:51:46.412743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789068811868578:2108] [node 4] ICS09 handshake done sender: [5:7577789068811868575:2120] self: [5:7577789068811868576:0] peer: [4:7577789069503386356:0] socket: 223 qp: -1
2025-11-28T14:51:46.412788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789068811868578:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:46.412829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:46.414767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789067998581299:2120] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.422191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789071362589944:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.422469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789071362589867:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.422817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067998581299:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.423296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067998581299:2120] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.423529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789067998581293:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.423600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589963:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
2025-11-28T14:51:46.423605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:46.423713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067998581293:2109] [node 6] ICP22 created new session: [3:7577789067998581322:2109]
2025-11-28T14:51:46.423764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067998581322:2109] [node 6] ICS09 handshake done sender: [3:7577789067998581299:2120] self: [3:7577789067998581298:0] peer: [6:7577789071362589962:0] socket: 232 qp: -1
2025-11-28T14:51:46.423809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067998581322:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:46.423854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:46.423894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789071362589944:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:46.424278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789071362589867:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6363) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789071362589944:2116] held: no
2025-11-28T14:51:46.424526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789071362589867:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.424570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:46.424626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789071362589867:2050] [node 3] ICP22 created new session: [6:7577789071362589964:2050]
2025-11-28T14:51:46.424712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789071362589964:2050] [node 3] ICS09 handshake done sender: [6:7577789071362589944:2116] self: [6:7577789071362589962:0] peer: [3:7577789067998581298:0] socket: 241 qp: -1
2025-11-28T14:51:46.424756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789071362589964:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:46.424803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:46.428346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789069503386357:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.436324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067998581310:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.438355Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789067998581304:2123] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.447571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789069503386365:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.447843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789069503386287:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.448089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067998581304:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.448659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067998581304:2123] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.448858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789069503386383:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56179
2025-11-28T14:51:46.448976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789067998581294:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.449052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:46.449093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067998581294:2107] [node 4] ICP22 created new session: [3:7577789067998581326:2107]
2025-11-28T14:51:46.449168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067998581326:2107] [node 4] ICS09 handshake done sender: [3:7577789067998581304:2123] self: [3:7577789067998581297:0] peer: [4:7577789069503386382:0] socket: 236 qp: -1
2025-11-28T14:51:46.449156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789069503386365:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:46.449206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067998581326:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:46.449335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789069503386287:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.449243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:46.449721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:46.449807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789069503386287:2050] [node 3] ICP22 created new session: [4:7577789069503386384:2050]
2025-11-28T14:51:46.449943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789069503386384:2050] [node 3] ICS09 handshake done sender: [4:7577789069503386365:2117] self: [4:7577789069503386382:0] peer: [3:7577789067998581297:0] socket: 242 qp: -1
2025-11-28T14:51:46.449994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789069503386384:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:46.450040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:46.461935Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789071278351260:2138] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.469460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789071362589945:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.469782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789071362589866:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.470108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071278351260:2138] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.470477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071278351260:2138] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:46.470650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789071278351255:2136] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.470698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:46.470745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071278351255:2136] [node 6] ICP22 created new session: [2:7577789071278351282:2136]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.470784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071278351282:2136] [node 6] ICS09 handshake done sender: [2:7577789071278351260:2138] self: [2:7577789071278351257:0] peer: [6:7577789071362589966:0] socket: 235 qp: -1
2025-11-28T14:51:46.470805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071278351282:2136] [node 6] ICS10 traffic start
2025-11-28T14:51:46.470854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:46.470846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789071362589967:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52858
2025-11-28T14:51:46.471058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789071362589945:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:46.471214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789071362589866:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.471257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:46.471314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789071362589866:2049] [node 2] ICP22 created new session: [6:7577789071362589968:2049]
2025-11-28T14:51:46.471356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789071362589968:2049] [node 2] ICS09 handshake done sender: [6:7577789071362589945:2117] self: [6:7577789071362589966:0] peer: [2:7577789071278351257:0] socket: 248 qp: -1
2025-11-28T14:51:46.471381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789071362589968:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:46.471410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:46.479078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789067998581301:2121] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.487347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071278351273:2145] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.497645Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789071278351258:2137] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.505193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067998581312:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.505471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789067998581221:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.505802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071278351258:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.506324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071278351258:2137] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.506530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789067998581329:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56088
2025-11-28T14:51:46.506552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789071278351164:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.506625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:46.506663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071278351164:2049] [node 3] ICP22 created new session: [2:7577789071278351286:2049]
2025-11-28T14:51:46.506722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071278351286:2049] [node 3] ICS09 handshake done sender: [2:7577789071278351258:2137] self: [2:7577789071278351256:0] peer: [3:7577789067998581328:0] socket: 230 qp: -1
2025-11-28T14:51:46.506759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071278351286:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:46.506796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:46.506857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067998581312:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:46.507936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789067998581221:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.507987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:46.508149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067998581221:2049] [node 2] ICP22 created new session: [3:7577789067998581330:2049]
2025-11-28T14:51:46.508499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067998581330:2049] [node 2] ICS09 handshake done sender: [3:7577789067998581312:2127] self: [3:7577789067998581328:0] peer: [2:7577789071278351256:0] socket: 249 qp: -1
2025-11-28T14:51:46.508558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067998581330:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:46.508676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:46.510023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789067998581221:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:46.510297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789071278351264:2140] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.518320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789068811868563:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.518556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789068811868484:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.518837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071278351264:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.519360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071278351264:2140] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.519569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789071278351254:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.519639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789068811868581:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42003
2025-11-28T14:51:46.519647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:46.519706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071278351254:2111] [node 5] ICP22 created new session: [2:7577789071278351289:2111]
2025-11-28T14:51:46.519783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071278351289:2111] [node 5] ICS09 handshake done sender: [2:7577789071278351264:2140] self: [2:7577789071278351259:0] peer: [5:7577789068811868580:0] socket: 245 qp: -1
2025-11-28T14:51:46.519826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071278351289:2111] [node 5] ICS10 traffic start
2025-11-28T14:51:46.519884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:46.519937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789068811868563:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:46.520291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789068811868484:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789068811868563:2116] held: no
2025-11-28T14:51:46.521190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789068811868484:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.521900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:46.521955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789068811868484:2049] [node 2] ICP22 created new session: [5:7577789068811868582:2049]
2025-11-28T14:51:46.522093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789068811868582:2049] [node 2] ICS09 handshake done sender: [5:7577789068811868563:2116] self: [5:7577789068811868580:0] peer: [2:7577789071278351259:0] socket: 246 qp: -1
2025-11-28T14:51:46.522141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789068811868582:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:46.522180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:46.524144Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789071278351265:2141] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.532923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789069503386372:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.533140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789069503386286:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.533452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071278351265:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.533969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071278351265:2141] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.534252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789069503386387:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56179
2025-11-28T14:51:46.534181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789071278351253:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.534289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:46.534343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071278351253:2107] [node 4] ICP22 created new session: [2:7577789071278351292:2107]
2025-11-28T14:51:46.534430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071278351292:2107] [node 4] ICS09 handshake done sender: [2:7577789071278351265:2141] self: [2:7577789071278351261:0] peer: [4:7577789069503386386:0] socket: 265 qp: -1
2025-11-28T14:51:46.534475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071278351292:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:46.534612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789069503386372:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:46.534524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:46.534858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789069503386286:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.535360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:46.535548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789069503386286:2049] [node 2] ICP22 created new session: [4:7577789069503386388:2049]
2025-11-28T14:51:46.535604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789069503386388:2049] [node 2] ICS09 handshake done sender: [4:7577789069503386372:2119] self: [4:7577789069503386386:0] peer: [2:7577789071278351261:0] socket: 266 qp: -1
2025-11-28T14:51:46.535638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789069503386388:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:46.535713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:46.536014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789069503386286:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:46.537511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789071362589940:2113] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.544628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071278351279:2147] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.544864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789071278351255:2136] [node 6] ICP18 handshake (actor [2:7577789071278351279:2147]) is too old
2025-11-28T14:51:46.548170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789067998581305:2124] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:46.548980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789068811868571:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.549194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789068811868485:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:46.549440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789067998581305:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:46.549818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789067998581305:2124] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:46.549988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789067998581295:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:46.550047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:46.550084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789068811868585:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42003
2025-11-28T14:51:46.550094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789067998581295:2108] [node 5] ICP22 created new session: [3:7577789067998581333:2108]
2025-11-28T14:51:46.550148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789067998581333:2108] [node 5] ICS09 handshake done sender: [3:7577789067998581305:2124] self: [3:7577789067998581300:0] peer: [5:7577789068811868584:0] socket: 270 qp: -1
2025-11-28T14:51:46.550179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789067998581333:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:46.550230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:46.550344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789068811868571:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:46.550822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789068811868485:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6363) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789068811868571:2119] held: no
2025-11-28T14:51:46.551318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789068811868485:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:46.551368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:46.551415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789068811868485:2050] [node 3] ICP22 created new session: [5:7577789068811868586:2050]
2025-11-28T14:51:46.551485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789068811868586:2050] [node 3] ICS09 handshake done sender: [5:7577789068811868571:2119] self: [5:7577789068811868584:0] peer: [3:7577789067998581300:0] socket: 239 qp: -1
2025-11-28T14:51:46.551532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789068811868586:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:46.551580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:46.586496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:46.586736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789071278351276:2048] [node 1] ICS07 socket disconnect 222 reason# EndOfStream
2025-11-28T14:51:46.586772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789071278351276:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.586900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:46.586958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789071278351276:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:46.587686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:46.587879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789067998581314:2048] [node 1] ICS07 socket disconnect 221 reason# EndOfStream
2025-11-28T14:51:46.587913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789067998581314:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.587996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:46.588061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789067998581314:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:46.589230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789071278351163:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:46.589275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789071278351276:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:46.590236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789067998581220:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:46.590277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789067998581314:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:46.591688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789071278351163:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.591688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:46.591831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789071278351276:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.591872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789071278351163:2048] [node 1] ICP30 unregister session Session# [2:7577789071278351276:2048] VirtualId# [2:7577789071278351271:0]
2025-11-28T14:51:46.591924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:46.591997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789069503386369:2048] [node 1] ICS07 socket disconnect 220 reason# EndOfStream
2025-11-28T14:51:46.592019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789067998581220:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.592085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789071278351163:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.592038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789069503386369:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.592215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789067998581314:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.592112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:46.592251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.592299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789067998581220:2048] [node 1] ICP30 unregister session Session# [3:7577789067998581314:2048] VirtualId# [3:7577789067998581311:0]
2025-11-28T14:51:46.592163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789069503386369:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:46.592341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:46.592421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789067998581220:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.592452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.593067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789069503386285:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.593150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789069503386369:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.593178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789069503386285:2048] [node 1] ICP30 unregister session Session# [4:7577789069503386369:2048] VirtualId# [4:7577789069503386366:0]
2025-11-28T14:51:46.593221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:46.593255Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789069503386285:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.593271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.593301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:46.593502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789068811868573:2048] [node 1] ICS01 socket: 262 reason# EndOfStream
2025-11-28T14:51:46.593555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789068811868483:2048] [node 1] ICP30 unregister session Session# [5:7577789068811868573:2048] VirtualId# [5:7577789068811868568:0]
2025-11-28T14:51:46.593643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789068811868573:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.593726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:46.593777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:46.594365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:46.594602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789071362589957:2048] [node 1] ICS01 socket: 259 reason# EndOfStream
2025-11-28T14:51:46.594645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789071362589865:2048] [node 1] ICP30 unregister session Session# [6:7577789071362589957:2048] VirtualId# [6:7577789071362589950:0]
2025-11-28T14:51:46.594673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789071362589957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.594753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:46.594813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:46.601771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:46.601979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789067998581330:2049] [node 2] ICS07 socket disconnect 249 reason# EndOfStream
2025-11-28T14:51:46.602012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789067998581330:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.602076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:46.602109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789067998581330:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:46.602595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789067998581220:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:46.602650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789069503386285:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:46.604307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:46.604666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789069503386388:2049] [node 2] ICS07 socket disconnect 266 reason# EndOfStream
2025-11-28T14:51:46.604701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789069503386388:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.604811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:46.604787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789067998581221:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:46.604824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789067998581330:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:46.604870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789069503386388:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:46.605446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:46.605646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789068811868582:2049] [node 2] ICS01 socket: 246 reason# EndOfStream
2025-11-28T14:51:46.605785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789068811868484:2049] [node 2] ICP30 unregister session Session# [5:7577789068811868582:2049] VirtualId# [5:7577789068811868580:0]
2025-11-28T14:51:46.605873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789067998581221:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.605832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789068811868582:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.606011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789067998581330:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.605906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:46.606038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789067998581221:2049] [node 2] ICP30 unregister session Session# [3:7577789067998581330:2049] VirtualId# [3:7577789067998581328:0]
2025-11-28T14:51:46.606082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:46.605966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:46.606081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789069503386286:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.606125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789067998581221:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.606149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789069503386388:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.606225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.606198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789069503386286:2049] [node 2] ICP30 unregister session Session# [4:7577789069503386388:2049] VirtualId# [4:7577789069503386386:0]
2025-11-28T14:51:46.606310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:46.606292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:46.606320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789069503386286:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.606337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.606511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789071362589968:2049] [node 2] ICS07 socket disconnect 248 reason# EndOfStream
2025-11-28T14:51:46.606548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789071362589968:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.606690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:46.606800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789071362589968:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:46.607558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789071362589866:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.607657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789071362589968:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.607688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789071362589866:2049] [node 2] ICP30 unregister session Session# [6:7577789071362589968:2049] VirtualId# [6:7577789071362589966:0]
2025-11-28T14:51:46.607725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:46.607748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789071362589866:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.607760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.615974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789069503386286:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:46.616055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:46.616255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789069503386384:2050] [node 3] ICS07 socket disconnect 242 reason# EndOfStream
2025-11-28T14:51:46.616289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789069503386384:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.616356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:46.616415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789069503386384:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:46.617098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:46.617183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789069503386287:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6363) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:46.617547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789069503386384:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:46.617572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789069503386287:2050] [node 3] ICP30 unregister session Session# [4:7577789069503386384:2050] VirtualId# [4:7577789069503386382:0]
2025-11-28T14:51:46.617612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:46.617644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789069503386287:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6363) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.617696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6363) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:46.617992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789071362589866:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:46.617944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789068811868586:2050] [node 3] ICS01 socket: 239 reason# EndOfStream
2025-11-28T14:51:46.618102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789068811868485:2050] [node 3] ICP30 unregister session Session# [5:7577789068811868586:2050] VirtualId# [5:7577789068811868584:0]
2025-11-28T14:51:46.618141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789068811868586:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.618219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:46.618398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:46.618826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:46.618987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789071362589964:2050] [node 3] ICS01 socket: 241 reason# EndOfStream
2025-11-28T14:51:46.619030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789071362589867:2050] [node 3] ICP30 unregister session Session# [6:7577789071362589964:2050] VirtualId# [6:7577789071362589962:0]
2025-11-28T14:51:46.619058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789071362589964:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.619136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:46.619201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:46.638274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:46.638513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789068811868578:2108] [node 4] ICS01 socket: 223 reason# EndOfStream
2025-11-28T14:51:46.638559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789068811868547:2108] [node 4] ICP30 unregister session Session# [5:7577789068811868578:2108] VirtualId# [5:7577789068811868576:0]
2025-11-28T14:51:46.638589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789068811868578:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.638688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:46.638741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:46.639142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:46.639322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789071362589960:2100] [node 4] ICS01 socket: 237 reason# EndOfStream
2025-11-28T14:51:46.639365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789071362589924:2100] [node 4] ICP30 unregister session Session# [6:7577789071362589960:2100] VirtualId# [6:7577789071362589955:0]
2025-11-28T14:51:46.639389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789071362589960:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.639476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:46.639536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:46.664329Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:46.664519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789071362589951:2105] [node 5] ICS01 socket: 233 reason# EndOfStream
2025-11-28T14:51:46.664552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789071362589929:2105] [node 5] ICP30 unregister session Session# [6:7577789071362589951:2105] VirtualId# [6:7577789071362589946:0]
2025-11-28T14:51:46.664577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789071362589951:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:46.664661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:46.664696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:46.783541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789069547188641:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:46.783597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789069547188642:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:46.804624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789068021765383:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.804681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789068021765384:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:46.825996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789071367547836:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.826051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789071367547837:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:46.846324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789069845335427:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.846383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789069845335428:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:46.846431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789069845335429:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:46.867071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789070773442511:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.867123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789070773442512:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:46.867157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789070773442513:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:46.887965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789070964980521:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:46.887999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789070964980522:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:46.888022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789070964980523:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:46.963020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789070964980581:2105] [node 5] ICP01 ready to work
2025-11-28T14:51:46.999745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789070773442574:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:46.999877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789069845335490:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:46.999727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789070964980584:2106] [node 4] ICP01 ready to work
2025-11-28T14:51:46.999745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789071367547926:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:46.999893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789069547188723:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:47.000320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789068021765447:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:47.000560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789069547188724:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:47.000563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789074140302787:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:47.000631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789070773442575:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:47.000652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789071367547927:2133] [node 6] ICP01 ready to work
2025-11-28T14:51:47.000744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789068021765448:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:47.000889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789069547188725:2128] [node 6] ICP01 ready to work
2025-11-28T14:51:47.001045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789072316732745:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:47.001039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789075662515224:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.011692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789074140302798:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12113
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.012768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789073842156032:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4734
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.013355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789074140302802:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12113
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.014182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789075068409885:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18717
2025-11-28T14:51:47.014258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789075068409886:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18717
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.015151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789073842156036:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4734
2025-11-28T14:51:47.015244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789073842156037:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4734
2025-11-28T14:51:47.016078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789073842156038:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4734
2025-11-28T14:51:47.016075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789075662515235:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15985
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.016979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789075662515236:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15985
2025-11-28T14:51:47.017106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789075662515237:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15985
2025-11-28T14:51:47.017381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789075662515238:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15985
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.020876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789075662515235:2142] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.021009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789072316732759:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3062
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.021249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789072316732760:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3062
2025-11-28T14:51:47.021332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789072316732761:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3062
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.021470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789072316732762:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3062
2025-11-28T14:51:47.021867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789072316732761:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.024125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789075068409886:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.024134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156032:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.024225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789072316732759:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.024577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156036:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.024597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789072316732760:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.024656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789075068409885:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.024720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156038:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.024622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789074140302798:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.024775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789074140302803:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12113
2025-11-28T14:51:47.024778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789068021765383:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.024863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789069845335428:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.024861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789070773442574:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.025050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789072316732751:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.025148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789074140302802:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.025077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789072316732751:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.025436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789073842156039:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4734
2025-11-28T14:51:47.026243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789074140302793:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.026263Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789074140302793:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.026388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156037:2136] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.026568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156025:2130] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.026582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789072316732751:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:47.026592Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789073842156025:2130] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.027529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789068021765447:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.027637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.027715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789068021765447:2107] [node 4] ICP22 created new session: [2:7577789072316732767:2107]
2025-11-28T14:51:47.027889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789072316732767:2107] [node 4] ICS09 handshake done sender: [2:7577789072316732751:2113] self: [2:7577789072316732750:0] peer: [4:7577789074140302804:0] socket: 218 qp: -1
2025-11-28T14:51:47.027970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789072316732767:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:47.027982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789069845335427:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.028021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:47.028111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789075068409889:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18717
2025-11-28T14:51:47.028181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789073842156025:2130] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:47.028339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789074140302793:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:47.028401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789075662515236:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.028770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156023:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.028837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789072316732762:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:51:47.028804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789073842156023:2129] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.029358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789075068409885:2115] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:51:47.029542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789069547188641:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.029508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789074140302806:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12113
2025-11-28T14:51:47.029649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789075662515237:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.029605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.029723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789070773442574:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.029672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789069547188641:2048] [node 2] ICP22 created new session: [1:7577789073842156045:2048]
2025-11-28T14:51:47.029758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789071367547837:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.029749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789073842156045:2048] [node 2] ICS09 handshake done sender: [1:7577789073842156025:2130] self: [1:7577789073842156024:0] peer: [2:7577789072316732765:0] socket: 229 qp: -1
2025-11-28T14:51:47.029935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789075662515238:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.029770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.029783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789073842156045:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:47.029821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:47.029926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789070773442574:2108] [node 4] ICP22 created new session: [5:7577789075068409890:2108]
2025-11-28T14:51:47.029992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789075068409890:2108] [node 4] ICS09 handshake done sender: [5:7577789075068409885:2115] self: [5:7577789075068409888:0] peer: [4:7577789074140302792:0] socket: 235 qp: -1
2025-11-28T14:51:47.030040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789075068409890:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:47.030078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789069845335490:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.030028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789072316732769:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3062
2025-11-28T14:51:47.030118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.030070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:47.030158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789069845335490:2108] [node 5] ICP22 created new session: [4:7577789074140302808:2108]
2025-11-28T14:51:47.030209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789074140302808:2108] [node 5] ICS09 handshake done sender: [4:7577789074140302793:2112] self: [4:7577789074140302792:0] peer: [5:7577789075068409888:0] socket: 227 qp: -1
2025-11-28T14:51:47.030236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789074140302808:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:47.030232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789071367547836:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.030274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.030407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156039:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.030693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789072316732749:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.030731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789072316732749:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.031510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789074140302803:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.031524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156027:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.031550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789073842156027:2131] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.032119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789075068409892:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18717
2025-11-28T14:51:47.032216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789074140302798:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.033776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789072316732760:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:47.033818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947895:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-28T14:51:47.034157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789069845335428:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.034219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.034197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789073842156023:2129] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:47.034296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789069845335428:2049] [node 2] ICP22 created new session: [4:7577789074140302812:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.034767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789074140302812:2049] [node 2] ICS09 handshake done sender: [4:7577789074140302798:2115] self: [4:7577789074140302804:0] peer: [2:7577789072316732750:0] socket: 220 qp: -1
2025-11-28T14:51:47.034526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789068021765383:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789072316732760:2117] held: no
2025-11-28T14:51:47.034821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789074140302812:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:47.034810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789068021765383:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.034821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789075662515246:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15985
2025-11-28T14:51:47.034892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:47.034845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.034867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789068021765383:2048] [node 1] ICP22 created new session: [2:7577789072316732773:2048]
2025-11-28T14:51:47.034930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789069547188723:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.034898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789072316732773:2048] [node 1] ICS09 handshake done sender: [2:7577789072316732760:2117] self: [2:7577789072316732765:0] peer: [1:7577789073842156024:0] socket: 253 qp: -1
2025-11-28T14:51:47.034983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.034921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789072316732773:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.035011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789069547188723:2107] [node 4] ICP22 created new session: [1:7577789073842156051:2107]
2025-11-28T14:51:47.034942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.035035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789073842156051:2107] [node 4] ICS09 handshake done sender: [1:7577789073842156023:2129] self: [1:7577789073842156022:0] peer: [4:7577789074140302807:0] socket: 225 qp: -1
2025-11-28T14:51:47.035056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789073842156051:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:47.035074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:47.035086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789075068409894:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18717
2025-11-28T14:51:47.035253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789075259947895:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.035362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789069845335429:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.035408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789070964980581:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.035535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789073842156027:2131] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:47.035650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789075068409881:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.035683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789075068409881:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.036809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789075068409881:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.037331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789075662515247:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15985
2025-11-28T14:51:47.037425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789075662515238:2145] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.038520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789070773442575:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.038610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947897:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-28T14:51:47.038595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:47.038708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789069547188642:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.038770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789070773442575:2109] [node 6] ICP22 created new session: [5:7577789075068409896:2109]
2025-11-28T14:51:47.038756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.038897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789075068409896:2109] [node 6] ICS09 handshake done sender: [5:7577789075068409881:2114] self: [5:7577789075068409880:0] peer: [6:7577789075259947896:0] socket: 221 qp: -1
2025-11-28T14:51:47.038938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789075068409896:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:47.038906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789069547188642:2049] [node 3] ICP22 created new session: [1:7577789073842156053:2049]
2025-11-28T14:51:47.038983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789073842156053:2049] [node 3] ICS09 handshake done sender: [1:7577789073842156027:2131] self: [1:7577789073842156026:0] peer: [3:7577789075662515245:0] socket: 232 qp: -1
2025-11-28T14:51:47.039015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:47.039011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789073842156053:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:47.038994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789075662515230:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.039028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789075662515230:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.039030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:47.039163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789072316732749:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:47.039496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789074140302811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12113
2025-11-28T14:51:47.039525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789071367547836:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.039642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.039704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789075259947895:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:47.039689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789071367547836:2048] [node 1] ICP22 created new session: [3:7577789075662515248:2048]
2025-11-28T14:51:47.039879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789075662515248:2048] [node 1] ICS09 handshake done sender: [3:7577789075662515238:2145] self: [3:7577789075662515245:0] peer: [1:7577789073842156026:0] socket: 241 qp: -1
2025-11-28T14:51:47.039908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789075662515248:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.039944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:47.040181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789070964980581:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.040232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789068021765384:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.040267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.040307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789070964980581:2105] [node 5] ICP22 created new session: [6:7577789075259947898:2105]
2025-11-28T14:51:47.040289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.040275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789069845335427:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789074140302802:2116] held: no
2025-11-28T14:51:47.040354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789075259947898:2105] [node 5] ICS09 handshake done sender: [6:7577789075259947895:2115] self: [6:7577789075259947896:0] peer: [5:7577789075068409880:0] socket: 267 qp: -1
2025-11-28T14:51:47.040331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789068021765384:2049] [node 3] ICP22 created new session: [2:7577789072316732777:2049]
2025-11-28T14:51:47.040459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789075259947898:2105] [node 5] ICS10 traffic start
2025-11-28T14:51:47.040451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789072316732777:2049] [node 3] ICS09 handshake done sender: [2:7577789072316732749:2112] self: [2:7577789072316732748:0] peer: [3:7577789075662515242:0] socket: 217 qp: -1
2025-11-28T14:51:47.040488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.040476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789072316732777:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:47.040541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:47.040671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789074140302802:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:47.040829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789075068409892:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.041176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789069845335427:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.041218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.041418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789069845335427:2048] [node 1] ICP22 created new session: [4:7577789074140302816:2048]
2025-11-28T14:51:47.041434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789072316732775:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3062
2025-11-28T14:51:47.041476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789074140302816:2048] [node 1] ICS09 handshake done sender: [4:7577789074140302802:2116] self: [4:7577789074140302807:0] peer: [1:7577789073842156022:0] socket: 233 qp: -1
2025-11-28T14:51:47.041509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789074140302816:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.041537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:47.042486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789070773442512:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.043276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789072316732753:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.043307Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789072316732753:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.045905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789075068409894:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.046027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789075662515235:2142] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.046287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789074140302818:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12113
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.046514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789070773442511:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.046565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789071367547837:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62987) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789075662515235:2142] held: no
2025-11-28T14:51:47.046631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789075068409899:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18717
2025-11-28T14:51:47.046699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789071367547837:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.046772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.046819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789071367547837:2049] [node 2] ICP22 created new session: [3:7577789075662515251:2049]
2025-11-28T14:51:47.046859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789075662515251:2049] [node 2] ICS09 handshake done sender: [3:7577789075662515235:2142] self: [3:7577789075662515242:0] peer: [2:7577789072316732748:0] socket: 223 qp: -1
2025-11-28T14:51:47.046834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947900:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-28T14:51:47.046889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789075662515251:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.046920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.047304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947901:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-28T14:51:47.047526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947902:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-28T14:51:47.047601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156029:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.047623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789073842156029:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.047870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789075259947900:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.047846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789072316732753:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:47.048007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789075662515230:2139] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:47.048285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789070964980584:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.048758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789068021765448:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.048728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789075259947902:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.048858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.048853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789073842156029:2132] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:47.048861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789074140302797:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.048895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789074140302797:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.048925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789075068409892:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:47.048931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789068021765448:2108] [node 5] ICP22 created new session: [2:7577789072316732780:2108]
2025-11-28T14:51:47.049584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789074140302803:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:47.048988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789072316732780:2108] [node 5] ICS09 handshake done sender: [2:7577789072316732753:2114] self: [2:7577789072316732752:0] peer: [5:7577789075068409898:0] socket: 264 qp: -1
2025-11-28T14:51:47.049481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789072316732780:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:47.049627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789070773442512:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.049630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789069547188724:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.049812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789071367547926:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.049786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.049534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.049841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789069547188724:2108] [node 5] ICP22 created new session: [1:7577789073842156057:2108]
2025-11-28T14:51:47.049798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.049923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789073842156057:2108] [node 5] ICS09 handshake done sender: [1:7577789073842156029:2132] self: [1:7577789073842156028:0] peer: [5:7577789075068409900:0] socket: 216 qp: -1
2025-11-28T14:51:47.049923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789070773442512:2049] [node 2] ICP22 created new session: [5:7577789075068409902:2049]
2025-11-28T14:51:47.050184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.050122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789073842156057:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:47.050167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789075068409902:2049] [node 2] ICS09 handshake done sender: [5:7577789075068409892:2118] self: [5:7577789075068409898:0] peer: [2:7577789072316732752:0] socket: 265 qp: -1
2025-11-28T14:51:47.050152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.050195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789075068409902:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:47.050274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789071367547926:2107] [node 4] ICP22 created new session: [3:7577789075662515255:2107]
2025-11-28T14:51:47.050226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:47.050300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789069845335429:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.050310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789075259947901:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.050317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789075662515255:2107] [node 4] ICS09 handshake done sender: [3:7577789075662515230:2139] self: [3:7577789075662515229:0] peer: [4:7577789074140302814:0] socket: 251 qp: -1
2025-11-28T14:51:47.050361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789075662515255:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:47.050438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.050429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:47.050513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789070964980521:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.050494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789069845335429:2050] [node 3] ICP22 created new session: [4:7577789074140302820:2050]
2025-11-28T14:51:47.050544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789074140302820:2050] [node 3] ICS09 handshake done sender: [4:7577789074140302803:2117] self: [4:7577789074140302814:0] peer: [3:7577789075662515229:0] socket: 257 qp: -1
2025-11-28T14:51:47.050685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789070964980522:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.050571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789074140302820:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:47.050599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:47.051091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789073842156031:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.051114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789073842156031:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.052696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789073842156031:2133] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:47.051903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789072316732755:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.051933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789072316732755:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.053676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789069547188725:2128] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.053727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:47.053749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789069547188725:2128] [node 6] ICP22 created new session: [1:7577789073842156060:2128]
2025-11-28T14:51:47.053791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789073842156060:2128] [node 6] ICS09 handshake done sender: [1:7577789073842156031:2133] self: [1:7577789073842156030:0] peer: [6:7577789075259947905:0] socket: 239 qp: -1
2025-11-28T14:51:47.053810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789073842156060:2128] [node 6] ICS10 traffic start
2025-11-28T14:51:47.054026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789072316732755:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:47.053831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:47.054304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789074140302797:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:47.055020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789072316732745:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.055073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: 2025-11-28T14:51:47.055309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789074140302787:2109] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:51:47.055192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789072316732745:2109] [node 6] ICP22 created new session: [2:7577789072316732783:2109]
2025-11-28T14:51:47.055440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:47.055369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789072316732783:2109] [node 6] ICS09 handshake done sender: [2:7577789072316732755:2115] self: [2:7577789072316732754:0] peer: [6:7577789075259947906:0] socket: 236 qp: -1
2025-11-28T14:51:47.055485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789074140302787:2109] [node 6] ICP22 created new session: [4:7577789074140302822:2109]
2025-11-28T14:51:47.055461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789072316732783:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:47.055530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789074140302822:2109] [node 6] ICS09 handshake done sender: [4:7577789074140302797:2114] self: [4:7577789074140302796:0] peer: [6:7577789075259947904:0] socket: 222 qp: -1
No RDMA context found for GID: 2025-11-28T14:51:47.055521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947907:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-28T14:51:47.055513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
2025-11-28T14:51:47.055564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789074140302822:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:47.055595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:47.055676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789075068409901:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18717
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.055849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789075662515254:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15985
2025-11-28T14:51:47.056610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789070773442511:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789075068409894:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.056860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947908:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-28T14:51:47.056885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789075068409894:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.057570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789070773442511:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.057537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947909:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-28T14:51:47.057606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.057643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789070773442511:2048] [node 1] ICP22 created new session: [5:7577789075068409905:2048]
2025-11-28T14:51:47.057689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789075068409905:2048] [node 1] ICS09 handshake done sender: [5:7577789075068409894:2119] self: [5:7577789075068409900:0] peer: [1:7577789073842156028:0] socket: 272 qp: -1
2025-11-28T14:51:47.057721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789075068409905:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.057749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.057906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947910:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.058276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789075068409904:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18717
2025-11-28T14:51:47.058303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789070964980521:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789075259947902:2119] held: no
2025-11-28T14:51:47.058982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789075662515254:2148] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.059196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789075068409904:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.059576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789075662515234:2141] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.059607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789075662515234:2141] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.059697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789070964980522:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62987) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789075259947901:2118] held: no
2025-11-28T14:51:47.059860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789075259947902:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:47.059292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789070773442513:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.060269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789075259947900:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:47.060470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789075259947901:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:47.060491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789075662515234:2141] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.060723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789070964980521:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.060714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789075068409908:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18717
2025-11-28T14:51:47.060796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.060783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789075662515224:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.060852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789070964980521:2048] [node 1] ICP22 created new session: [6:7577789075259947911:2048]
2025-11-28T14:51:47.060863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.060915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789075259947911:2048] [node 1] ICS09 handshake done sender: [6:7577789075259947902:2119] self: [6:7577789075259947905:0] peer: [1:7577789073842156030:0] socket: 247 qp: -1
2025-11-28T14:51:47.060911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789075662515224:2112] [node 5] ICP22 created new session: [3:7577789075662515260:2112]
2025-11-28T14:51:47.060949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789075259947911:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.060952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789075662515260:2112] [node 5] ICS09 handshake done sender: [3:7577789075662515234:2141] self: [3:7577789075662515233:0] peer: [5:7577789075068409907:0] socket: 224 qp: -1
2025-11-28T14:51:47.061014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:47.061013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789075662515260:2112] [node 5] ICS10 traffic start
2025-11-28T14:51:47.061048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.061101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789075068409904:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:47.061260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789070773442513:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.061305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.061338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789070773442513:2050] [node 3] ICP22 created new session: [5:7577789075068409909:2050]
2025-11-28T14:51:47.061378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789075068409909:2050] [node 3] ICS09 handshake done sender: [5:7577789075068409904:2122] self: [5:7577789075068409907:0] peer: [3:7577789075662515233:0] socket: 234 qp: -1
2025-11-28T14:51:47.061401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789075068409909:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:47.061420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:47.061528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789075259947907:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.061572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789070964980584:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.061621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.061670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789070964980584:2106] [node 4] ICP22 created new session: [6:7577789075259947914:2106]
2025-11-28T14:51:47.061731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789075259947914:2106] [node 4] ICS09 handshake done sender: [6:7577789075259947900:2117] self: [6:7577789075259947904:0] peer: [4:7577789074140302796:0] socket: 248 qp: -1
2025-11-28T14:51:47.061767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789075259947914:2106] [node 4] ICS10 traffic start
2025-11-28T14:51:47.061803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:47.062366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789070964980522:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.062422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.062456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789070964980522:2049] [node 2] ICP22 created new session: [6:7577789075259947916:2049]
2025-11-28T14:51:47.062503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789075259947916:2049] [node 2] ICS09 handshake done sender: [6:7577789075259947901:2118] self: [6:7577789075259947906:0] peer: [2:7577789072316732754:0] socket: 249 qp: -1
2025-11-28T14:51:47.062543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789075259947916:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:47.062590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.063407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789074140302824:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12113
2025-11-28T14:51:47.063743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789070964980523:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.064021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789075662515232:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.064050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789075662515232:2140] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.064747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789075662515232:2140] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.064974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789075259947919:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-28T14:51:47.065038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789071367547927:2133] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.065105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:47.065148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789071367547927:2133] [node 6] ICP22 created new session: [3:7577789075662515263:2133]
2025-11-28T14:51:47.065188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789075662515263:2133] [node 6] ICS09 handshake done sender: [3:7577789075662515232:2140] self: [3:7577789075662515231:0] peer: [6:7577789075259947918:0] socket: 242 qp: -1
2025-11-28T14:51:47.065214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789075662515263:2133] [node 6] ICS10 traffic start
2025-11-28T14:51:47.065241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:47.065419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789075259947907:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:47.065606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789070964980523:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.065666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.065707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789070964980523:2050] [node 3] ICP22 created new session: [6:7577789075259947920:2050]
2025-11-28T14:51:47.065750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789075259947920:2050] [node 3] ICS09 handshake done sender: [6:7577789075259947907:2120] self: [6:7577789075259947918:0] peer: [3:7577789075662515231:0] socket: 250 qp: -1
2025-11-28T14:51:47.065780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789075259947920:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:47.065819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:47.080105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:47.080346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789072316732773:2048] [node 1] ICS01 socket: 253 reason# EndOfStream
2025-11-28T14:51:47.080404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789068021765383:2048] [node 1] ICP30 unregister session Session# [2:7577789072316732773:2048] VirtualId# [2:7577789072316732765:0]
2025-11-28T14:51:47.080441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789072316732773:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.080539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.080617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.081223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:47.081459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789075662515248:2048] [node 1] ICS01 socket: 241 reason# EndOfStream
2025-11-28T14:51:47.081525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789071367547836:2048] [node 1] ICP30 unregister session Session# [3:7577789075662515248:2048] VirtualId# [3:7577789075662515245:0]
2025-11-28T14:51:47.081574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789075662515248:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.081659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.081716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.083189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:47.083375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789074140302816:2048] [node 1] ICS01 socket: 233 reason# EndOfStream
2025-11-28T14:51:47.083417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789069845335427:2048] [node 1] ICP30 unregister session Session# [4:7577789074140302816:2048] VirtualId# [4:7577789074140302807:0]
2025-11-28T14:51:47.083442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789074140302816:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.083512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.083556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.083917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:47.084087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789075068409905:2048] [node 1] ICS01 socket: 272 reason# EndOfStream
2025-11-28T14:51:47.084125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789070773442511:2048] [node 1] ICP30 unregister session Session# [5:7577789075068409905:2048] VirtualId# [5:7577789075068409900:0]
2025-11-28T14:51:47.084173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789075068409905:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.084242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.084287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.084593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:47.084821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789075259947911:2048] [node 1] ICS01 socket: 247 reason# EndOfStream
2025-11-28T14:51:47.084872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789070964980521:2048] [node 1] ICP30 unregister session Session# [6:7577789075259947911:2048] VirtualId# [6:7577789075259947905:0]
2025-11-28T14:51:47.084901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789075259947911:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.084966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.085019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.107027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:47.107185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789075662515251:2049] [node 2] ICS01 socket: 223 reason# EndOfStream
2025-11-28T14:51:47.107220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789071367547837:2049] [node 2] ICP30 unregister session Session# [3:7577789075662515251:2049] VirtualId# [3:7577789075662515242:0]
2025-11-28T14:51:47.107240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789075662515251:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.107343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:47.107391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:47.109725Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:47.109869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789074140302812:2049] [node 2] ICS01 socket: 220 reason# EndOfStream
2025-11-28T14:51:47.109916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789069845335428:2049] [node 2] ICP30 unregister session Session# [4:7577789074140302812:2049] VirtualId# [4:7577789074140302804:0]
2025-11-28T14:51:47.109945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789074140302812:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.110014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:47.110045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:47.110707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:47.110860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789075068409902:2049] [node 2] ICS01 socket: 265 reason# EndOfStream
2025-11-28T14:51:47.110897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789070773442512:2049] [node 2] ICP30 unregister session Session# [5:7577789075068409902:2049] VirtualId# [5:7577789075068409898:0]
2025-11-28T14:51:47.110930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789075068409902:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.111002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:47.111050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:47.111645Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:47.111842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789075259947916:2049] [node 2] ICS01 socket: 249 reason# EndOfStream
2025-11-28T14:51:47.111880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789070964980522:2049] [node 2] ICP30 unregister session Session# [6:7577789075259947916:2049] VirtualId# [6:7577789075259947906:0]
2025-11-28T14:51:47.111914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789075259947916:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.111989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:47.112041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:47.132198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:47.132412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789074140302820:2050] [node 3] ICS07 socket disconnect 257 reason# EndOfStream
2025-11-28T14:51:47.132446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789074140302820:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.132542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:47.132586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789074140302820:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:47.133282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:47.133442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789069845335429:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28990) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.133490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789074140302820:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.133525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789069845335429:2050] [node 3] ICP30 unregister session Session# [4:7577789074140302820:2050] VirtualId# [4:7577789074140302814:0]
2025-11-28T14:51:47.133472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789075068409909:2050] [node 3] ICS01 socket: 234 reason# EndOfStream
2025-11-28T14:51:47.133568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:47.133611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789069845335429:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28990) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.133520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789070773442513:2050] [node 3] ICP30 unregister session Session# [5:7577789075068409909:2050] VirtualId# [5:7577789075068409907:0]
2025-11-28T14:51:47.133550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789075068409909:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.133638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:47.133685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:47.133646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28990) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.134353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:47.134534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789075259947920:2050] [node 3] ICS07 socket disconnect 250 reason# EndOfStream
2025-11-28T14:51:47.134571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789075259947920:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.134638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:47.134693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789075259947920:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:47.135351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789070964980523:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28990) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.135406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789075259947920:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.135437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789070964980523:2050] [node 3] ICP30 unregister session Session# [6:7577789075259947920:2050] VirtualId# [6:7577789075259947918:0]
2025-11-28T14:51:47.135474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:47.135498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789070964980523:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28990) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.135511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28990) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.145402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789070964980523:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:47.157463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:47.157624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789075068409890:2108] [node 4] ICS01 socket: 235 reason# EndOfStream
2025-11-28T14:51:47.157658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789070773442574:2108] [node 4] ICP30 unregister session Session# [5:7577789075068409890:2108] VirtualId# [5:7577789075068409888:0]
2025-11-28T14:51:47.157678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789075068409890:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.157750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:47.157785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:47.158347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:47.158542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789075259947914:2106] [node 4] ICS07 socket disconnect 248 reason# EndOfStream
2025-11-28T14:51:47.158570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789075259947914:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.158643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:47.158697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789075259947914:2106] [node 4] ICS15 start handshake
2025-11-28T14:51:47.159378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789070964980584:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20783) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.159426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789075259947914:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.159458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789070964980584:2106] [node 4] ICP30 unregister session Session# [6:7577789075259947914:2106] VirtualId# [6:7577789075259947904:0]
2025-11-28T14:51:47.159498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:47.159557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789070964980584:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.159576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.168928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789070964980584:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:51:47.176839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:47.177087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789075259947898:2105] [node 5] ICS01 socket: 267 reason# EndOfStream
2025-11-28T14:51:47.177130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789070964980581:2105] [node 5] ICP30 unregister session Session# [6:7577789075259947898:2105] VirtualId# [6:7577789075259947896:0]
2025-11-28T14:51:47.177169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789075259947898:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.177264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:47.177319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:47.234839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789072629166352:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:47.234950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789072629166353:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:47.252080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071701054504:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:47.252092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071701054503:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.268617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789072762119626:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.268660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789072762119627:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:47.285804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789073558433571:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.285863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789073558433572:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:47.285948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789073558433573:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:47.302745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789072965938495:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.302748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789072965938496:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:47.302781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789072965938497:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:47.320058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789074032591079:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.320103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789074032591080:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:47.320122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789074032591081:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:47.415707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789074032591142:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:47.419117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789072762119690:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:47.419278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789073558433652:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:47.419177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789072965938559:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:47.419273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789072965938560:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:47.419118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789072762119689:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:47.419485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789074032591144:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:47.419515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789072762119691:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:47.419809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789073558433653:2116] [node 6] ICP01 ready to work
2025-11-28T14:51:47.421404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789073558433571:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22380028ns) {Configure:104(22373197ns) {ProcessConfigured:135(97541ns) {StartInitialHandshake:157(94924ns) {DropHandshakes:470(2004ns) {DropIncomingHandshake:442(546ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(554ns) PrepareNewSessionHandshake:477(1072ns) {SwitchToState:179(478ns)}}}}}
2025-11-28T14:51:47.423092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789071701054503:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46309770ns) {Configure:104(46300507ns) {ProcessConfigured:135(296491ns) {StartInitialHandshake:157(294473ns) {DropHandshakes:470(1372ns) {DropIncomingHandshake:442(489ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(505ns) PrepareNewSessionHandshake:477(833ns) {SwitchToState:179(323ns)}}}}}
2025-11-28T14:51:47.423132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789071701054504:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45744349ns) {Configure:104(45741662ns) {ProcessConfigured:135(286852ns) {StartInitialHandshake:157(284891ns) {DropHandshakes:470(9629ns) {DropIncomingHandshake:442(800ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(435ns) PrepareNewSessionHandshake:477(888ns) {SwitchToState:179(344ns)}}}}}
2025-11-28T14:51:47.423127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789072629166353:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47623046ns) {Configure:104(47225034ns) {ProcessConfigured:135(199347ns) {StartInitialHandshake:157(197743ns) {DropHandshakes:470(1357ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(520ns) PrepareNewSessionHandshake:477(1227ns) {SwitchToState:179(645ns)}}}}}
2025-11-28T14:51:47.423134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789072629166352:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47678424ns) {Configure:104(47654452ns) {ProcessConfigured:135(201783ns) {StartInitialHandshake:157(199217ns) {DropHandshakes:470(1618ns) {DropIncomingHandshake:442(407ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(1083ns) {SwitchToState:179(554ns)}}}}}
2025-11-28T14:51:47.423736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789072629166426:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:47.423815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071701054572:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:47.424044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071701054573:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:47.424171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789072629166427:2114] [node 5] ICP01 ready to work
2025-11-28T14:51:47.424619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789072629166428:2115] [node 6] ICP01 ready to work
2025-11-28T14:51:47.424755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789071701054574:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.432387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789073558433667:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11572
2025-11-28T14:51:47.434488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789073558433667:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.434842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789073558433573:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.435353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789072762119693:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.435783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789072965938569:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21568
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.435392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789072762119693:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.436999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789072762119693:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:47.436982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789072965938570:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21568
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.437055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071701054579:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.437250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071701054581:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24115
2025-11-28T14:51:47.437271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789072965938571:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21568
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.437328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789072762119706:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59201
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.437446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071701054580:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24115
2025-11-28T14:51:47.437570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789072965938569:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.437770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789073558433669:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11572
2025-11-28T14:51:47.437701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071701054579:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.438265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789072965938570:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.438443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789072965938571:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.438489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789072762119690:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.438613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.438689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789073558433667:2133] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:47.438493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789072965938560:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.438770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071701054581:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.438757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789072762119690:2107] [node 4] ICP22 created new session: [3:7577789072762119709:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.439649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789072762119709:2107] [node 4] ICS09 handshake done sender: [3:7577789072762119693:2110] self: [3:7577789072762119692:0] peer: [4:7577789073558433668:0] socket: 225 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.439599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789072965938497:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.440232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071701054580:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.439578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789072629166433:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55131
2025-11-28T14:51:47.440091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789073558433573:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.439871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789072762119709:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:47.440346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071701054587:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.440025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789072629166434:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55131
2025-11-28T14:51:47.440530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789072965938577:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21568
2025-11-28T14:51:47.439916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:47.440333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.440795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789073558433573:2050] [node 3] ICP22 created new session: [4:7577789073558433670:2050]
2025-11-28T14:51:47.440843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789072629166439:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55131
2025-11-28T14:51:47.440840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789073558433659:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.441743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789072629166440:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55131
2025-11-28T14:51:47.440866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789073558433659:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.441015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789073558433670:2050] [node 3] ICS09 handshake done sender: [4:7577789073558433667:2133] self: [4:7577789073558433668:0] peer: [3:7577789072762119692:0] socket: 229 qp: -1
2025-11-28T14:51:47.442171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166434:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.441517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789073558433670:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:47.443864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789072762119697:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.442527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166433:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.442108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:47.443898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789072762119697:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.453390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789072762119707:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59201
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:51:47.453774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166440:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.453952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789072762119708:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59201
2025-11-28T14:51:47.453928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789073558433672:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11572
2025-11-28T14:51:47.454125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789073558433659:2130] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.454295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789072762119697:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:47.454390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789073558433652:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.454481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789072965938581:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21568
2025-11-28T14:51:47.454797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.455116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789073558433652:2108] [node 5] ICP22 created new session: [4:7577789073558433675:2108]
2025-11-28T14:51:47.455167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789073558433675:2108] [node 5] ICS09 handshake done sender: [4:7577789073558433659:2130] self: [4:7577789073558433658:0] peer: [5:7577789072965938578:0] socket: 224 qp: -1
2025-11-28T14:51:47.455224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789073558433675:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:47.455812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.456351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789072762119689:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.456419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.456588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789072762119689:2108] [node 5] ICP22 created new session: [3:7577789072762119713:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.456718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789072762119713:2108] [node 5] ICS09 handshake done sender: [3:7577789072762119697:2112] self: [3:7577789072762119696:0] peer: [5:7577789072965938579:0] socket: 228 qp: -1
2025-11-28T14:51:47.456868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789072762119713:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:47.456926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.457451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591158:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.457718Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789072965938569:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:47.458003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591159:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.458566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789074032591158:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.458804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789074032591144:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.458828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789072965938582:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21568
2025-11-28T14:51:47.459302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789073558433666:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.459334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789073558433666:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.459932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789074032591159:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.459902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789072965938560:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.460036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.460080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789073558433666:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:47.460090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789072965938560:2108] [node 4] ICP22 created new session: [5:7577789072965938583:2108]
2025-11-28T14:51:47.460144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789072965938583:2108] [node 4] ICS09 handshake done sender: [5:7577789072965938569:2115] self: [5:7577789072965938578:0] peer: [4:7577789073558433658:0] socket: 234 qp: -1
2025-11-28T14:51:47.460168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789074032591081:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.460187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789072965938583:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:47.460224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.460811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789072762119704:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.460478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789073558433653:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.460853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789072762119704:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.460546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:47.460592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789073558433653:2116] [node 6] ICP22 created new session: [4:7577789073558433678:2116]
2025-11-28T14:51:47.460630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789073558433678:2116] [node 6] ICS09 handshake done sender: [4:7577789073558433666:2132] self: [4:7577789073558433665:0] peer: [6:7577789074032591160:0] socket: 271 qp: -1
2025-11-28T14:51:47.461102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591161:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
2025-11-28T14:51:47.460687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789073558433678:2116] [node 6] ICS10 traffic start
2025-11-28T14:51:47.460719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:47.461448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789074032591158:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:47.462470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789074032591144:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.462548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.462627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789074032591144:2108] [node 4] ICP22 created new session: [6:7577789074032591163:2108]
2025-11-28T14:51:47.462827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789074032591163:2108] [node 4] ICS09 handshake done sender: [6:7577789074032591158:2116] self: [6:7577789074032591160:0] peer: [4:7577789073558433665:0] socket: 272 qp: -1
2025-11-28T14:51:47.462852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789074032591163:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:47.462631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789072762119704:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:47.462959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789072762119691:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.463503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:47.462891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:47.463680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789072762119691:2109] [node 6] ICP22 created new session: [3:7577789072762119717:2109]
2025-11-28T14:51:47.463760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789072762119717:2109] [node 6] ICS09 handshake done sender: [3:7577789072762119704:2114] self: [3:7577789072762119703:0] peer: [6:7577789074032591162:0] socket: 274 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.463851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789072762119717:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:47.463894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:47.464005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591164:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
2025-11-28T14:51:47.464178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789072965938570:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:47.464491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789074032591159:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:47.464760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789072965938497:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.464789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789074032591081:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.464845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.464891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789074032591081:2050] [node 3] ICP22 created new session: [6:7577789074032591167:2050]
2025-11-28T14:51:47.464844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.464898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789072965938497:2050] [node 3] ICP22 created new session: [5:7577789072965938585:2050]
2025-11-28T14:51:47.464941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789074032591167:2050] [node 3] ICS09 handshake done sender: [6:7577789074032591159:2117] self: [6:7577789074032591162:0] peer: [3:7577789072762119703:0] socket: 275 qp: -1
2025-11-28T14:51:47.464948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789072965938585:2050] [node 3] ICS09 handshake done sender: [5:7577789072965938570:2116] self: [5:7577789072965938579:0] peer: [3:7577789072762119696:0] socket: 235 qp: -1
2025-11-28T14:51:47.464977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789074032591167:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:47.465079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:47.464990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789072965938585:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:47.465108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.466746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789072762119719:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59201
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.467847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591169:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
2025-11-28T14:51:47.469851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789074032591169:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.470040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789074032591142:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.470248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789072965938573:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.470282Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789072965938573:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:47.471057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789072965938573:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.471456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789072965938559:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.471515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:47.471576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789072965938559:2110] [node 6] ICP22 created new session: [5:7577789072965938589:2110]
2025-11-28T14:51:47.471535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591171:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
2025-11-28T14:51:47.471636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789072965938589:2110] [node 6] ICS09 handshake done sender: [5:7577789072965938573:2118] self: [5:7577789072965938572:0] peer: [6:7577789074032591170:0] socket: 280 qp: -1
2025-11-28T14:51:47.471679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789072965938589:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:47.471746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:47.471767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789074032591169:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:47.472041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789074032591142:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.472121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.472162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789074032591142:2106] [node 5] ICP22 created new session: [6:7577789074032591172:2106]
2025-11-28T14:51:47.472210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789074032591172:2106] [node 5] ICS09 handshake done sender: [6:7577789074032591169:2120] self: [6:7577789074032591170:0] peer: [5:7577789072965938572:0] socket: 281 qp: -1
2025-11-28T14:51:47.472251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789074032591172:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:47.472308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.476888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789071701054571:2113] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.485219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071701054588:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24115
2025-11-28T14:51:47.496865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789071701054570:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:47.505040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071701054587:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.505258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789072762119706:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.505515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789072762119627:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.505758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071701054570:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.506093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071701054570:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:47.506768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789071701054504:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.506823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.506857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071701054504:2049] [node 3] ICP22 created new session: [2:7577789071701054594:2049]
2025-11-28T14:51:47.506884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071701054594:2049] [node 3] ICS09 handshake done sender: [2:7577789071701054570:2112] self: [2:7577789071701054569:0] peer: [3:7577789072762119720:0] socket: 243 qp: -1
2025-11-28T14:51:47.506904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071701054594:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:47.506927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:47.506891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789072762119721:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59201
2025-11-28T14:51:47.507201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789072762119706:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:47.507572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789072762119627:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.507643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.507691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789072762119627:2049] [node 2] ICP22 created new session: [3:7577789072762119722:2049]
2025-11-28T14:51:47.507738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789072762119722:2049] [node 2] ICS09 handshake done sender: [3:7577789072762119706:2115] self: [3:7577789072762119720:0] peer: [2:7577789071701054569:0] socket: 244 qp: -1
2025-11-28T14:51:47.507771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789072762119722:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:47.507844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.509546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789073558433680:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11572
2025-11-28T14:51:47.522961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789072629166424:2117] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:47.530933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789072762119707:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.530951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166439:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.531166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789072762119626:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.544230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789072965938566:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:47.553789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789072762119708:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.554014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789072762119689:2108] [node 5] ICP18 handshake (actor [3:7577789072762119708:2117]) is too old
2025-11-28T14:51:47.573902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789072629166431:2119] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.581010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789072629166444:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55131
2025-11-28T14:51:47.581082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789073558433672:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.581230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166424:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.581283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789073558433571:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.581622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166444:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.581684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166431:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.581846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789072629166424:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.582000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789072762119725:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59201
2025-11-28T14:51:47.582277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789072629166353:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.582311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:47.582377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789072762119707:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:47.582345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789072629166353:2049] [node 3] ICP22 created new session: [1:7577789072629166449:2049]
2025-11-28T14:51:47.582385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789072629166449:2049] [node 3] ICS09 handshake done sender: [1:7577789072629166424:2117] self: [1:7577789072629166422:0] peer: [3:7577789072762119724:0] socket: 250 qp: -1
2025-11-28T14:51:47.582408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789072629166449:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:47.582477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.582777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789073558433682:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11572
2025-11-28T14:51:47.582753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789072762119626:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.582818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.582887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789072629166353:2049] [node 3] ICP18 handshake (actor [1:7577789072629166444:2126]) is too old
2025-11-28T14:51:47.582913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789072762119626:2048] [node 1] ICP22 created new session: [3:7577789072762119726:2048]
2025-11-28T14:51:47.582962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789072762119726:2048] [node 1] ICS09 handshake done sender: [3:7577789072762119707:2116] self: [3:7577789072762119724:0] peer: [1:7577789072629166422:0] socket: 256 qp: -1
2025-11-28T14:51:47.582992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789072762119726:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.583020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:47.584185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789072629166431:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:47.584414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789073558433672:2135] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:47.584916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789073558433571:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.584978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.585009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789073558433571:2048] [node 1] ICP22 created new session: [4:7577789073558433683:2048]
2025-11-28T14:51:47.585042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789073558433683:2048] [node 1] ICS09 handshake done sender: [4:7577789073558433672:2135] self: [4:7577789073558433681:0] peer: [1:7577789072629166429:0] socket: 265 qp: -1
2025-11-28T14:51:47.585066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789073558433683:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.585110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:47.585272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789072629166426:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.585321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.585357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789072629166426:2107] [node 4] ICP22 created new session: [1:7577789072629166452:2107]
2025-11-28T14:51:47.585403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789072629166452:2107] [node 4] ICS09 handshake done sender: [1:7577789072629166431:2119] self: [1:7577789072629166429:0] peer: [4:7577789073558433681:0] socket: 263 qp: -1
2025-11-28T14:51:47.585442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789072629166452:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:47.585477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.587241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789072965938591:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21568
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.587659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591174:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
2025-11-28T14:51:47.595950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789071701054576:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:47.604499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789072965938577:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.604782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789072965938496:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.605082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071701054576:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.605569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071701054576:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.605868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789072965938593:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21568
2025-11-28T14:51:47.606002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789071701054572:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.606087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.606141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071701054572:2108] [node 5] ICP22 created new session: [2:7577789071701054598:2108]
2025-11-28T14:51:47.606210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789072965938577:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:47.606202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071701054598:2108] [node 5] ICS09 handshake done sender: [2:7577789071701054576:2114] self: [2:7577789071701054575:0] peer: [5:7577789072965938592:0] socket: 255 qp: -1
2025-11-28T14:51:47.606250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071701054598:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:47.606549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789072965938496:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.606291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.606636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.606692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789072965938496:2049] [node 2] ICP22 created new session: [5:7577789072965938594:2049]
2025-11-28T14:51:47.606747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789072965938594:2049] [node 2] ICS09 handshake done sender: [5:7577789072965938577:2119] self: [5:7577789072965938592:0] peer: [2:7577789071701054575:0] socket: 257 qp: -1
2025-11-28T14:51:47.606789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789072965938594:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:47.606848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.610037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591175:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
2025-11-28T14:51:47.622797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789071701054578:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:47.630795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789073558433680:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.630983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789073558433572:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.631164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071701054578:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.631488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071701054578:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.631799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789073558433686:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11572
2025-11-28T14:51:47.631931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789071701054573:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.631981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:47.632028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071701054573:2107] [node 4] ICP22 created new session: [2:7577789071701054602:2107]
2025-11-28T14:51:47.632053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789073558433680:2136] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:47.632078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071701054602:2107] [node 4] ICS09 handshake done sender: [2:7577789071701054578:2115] self: [2:7577789071701054577:0] peer: [4:7577789073558433685:0] socket: 240 qp: -1
2025-11-28T14:51:47.632115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071701054602:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:47.632336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789073558433572:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.632315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:47.632427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.632480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789073558433572:2049] [node 2] ICP22 created new session: [4:7577789073558433687:2049]
2025-11-28T14:51:47.632531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789073558433687:2049] [node 2] ICS09 handshake done sender: [4:7577789073558433680:2136] self: [4:7577789073558433685:0] peer: [2:7577789071701054577:0] socket: 226 qp: -1
2025-11-28T14:51:47.632570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789073558433687:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:47.632649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:47.637923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789072629166425:2118] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:47.645188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071701054588:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.645306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789071701054503:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.647993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789072629166432:2120] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:47.655818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166425:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.655936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789072965938591:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.656200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789072965938495:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.656293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789072629166425:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.656557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789071701054605:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24115
2025-11-28T14:51:47.656734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166432:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.656805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789072629166352:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.656879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.656980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789072629166352:2048] [node 2] ICP22 created new session: [1:7577789072629166456:2048]
2025-11-28T14:51:47.657049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789072629166456:2048] [node 2] ICS09 handshake done sender: [1:7577789072629166425:2118] self: [1:7577789072629166423:0] peer: [2:7577789071701054604:0] socket: 258 qp: -1
2025-11-28T14:51:47.657068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071701054588:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:47.657096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789072629166456:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:47.657135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:47.657328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789071701054503:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.657367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.657397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071701054503:2048] [node 1] ICP22 created new session: [2:7577789071701054606:2048]
2025-11-28T14:51:47.657443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071701054606:2048] [node 1] ICS09 handshake done sender: [2:7577789071701054588:2121] self: [2:7577789071701054604:0] peer: [1:7577789072629166423:0] socket: 262 qp: -1
2025-11-28T14:51:47.657473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071701054606:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.657498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.658606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789072965938597:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21568
2025-11-28T14:51:47.658723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789072629166432:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:47.658923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789072965938591:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:47.660600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789072629166427:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.660847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:47.659161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789072965938495:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.660627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.660896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789072629166427:2114] [node 5] ICP22 created new session: [1:7577789072629166459:2114]
2025-11-28T14:51:47.660960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789072965938495:2048] [node 1] ICP22 created new session: [5:7577789072965938598:2048]
2025-11-28T14:51:47.660981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789071701054585:2119] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:47.661029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789072629166459:2114] [node 5] ICS09 handshake done sender: [1:7577789072629166432:2120] self: [1:7577789072629166430:0] peer: [5:7577789072965938596:0] socket: 222 qp: -1
2025-11-28T14:51:47.661017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789072965938598:2048] [node 1] ICS09 handshake done sender: [5:7577789072965938591:2122] self: [5:7577789072965938596:0] peer: [1:7577789072629166430:0] socket: 268 qp: -1
2025-11-28T14:51:47.661147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789072629166459:2114] [node 5] ICS10 traffic start
2025-11-28T14:51:47.661179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:47.661053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789072965938598:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.661152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:47.669489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789074032591175:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.669760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789074032591080:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.670070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789071701054585:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:51:47.670627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789071701054585:2119] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:51:47.670756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591177:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
2025-11-28T14:51:47.670863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789071701054574:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.670918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:47.670954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789071701054574:2110] [node 6] ICP22 created new session: [2:7577789071701054609:2110]
2025-11-28T14:51:47.670991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789071701054609:2110] [node 6] ICS09 handshake done sender: [2:7577789071701054585:2119] self: [2:7577789071701054582:0] peer: [6:7577789074032591176:0] socket: 249 qp: -1
2025-11-28T14:51:47.671028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789074032591175:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:47.671015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789071701054609:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:47.671049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:47.671284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789074032591080:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.671373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:47.671416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789074032591080:2049] [node 2] ICP22 created new session: [6:7577789074032591178:2049]
2025-11-28T14:51:47.671510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789074032591178:2049] [node 2] ICS09 handshake done sender: [6:7577789074032591175:2123] self: [6:7577789074032591176:0] peer: [2:7577789071701054582:0] socket: 254 qp: -1
2025-11-28T14:51:47.671547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789074032591178:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:47.671652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:47.675395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789072629166437:2123] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:47.676357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789074032591174:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.676517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789074032591079:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:47.676724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789072629166437:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:47.677139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789072629166437:2123] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:47.677335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789072629166428:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:47.677272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789074032591181:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37135
2025-11-28T14:51:47.677390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:47.677432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789072629166428:2115] [node 6] ICP22 created new session: [1:7577789072629166462:2115]
2025-11-28T14:51:47.677478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789072629166462:2115] [node 6] ICS09 handshake done sender: [1:7577789072629166437:2123] self: [1:7577789072629166436:0] peer: [6:7577789074032591180:0] socket: 239 qp: -1
2025-11-28T14:51:47.677513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789072629166462:2115] [node 6] ICS10 traffic start
2025-11-28T14:51:47.677539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:47.677527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789074032591174:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:47.677753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789074032591079:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:47.677822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:47.677859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789074032591079:2048] [node 1] ICP22 created new session: [6:7577789074032591182:2048]
2025-11-28T14:51:47.677885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789074032591182:2048] [node 1] ICS09 handshake done sender: [6:7577789074032591174:2122] self: [6:7577789074032591180:0] peer: [1:7577789072629166436:0] socket: 273 qp: -1
2025-11-28T14:51:47.677950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789074032591182:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:47.678001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:47.681319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:47.681531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789071701054606:2048] [node 1] ICS01 socket: 262 reason# EndOfStream
2025-11-28T14:51:47.681574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789071701054503:2048] [node 1] ICP30 unregister session Session# [2:7577789071701054606:2048] VirtualId# [2:7577789071701054604:0]
2025-11-28T14:51:47.681608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789071701054606:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.681714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.681767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.682207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:47.682363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789072762119726:2048] [node 1] ICS01 socket: 256 reason# EndOfStream
2025-11-28T14:51:47.682407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789072762119626:2048] [node 1] ICP30 unregister session Session# [3:7577789072762119726:2048] VirtualId# [3:7577789072762119724:0]
2025-11-28T14:51:47.682439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789072762119726:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.682523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.682567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.683738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:47.683955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789073558433683:2048] [node 1] ICS01 socket: 265 reason# EndOfStream
2025-11-28T14:51:47.684024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789073558433571:2048] [node 1] ICP30 unregister session Session# [4:7577789073558433683:2048] VirtualId# [4:7577789073558433681:0]
2025-11-28T14:51:47.684071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789073558433683:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.684167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.684241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.684383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:47.684531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789072965938598:2048] [node 1] ICS01 socket: 268 reason# EndOfStream
2025-11-28T14:51:47.684565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789072965938495:2048] [node 1] ICP30 unregister session Session# [5:7577789072965938598:2048] VirtualId# [5:7577789072965938596:0]
2025-11-28T14:51:47.684593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789072965938598:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.684661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.684710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.685079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:47.685287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789074032591182:2048] [node 1] ICS07 socket disconnect 273 reason# EndOfStream
2025-11-28T14:51:47.685315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789074032591182:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.685371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:47.685433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789074032591182:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:47.686094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789074032591079:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23511) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.686138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789074032591182:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.686191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789074032591079:2048] [node 1] ICP30 unregister session Session# [6:7577789074032591182:2048] VirtualId# [6:7577789074032591180:0]
2025-11-28T14:51:47.686230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:47.686304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789074032591079:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23511) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.686316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23511) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.691013Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:47.691229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789072762119722:2049] [node 2] ICS07 socket disconnect 244 reason# EndOfStream
2025-11-28T14:51:47.691257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789072762119722:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.691360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:47.691421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789072762119722:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:47.692174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789072762119627:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.692272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789072762119722:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.692345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789072762119627:2049] [node 2] ICP30 unregister session Session# [3:7577789072762119722:2049] VirtualId# [3:7577789072762119720:0]
2025-11-28T14:51:47.692416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:47.692448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789072762119627:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.692464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.693452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:47.693604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789073558433687:2049] [node 2] ICS07 socket disconnect 226 reason# EndOfStream
2025-11-28T14:51:47.693629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789073558433687:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.693712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:47.693748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789073558433687:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:47.694231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:47.694467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789072965938594:2049] [node 2] ICS01 socket: 257 reason# EndOfStream
2025-11-28T14:51:47.694509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789072965938496:2049] [node 2] ICP30 unregister session Session# [5:7577789072965938594:2049] VirtualId# [5:7577789072965938592:0]
2025-11-28T14:51:47.694500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789073558433572:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.694621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789073558433687:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.694611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789072965938594:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.694664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789073558433572:2049] [node 2] ICP30 unregister session Session# [4:7577789073558433687:2049] VirtualId# [4:7577789073558433685:0]
2025-11-28T14:51:47.694730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:47.694718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:47.694787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:47.694802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789073558433572:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.694850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.694852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:47.695106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789074032591178:2049] [node 2] ICS07 socket disconnect 254 reason# EndOfStream
2025-11-28T14:51:47.695138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789074032591178:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.695206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:47.695251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789074032591178:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:47.696005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789074032591080:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.696094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789074032591178:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.696127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789074032591080:2049] [node 2] ICP30 unregister session Session# [6:7577789074032591178:2049] VirtualId# [6:7577789074032591176:0]
2025-11-28T14:51:47.696167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:47.696187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789074032591080:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.696194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.696256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789074032591079:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:47.704513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789073558433572:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:47.706508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789074032591080:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:47.731080Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:47.731240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789073558433670:2050] [node 3] ICS07 socket disconnect 229 reason# EndOfStream
2025-11-28T14:51:47.731265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789073558433670:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.731379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:47.731437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789073558433670:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:47.731816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:47.732023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789072965938585:2050] [node 3] ICS07 socket disconnect 235 reason# EndOfStream
2025-11-28T14:51:47.732060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789072965938585:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.732061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789073558433573:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16871) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.732171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789073558433670:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.732147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:47.732200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789073558433573:2050] [node 3] ICP30 unregister session Session# [4:7577789073558433670:2050] VirtualId# [4:7577789073558433668:0]
2025-11-28T14:51:47.732199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789072965938585:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:47.732237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:47.732257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789073558433573:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.732265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.732508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:47.732747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789074032591167:2050] [node 3] ICS07 socket disconnect 275 reason# EndOfStream
2025-11-28T14:51:47.732776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789074032591167:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.732831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789072965938497:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16871) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.732856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:47.732864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789072965938585:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.732911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789074032591167:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:47.732928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789072965938497:2050] [node 3] ICP30 unregister session Session# [5:7577789072965938585:2050] VirtualId# [5:7577789072965938579:0]
2025-11-28T14:51:47.732991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:47.733012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789072965938497:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.733022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.733709Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789074032591081:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16871) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.733785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789074032591167:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.733807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789074032591081:2050] [node 3] ICP30 unregister session Session# [6:7577789074032591167:2050] VirtualId# [6:7577789074032591162:0]
2025-11-28T14:51:47.733846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:47.733865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789074032591081:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.733876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.742373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789072965938497:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:47.743412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789074032591081:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:47.752022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:47.752197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789072965938583:2108] [node 4] ICS07 socket disconnect 234 reason# EndOfStream
2025-11-28T14:51:47.752221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789072965938583:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.752322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:47.752387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789072965938583:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:47.752769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:47.752946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789072965938560:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13357) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.753020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789072965938583:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.753003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789074032591163:2108] [node 4] ICS07 socket disconnect 272 reason# EndOfStream
2025-11-28T14:51:47.753045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789074032591163:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.753058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789072965938560:2108] [node 4] ICP30 unregister session Session# [5:7577789072965938583:2108] VirtualId# [5:7577789072965938578:0]
2025-11-28T14:51:47.753103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:47.753118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:47.753144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789072965938560:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.753180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789074032591163:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:47.753194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.753886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789074032591144:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13357) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:47.753963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789074032591163:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:47.754025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789074032591144:2108] [node 4] ICP30 unregister session Session# [6:7577789074032591163:2108] VirtualId# [6:7577789074032591160:0]
2025-11-28T14:51:47.754061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:47.754079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789074032591144:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.754091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:47.763896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789074032591144:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:47.782165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:47.782396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789074032591172:2106] [node 5] ICS01 socket: 281 reason# EndOfStream
2025-11-28T14:51:47.782439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789074032591142:2106] [node 5] ICP30 unregister session Session# [6:7577789074032591172:2106] VirtualId# [6:7577789074032591170:0]
2025-11-28T14:51:47.782469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789074032591172:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:47.782566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:47.782626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:47.868342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789071829665745:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:47.868505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789071829665746:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:47.889229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789073305583751:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.889242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789073305583752:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:47.909220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789072029094876:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.909289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789072029094877:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:47.927653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789074894307981:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.927693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789074894307983:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:47.927782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789074894307982:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:47.946708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789071835221260:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.946737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789071835221262:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:47.946748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789071835221261:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:47.968582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789074518504224:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:47.968654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789074518504226:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:47.968775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789074518504225:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:48.080320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789078813471579:2104] [node 5] ICP01 ready to work
2025-11-28T14:51:48.080408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789076124633104:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:48.082621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789079189275340:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:48.082646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789076324062236:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:48.083026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789077600551137:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:48.082967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789076324062235:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:48.083949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789079189275341:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:48.083870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789078813471582:2105] [node 4] ICP01 ready to work
2025-11-28T14:51:48.083545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789076324062237:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:48.084856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789076124633105:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:48.084907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789077600551138:2134] [node 5] ICP01 ready to work
2025-11-28T14:51:48.090917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789076124633109:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:48.090933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789071829665746:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51917171ns) {Configure:104(51911643ns) {ProcessConfigured:135(46091ns) {StartInitialHandshake:157(44199ns) {DropHandshakes:470(2033ns) {DropIncomingHandshake:442(576ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:477(893ns) {SwitchToState:179(395ns)}}}}}
2025-11-28T14:51:48.091056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789077600551140:2123] [node 6] ICP01 ready to work
2025-11-28T14:51:48.091658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789073305583751:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(57620025ns) {Configure:104(57615745ns) {ProcessConfigured:135(110615ns) {StartInitialHandshake:157(108528ns) {DropHandshakes:470(1492ns) {DropIncomingHandshake:442(523ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(522ns) PrepareNewSessionHandshake:477(923ns) {SwitchToState:179(227ns)}}}}}
2025-11-28T14:51:48.091694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789071829665745:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58920157ns) {Configure:104(58910951ns) {ProcessConfigured:135(132516ns) {StartInitialHandshake:157(129408ns) {DropHandshakes:470(2926ns) {DropIncomingHandshake:442(798ns) DropOutgoingHandshake:457(659ns)} GenerateSessionVirtualId:988(1013ns) PrepareNewSessionHandshake:477(1388ns) {SwitchToState:179(609ns)}}}}}
2025-11-28T14:51:48.091912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789071835221261:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47787378ns) {Configure:104(47784659ns) {ProcessConfigured:135(340840ns) {StartInitialHandshake:157(338584ns) {DropHandshakes:470(5134ns) {DropIncomingHandshake:442(413ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(803ns) PrepareNewSessionHandshake:477(1386ns) {SwitchToState:179(774ns)}}}}}
2025-11-28T14:51:48.091923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789074518504225:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(37540007ns) {Configure:104(37530520ns) {ProcessConfigured:135(347573ns) {StartInitialHandshake:157(345333ns) {DropHandshakes:470(1132ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(3397ns) {SwitchToState:179(556ns)}}}}}
2025-11-28T14:51:48.092917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789071835221260:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49001632ns) {Configure:104(48996478ns) {ProcessConfigured:135(488665ns) {StartInitialHandshake:157(486562ns) {DropHandshakes:470(1279ns) {DropIncomingHandshake:442(424ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:477(2239ns) {SwitchToState:179(980ns)}}}}}
2025-11-28T14:51:48.093212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789074894307983:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(50710890ns) {Configure:104(50706199ns) {ProcessConfigured:135(48780ns) {StartInitialHandshake:157(46313ns) {DropHandshakes:470(1134ns) {DropIncomingHandshake:442(347ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(472ns) PrepareNewSessionHandshake:477(543ns) {SwitchToState:179(191ns)}}}}}
2025-11-28T14:51:48.092955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789071835221262:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48765033ns) {Configure:104(48762415ns) {ProcessConfigured:135(492462ns) {StartInitialHandshake:157(490233ns) {DropHandshakes:470(394685ns) {DropIncomingHandshake:442(399ns) DropOutgoingHandshake:457(632ns)} GenerateSessionVirtualId:988(996ns) PrepareNewSessionHandshake:477(1091ns) {SwitchToState:179(557ns)}}}}}
2025-11-28T14:51:48.093210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789076130188634:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:48.093693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789073305583752:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(58899947ns) {Configure:104(58896331ns) {ProcessConfigured:135(759807ns) {StartInitialHandshake:157(756485ns) {DropHandshakes:470(1131ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(681ns) PrepareNewSessionHandshake:477(2389ns) {SwitchToState:179(1002ns)}}}}}
2025-11-28T14:51:48.095290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789076130188636:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.099695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076130188635:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64579
2025-11-28T14:51:48.101068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076130188635:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.102420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076324062244:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.102873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789077600551151:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39207
2025-11-28T14:51:48.102816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789079189275348:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42481
2025-11-28T14:51:48.102949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789077600551152:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39207
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.102931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789076124633120:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30577
2025-11-28T14:51:48.103360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789079189275353:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42481
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.113547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789076124633121:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30577
2025-11-28T14:51:48.103623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789079189275348:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.114320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076324062244:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.113632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789076124633122:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30577
2025-11-28T14:51:48.130264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789077600551155:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39207
2025-11-28T14:51:48.120935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789076124633123:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30577
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.138854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789077600551156:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39207
2025-11-28T14:51:48.143053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789077600551151:2142] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.145175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789076124633120:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.145265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789076124633121:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.145141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076324062245:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-28T14:51:48.146049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789073305583751:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.146889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789076124633110:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.146922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789076124633110:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.146668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789079189275353:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.148114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076324062251:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-28T14:51:48.146978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789077600551152:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.148153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076324062252:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.148521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076324062255:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-28T14:51:48.148322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789079189275357:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42481
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.148759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789076124633110:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:48.148588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076324062245:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.148793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076324062251:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.149246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789079189275358:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42481
2025-11-28T14:51:48.149695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789071829665745:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.149131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076324062252:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.150323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:48.150291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789074894307981:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.149643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789072029094877:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.150961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789071829665745:2048] [node 2] ICP22 created new session: [1:7577789076124633127:2048]
2025-11-28T14:51:48.149731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789072029094876:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.150966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789077600551143:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.151017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789077600551143:2138] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.151020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789076124633127:2048] [node 2] ICS09 handshake done sender: [1:7577789076124633110:2111] self: [1:7577789076124633108:0] peer: [2:7577789077600551157:0] socket: 236 qp: -1
2025-11-28T14:51:48.157413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789077600551159:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39207
2025-11-28T14:51:48.157049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789076124633127:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:48.157123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:48.157542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789076124633107:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.157575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789076124633107:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.162591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789073305583751:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789077600551151:2142] held: no
2025-11-28T14:51:48.161010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789076124633107:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:48.161794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789071829665746:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.162868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471598:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.162562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:48.162605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789071829665746:2049] [node 3] ICP22 created new session: [1:7577789076124633131:2049]
2025-11-28T14:51:48.162727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789076124633131:2049] [node 3] ICS09 handshake done sender: [1:7577789076124633107:2110] self: [1:7577789076124633106:0] peer: [3:7577789076324062258:0] socket: 235 qp: -1
2025-11-28T14:51:48.162954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789077600551151:2142] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:48.162831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789076124633131:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:48.163073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789077600551143:2138] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:48.162858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:48.163094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076324062260:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-28T14:51:48.163734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789073305583751:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.163771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:48.163838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789078813471598:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.163992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789074518504224:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.163962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789073305583751:2048] [node 1] ICP22 created new session: [2:7577789077600551162:2048]
2025-11-28T14:51:48.163967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789073305583752:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.164469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076130188645:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64579
2025-11-28T14:51:48.164041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789077600551162:2048] [node 1] ICS09 handshake done sender: [2:7577789077600551151:2142] self: [2:7577789077600551157:0] peer: [1:7577789076124633108:0] socket: 238 qp: -1
2025-11-28T14:51:48.164698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789076124633114:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.164072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789077600551162:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:48.164746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076130188646:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64579
2025-11-28T14:51:48.164727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789076124633114:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.164085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:48.164434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789073305583752:2049] [node 3] ICP22 created new session: [2:7577789077600551163:2049]
2025-11-28T14:51:48.165239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076130188645:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.164699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:48.165595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789071835221260:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.164778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789077600551163:2049] [node 3] ICS09 handshake done sender: [2:7577789077600551143:2138] self: [2:7577789077600551142:0] peer: [3:7577789076324062259:0] socket: 251 qp: -1
2025-11-28T14:51:48.164888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789077600551163:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:48.166072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789076124633119:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.166074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076324062245:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:48.164916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:48.166111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789076124633119:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.166362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789076124633114:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:48.167065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789076124633109:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.167409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:48.167643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789076124633109:2109] [node 6] ICP22 created new session: [1:7577789076124633135:2109]
2025-11-28T14:51:48.168926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789072029094876:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:48.168892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789076124633135:2109] [node 6] ICS09 handshake done sender: [1:7577789076124633114:2113] self: [1:7577789076124633112:0] peer: [6:7577789078813471599:0] socket: 248 qp: -1
::ffff:127.1.0.0
2025-11-28T14:51:48.168932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789076124633135:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:48.169007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:48.169006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:48.169299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789072029094876:2048] [node 1] ICP22 created new session: [3:7577789076324062262:2048]
2025-11-28T14:51:48.169007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789076124633119:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:48.169301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471600:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.169368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789076324062262:2048] [node 1] ICS09 handshake done sender: [3:7577789076324062245:2114] self: [3:7577789076324062258:0] peer: [1:7577789076124633106:0] socket: 243 qp: -1
2025-11-28T14:51:48.169402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789076324062262:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:48.169582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:48.169443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789076124633105:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.169676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.169754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789076124633105:2108] [node 5] ICP22 created new session: [1:7577789076124633138:2108]
2025-11-28T14:51:48.169814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789076124633138:2108] [node 5] ICS09 handshake done sender: [1:7577789076124633119:2114] self: [1:7577789076124633118:0] peer: [5:7577789076130188647:0] socket: 275 qp: -1
2025-11-28T14:51:48.169821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076130188648:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64579
2025-11-28T14:51:48.169845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789076124633138:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:48.169876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:48.170118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471601:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.170837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789078813471598:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:48.170840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076130188645:2125] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:48.171208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789078813471601:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.171307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789074518504224:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.171420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789071835221260:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789076130188645:2125] held: no
2025-11-28T14:51:48.171357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:48.171591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789074518504224:2048] [node 1] ICP22 created new session: [6:7577789078813471602:2048]
2025-11-28T14:51:48.171722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789071835221260:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.172148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:48.171632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789078813471602:2048] [node 1] ICS09 handshake done sender: [6:7577789078813471598:2115] self: [6:7577789078813471599:0] peer: [1:7577789076124633112:0] socket: 272 qp: -1
2025-11-28T14:51:48.172267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076130188646:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.172253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789077600551149:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.171651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789074518504225:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.172801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789071835221260:2048] [node 1] ICP22 created new session: [5:7577789076130188649:2048]
2025-11-28T14:51:48.172309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789077600551149:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.171709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789078813471602:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:48.173000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789076130188649:2048] [node 1] ICS09 handshake done sender: [5:7577789076130188645:2125] self: [5:7577789076130188647:0] peer: [1:7577789076124633118:0] socket: 276 qp: -1
2025-11-28T14:51:48.171942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:48.173416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076324062261:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.173632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789077600551168:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39207
2025-11-28T14:51:48.174603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789077600551149:2141] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:48.173384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789076130188649:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:48.175033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789077600551147:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.175066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789077600551147:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.173395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789071835221261:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.173431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:48.175782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471606:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.175844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076324062252:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:48.175864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789077600551140:2123] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.175937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:48.176184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789078813471601:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:48.176204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789072029094877:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.176039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789077600551140:2123] [node 6] ICP22 created new session: [2:7577789077600551169:2123]
2025-11-28T14:51:48.176153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789077600551169:2123] [node 6] ICS09 handshake done sender: [2:7577789077600551149:2141] self: [2:7577789077600551148:0] peer: [6:7577789078813471603:0] socket: 284 qp: -1
2025-11-28T14:51:48.176255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:48.176303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789072029094877:2049] [node 2] ICP22 created new session: [3:7577789076324062264:2049]
2025-11-28T14:51:48.176329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789074518504225:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.176235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789077600551169:2123] [node 6] ICS10 traffic start
2025-11-28T14:51:48.176590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789076324062264:2049] [node 2] ICS09 handshake done sender: [3:7577789076324062252:2117] self: [3:7577789076324062259:0] peer: [2:7577789077600551142:0] socket: 260 qp: -1
2025-11-28T14:51:48.176478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:48.176621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789076324062264:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:48.176275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:48.176627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789074518504225:2049] [node 2] ICP22 created new session: [6:7577789078813471607:2049]
2025-11-28T14:51:48.176594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789077600551168:2147] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.176665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:48.176687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789078813471607:2049] [node 2] ICS09 handshake done sender: [6:7577789078813471601:2117] self: [6:7577789078813471603:0] peer: [2:7577789077600551148:0] socket: 285 qp: -1
2025-11-28T14:51:48.176732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789077600551147:2140] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:48.176712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789078813471607:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:48.176746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.176885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076130188653:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64579
2025-11-28T14:51:48.177431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076130188646:2126] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:48.177408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789077600551138:2134] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.177571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:48.177739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789071835221261:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.177836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789077600551138:2134] [node 5] ICP22 created new session: [2:7577789077600551172:2134]
2025-11-28T14:51:48.177948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:48.178010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789077600551172:2134] [node 5] ICS09 handshake done sender: [2:7577789077600551147:2140] self: [2:7577789077600551145:0] peer: [5:7577789076130188650:0] socket: 277 qp: -1
2025-11-28T14:51:48.178029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789071835221261:2049] [node 2] ICP22 created new session: [5:7577789076130188654:2049]
2025-11-28T14:51:48.178036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789077600551172:2134] [node 5] ICS10 traffic start
2025-11-28T14:51:48.178087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789076130188654:2049] [node 2] ICS09 handshake done sender: [5:7577789076130188646:2126] self: [5:7577789076130188650:0] peer: [2:7577789077600551145:0] socket: 278 qp: -1
2025-11-28T14:51:48.178068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:48.178125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789076130188654:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:48.178161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:48.179044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789077600551140:2123] [node 6] ICP18 handshake (actor [2:7577789077600551168:2147]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.181178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789079189275345:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.188671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471609:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.188827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789076124633122:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.189119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789076124633113:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.189168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789076124633113:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:51:48.189198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789078813471609:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:51:48.189354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789079189275361:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42481
2025-11-28T14:51:48.189395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789078813471579:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.189655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076130188641:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.189694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789076130188641:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.190570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789079189275361:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.190858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789076124633113:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:48.191107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789076124633104:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.191182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.191282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789076124633104:2107] [node 4] ICP22 created new session: [1:7577789076124633142:2107]
2025-11-28T14:51:48.191385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076130188641:2123] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:48.191354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789076124633142:2107] [node 4] ICS09 handshake done sender: [1:7577789076124633113:2112] self: [1:7577789076124633111:0] peer: [4:7577789079189275360:0] socket: 237 qp: -1
2025-11-28T14:51:48.191404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789076124633142:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:48.191674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471611:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.191607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789079189275362:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42481
2025-11-28T14:51:48.191448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:48.191845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789076130188636:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.192207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789078813471609:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:48.192143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:48.192507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789076130188636:2109] [node 6] ICP22 created new session: [5:7577789076130188658:2109]
2025-11-28T14:51:48.192578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789078813471579:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.192561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789076130188658:2109] [node 6] ICS09 handshake done sender: [5:7577789076130188641:2123] self: [5:7577789076130188640:0] peer: [6:7577789078813471610:0] socket: 240 qp: -1
2025-11-28T14:51:48.192952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789076130188658:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:48.192665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:48.193186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:48.194125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789074894307981:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789079189275353:2116] held: no
2025-11-28T14:51:48.192720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789078813471579:2104] [node 5] ICP22 created new session: [6:7577789078813471612:2104]
2025-11-28T14:51:48.194335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789079189275353:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:48.192768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789078813471612:2104] [node 5] ICS09 handshake done sender: [6:7577789078813471609:2119] self: [6:7577789078813471610:0] peer: [5:7577789076130188640:0] socket: 242 qp: -1
2025-11-28T14:51:48.192797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789078813471612:2104] [node 5] ICS10 traffic start
2025-11-28T14:51:48.192855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:48.194531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789078813471579:2104] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:48.195784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789074894307981:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.195872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:48.195912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789074894307981:2048] [node 1] ICP22 created new session: [4:7577789079189275364:2048]
2025-11-28T14:51:48.195972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789079189275364:2048] [node 1] ICS09 handshake done sender: [4:7577789079189275353:2116] self: [4:7577789079189275360:0] peer: [1:7577789076124633111:0] socket: 239 qp: -1
2025-11-28T14:51:48.196018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789079189275364:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:48.196075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.197848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471614:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.198673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789079189275347:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.207503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789077600551155:2144] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.211795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789077600551146:2139] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.219531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789079189275357:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.219703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789074894307982:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.219954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789077600551146:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.220415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789077600551146:2139] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.220601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789077600551137:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.220584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789079189275368:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42481
2025-11-28T14:51:48.220706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:48.220771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789077600551137:2107] [node 4] ICP22 created new session: [2:7577789077600551176:2107]
2025-11-28T14:51:48.220817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789077600551176:2107] [node 4] ICS09 handshake done sender: [2:7577789077600551146:2139] self: [2:7577789077600551144:0] peer: [4:7577789079189275367:0] socket: 249 qp: -1
2025-11-28T14:51:48.220863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789077600551176:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:48.220938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:48.221794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789079189275357:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:48.222053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789074894307982:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789079189275357:2117] held: no
2025-11-28T14:51:48.222137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789074894307982:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.223366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:48.223415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789074894307982:2049] [node 2] ICP22 created new session: [4:7577789079189275369:2049]
2025-11-28T14:51:48.223446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789079189275369:2049] [node 2] ICS09 handshake done sender: [4:7577789079189275357:2117] self: [4:7577789079189275367:0] peer: [2:7577789077600551144:0] socket: 250 qp: -1
2025-11-28T14:51:48.223468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789079189275369:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:48.223497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:48.223804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789079189275343:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.231785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076130188660:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64579
2025-11-28T14:51:48.246462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076324062243:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.256357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076324062255:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.256454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789076124633123:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.256736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789071829665746:2049] [node 3] ICP18 handshake (actor [1:7577789076124633123:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.257711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076130188661:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64579
2025-11-28T14:51:48.274157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076324062242:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.282194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789077600551156:2145] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.282398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789073305583752:2049] [node 3] ICP18 handshake (actor [2:7577789077600551156:2145]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.283183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471615:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.299261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076324062240:2110] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.307081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789079189275358:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.307210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789074894307983:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.307506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076324062240:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.307902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076324062240:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.308058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789079189275373:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42481
2025-11-28T14:51:48.308267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789076324062236:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.308352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:48.308409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789076324062236:2107] [node 4] ICP22 created new session: [3:7577789076324062270:2107]
2025-11-28T14:51:48.308458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789076324062270:2107] [node 4] ICS09 handshake done sender: [3:7577789076324062240:2110] self: [3:7577789076324062238:0] peer: [4:7577789079189275372:0] socket: 264 qp: -1
2025-11-28T14:51:48.308494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789076324062270:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:48.308535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:48.308544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789079189275358:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:48.308788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789074894307983:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.308844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:48.308880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789074894307983:2050] [node 3] ICP22 created new session: [4:7577789079189275374:2050]
2025-11-28T14:51:48.308922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789079189275374:2050] [node 3] ICS09 handshake done sender: [4:7577789079189275358:2118] self: [4:7577789079189275372:0] peer: [3:7577789076324062238:0] socket: 267 qp: -1
2025-11-28T14:51:48.308948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789079189275374:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:48.308993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:48.312337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789079189275351:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.319892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789078813471614:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.320113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789078813471582:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.320396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789079189275351:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.320944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789079189275351:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.321212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789079189275341:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.321165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471617:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.321300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:48.321355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789079189275341:2109] [node 6] ICP22 created new session: [4:7577789079189275377:2109]
2025-11-28T14:51:48.321409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789079189275377:2109] [node 6] ICS09 handshake done sender: [4:7577789079189275351:2114] self: [4:7577789079189275349:0] peer: [6:7577789078813471616:0] socket: 228 qp: -1
2025-11-28T14:51:48.321438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789078813471614:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:48.321451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789079189275377:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:48.321553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:48.321647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789078813471582:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.323048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:48.323428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789078813471582:2105] [node 4] ICP22 created new session: [6:7577789078813471618:2105]
2025-11-28T14:51:48.323017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789079189275352:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.323555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789078813471618:2105] [node 4] ICS09 handshake done sender: [6:7577789078813471614:2121] self: [6:7577789078813471616:0] peer: [4:7577789079189275349:0] socket: 232 qp: -1
2025-11-28T14:51:48.323665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789078813471618:2105] [node 4] ICS10 traffic start
2025-11-28T14:51:48.329328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:48.330962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076130188660:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.332268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789076130188634:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.332596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789079189275352:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.332686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789078813471582:2105] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:48.333092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789079189275352:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.333478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789079189275340:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.333599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:48.333582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076130188663:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64579
2025-11-28T14:51:48.333653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789079189275340:2108] [node 5] ICP22 created new session: [4:7577789079189275380:2108]
2025-11-28T14:51:48.333699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789079189275380:2108] [node 5] ICS09 handshake done sender: [4:7577789079189275352:2115] self: [4:7577789079189275350:0] peer: [5:7577789076130188662:0] socket: 256 qp: -1
2025-11-28T14:51:48.333738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789079189275380:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:48.333807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:48.333884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076130188660:2129] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:48.334406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789076130188634:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789076130188660:2129] held: no
2025-11-28T14:51:48.334525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789076130188634:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.334583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:48.334693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789076130188634:2108] [node 4] ICP22 created new session: [5:7577789076130188664:2108]
2025-11-28T14:51:48.334753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789076130188664:2108] [node 4] ICS09 handshake done sender: [5:7577789076130188660:2129] self: [5:7577789076130188662:0] peer: [4:7577789079189275350:0] socket: 258 qp: -1
2025-11-28T14:51:48.334800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789076130188664:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:48.334850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:48.337795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076324062247:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.359149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076130188661:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.359392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789071835221262:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.359714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076324062247:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.360242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076324062247:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.360500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789076324062235:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.360596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:48.360606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076130188667:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64579
2025-11-28T14:51:48.360676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789076324062235:2108] [node 5] ICP22 created new session: [3:7577789076324062273:2108]
2025-11-28T14:51:48.360736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789076324062273:2108] [node 5] ICS09 handshake done sender: [3:7577789076324062247:2115] self: [3:7577789076324062246:0] peer: [5:7577789076130188666:0] socket: 266 qp: -1
2025-11-28T14:51:48.360776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789076324062273:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:48.360816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:48.360951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076130188661:2130] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:48.361438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789071835221262:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.361976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:48.362039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789071835221262:2050] [node 3] ICP22 created new session: [5:7577789076130188668:2050]
2025-11-28T14:51:48.362088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789076130188668:2050] [node 3] ICS09 handshake done sender: [5:7577789076130188661:2130] self: [5:7577789076130188666:0] peer: [3:7577789076324062246:0] socket: 268 qp: -1
2025-11-28T14:51:48.364094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076324062254:2118] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.362172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789076130188668:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:48.364845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789078813471615:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.362227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:48.364987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789071835221262:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:48.365055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789074518504226:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.365299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076324062254:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.365977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789078813471621:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18733
2025-11-28T14:51:48.365978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076324062254:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:48.366160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789076324062237:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.366237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:48.366357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789076324062237:2109] [node 6] ICP22 created new session: [3:7577789076324062276:2109]
2025-11-28T14:51:48.366235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789078813471615:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:48.366426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789076324062276:2109] [node 6] ICS09 handshake done sender: [3:7577789076324062254:2118] self: [3:7577789076324062253:0] peer: [6:7577789078813471620:0] socket: 263 qp: -1
2025-11-28T14:51:48.366517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789076324062276:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:48.366577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789074518504226:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.366565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:48.366768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:48.366813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789074518504226:2050] [node 3] ICP22 created new session: [6:7577789078813471622:2050]
2025-11-28T14:51:48.366915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789078813471622:2050] [node 3] ICS09 handshake done sender: [6:7577789078813471615:2122] self: [6:7577789078813471620:0] peer: [3:7577789076324062253:0] socket: 265 qp: -1
2025-11-28T14:51:48.366993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789078813471622:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:48.367041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:48.367300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789074518504226:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:48.410128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:48.410305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789077600551162:2048] [node 1] ICS01 socket: 238 reason# EndOfStream
2025-11-28T14:51:48.410343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789073305583751:2048] [node 1] ICP30 unregister session Session# [2:7577789077600551162:2048] VirtualId# [2:7577789077600551157:0]
2025-11-28T14:51:48.410371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789077600551162:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.410466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:48.410506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:48.411344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:48.411540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789076324062262:2048] [node 1] ICS01 socket: 243 reason# EndOfStream
2025-11-28T14:51:48.411579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789072029094876:2048] [node 1] ICP30 unregister session Session# [3:7577789076324062262:2048] VirtualId# [3:7577789076324062258:0]
2025-11-28T14:51:48.411613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789076324062262:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.411697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:48.411775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:48.413539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:48.413745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789079189275364:2048] [node 1] ICS01 socket: 239 reason# EndOfStream
2025-11-28T14:51:48.413786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789074894307981:2048] [node 1] ICP30 unregister session Session# [4:7577789079189275364:2048] VirtualId# [4:7577789079189275360:0]
2025-11-28T14:51:48.413817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789079189275364:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.413931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:48.413996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:48.414423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:48.414643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789076130188649:2048] [node 1] ICS01 socket: 276 reason# EndOfStream
2025-11-28T14:51:48.414685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789071835221260:2048] [node 1] ICP30 unregister session Session# [5:7577789076130188649:2048] VirtualId# [5:7577789076130188647:0]
2025-11-28T14:51:48.414711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789076130188649:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.414802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:48.414882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:48.415315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:48.415492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789078813471602:2048] [node 1] ICS07 socket disconnect 272 reason# EndOfStream
2025-11-28T14:51:48.415522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789078813471602:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.415594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:48.415650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789078813471602:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:48.416442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789074518504224:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29047) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:48.416494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789078813471602:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:48.416553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789074518504224:2048] [node 1] ICP30 unregister session Session# [6:7577789078813471602:2048] VirtualId# [6:7577789078813471599:0]
2025-11-28T14:51:48.416604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:48.416649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789074518504224:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.416674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.426445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789074518504224:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:48.434924Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:48.435182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789076324062264:2049] [node 2] ICS07 socket disconnect 260 reason# EndOfStream
2025-11-28T14:51:48.435218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789076324062264:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.435379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:48.435459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789076324062264:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:48.437960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789072029094877:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:48.438003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789076324062264:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:48.438020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:48.438324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789079189275369:2049] [node 2] ICS01 socket: 250 reason# EndOfStream
2025-11-28T14:51:48.438355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789074894307982:2049] [node 2] ICP30 unregister session Session# [4:7577789079189275369:2049] VirtualId# [4:7577789079189275367:0]
2025-11-28T14:51:48.438377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789079189275369:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.438453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:48.438488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:48.439011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789072029094877:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:48.439053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789076324062264:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:48.439111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789072029094877:2049] [node 2] ICP30 unregister session Session# [3:7577789076324062264:2049] VirtualId# [3:7577789076324062259:0]
2025-11-28T14:51:48.439157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:48.439181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789072029094877:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.439192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.439367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:48.439567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789078813471607:2049] [node 2] ICS07 socket disconnect 285 reason# EndOfStream
2025-11-28T14:51:48.439604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789078813471607:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.439682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:48.439728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789078813471607:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:48.440348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789074518504225:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:48.440414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789078813471607:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:48.440340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:48.440486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789074518504225:2049] [node 2] ICP30 unregister session Session# [6:7577789078813471607:2049] VirtualId# [6:7577789078813471603:0]
2025-11-28T14:51:48.440555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789076130188654:2049] [node 2] ICS07 socket disconnect 278 reason# EndOfStream
2025-11-28T14:51:48.440556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:48.440591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789074518504225:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.440602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.440579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789076130188654:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.440654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:48.440699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789076130188654:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:48.441401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789071835221261:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:48.441436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789076130188654:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:48.441456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789071835221261:2049] [node 2] ICP30 unregister session Session# [5:7577789076130188654:2049] VirtualId# [5:7577789076130188650:0]
2025-11-28T14:51:48.441481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:48.441538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789071835221261:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.441569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.449963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789074518504225:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:48.451135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789071835221261:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:48.451999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:48.452226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789079189275374:2050] [node 3] ICS01 socket: 267 reason# EndOfStream
2025-11-28T14:51:48.452271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789074894307983:2050] [node 3] ICP30 unregister session Session# [4:7577789079189275374:2050] VirtualId# [4:7577789079189275372:0]
2025-11-28T14:51:48.452326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789079189275374:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.452466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:48.452522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:48.461866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:48.461916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:48.462136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789078813471622:2050] [node 3] ICS07 socket disconnect 265 reason# EndOfStream
2025-11-28T14:51:48.462177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789078813471622:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.462189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789076130188668:2050] [node 3] ICS07 socket disconnect 268 reason# EndOfStream
2025-11-28T14:51:48.462232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789076130188668:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.462349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:48.462334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:48.462407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789076130188668:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:48.462394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789078813471622:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:48.463038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789074518504226:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:48.463086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789078813471622:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:48.463118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789071835221262:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:48.463118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789074518504226:2050] [node 3] ICP30 unregister session Session# [6:7577789078813471622:2050] VirtualId# [6:7577789078813471620:0]
2025-11-28T14:51:48.463311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:48.463197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789076130188668:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:48.463386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789074518504226:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.463352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789071835221262:2050] [node 3] ICP30 unregister session Session# [5:7577789076130188668:2050] VirtualId# [5:7577789076130188666:0]
2025-11-28T14:51:48.463406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:48.463457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789071835221262:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.463432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.463472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.464767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:48.464941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789076130188664:2108] [node 4] ICS01 socket: 258 reason# EndOfStream
2025-11-28T14:51:48.464986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789076130188634:2108] [node 4] ICP30 unregister session Session# [5:7577789076130188664:2108] VirtualId# [5:7577789076130188662:0]
2025-11-28T14:51:48.465013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789076130188664:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.465076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:48.465115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:48.465699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:48.465893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789078813471618:2105] [node 4] ICS07 socket disconnect 232 reason# EndOfStream
2025-11-28T14:51:48.465918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789078813471618:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.465988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:48.466051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789078813471618:2105] [node 4] ICS15 start handshake
2025-11-28T14:51:48.466968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789078813471582:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61861) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:48.467061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789078813471618:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:48.467139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789078813471582:2105] [node 4] ICP30 unregister session Session# [6:7577789078813471618:2105] VirtualId# [6:7577789078813471616:0]
2025-11-28T14:51:48.467193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:48.467220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789078813471582:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.467248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.473501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789074518504226:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:48.477249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789078813471582:2105] [node 4] ICP33 wake up from error state
2025-11-28T14:51:48.494631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:48.494859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789078813471612:2104] [node 5] ICS07 socket disconnect 242 reason# EndOfStream
2025-11-28T14:51:48.494892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789078813471612:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:48.495243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:48.495316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789078813471612:2104] [node 5] ICS15 start handshake
2025-11-28T14:51:48.496166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789078813471579:2104] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17404) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:48.496845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789078813471612:2104] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:48.496906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789078813471579:2104] [node 5] ICP30 unregister session Session# [6:7577789078813471612:2104] VirtualId# [6:7577789078813471610:0]
2025-11-28T14:51:48.496980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:48.498548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789078813471579:2104] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17404) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:48.498566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17404) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:48.556784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789079794742900:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:48.556989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789079794742901:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:48.582228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789079873551100:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:48.582648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789079873551099:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:48.604321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789076357964055:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:48.604412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789076357964054:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:48.626555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789078625460942:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:48.626574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789078625460941:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:48.626692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789078625460943:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:48.648189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789076365648671:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:48.648237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789076365648672:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:48.648264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789076365648673:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:48.673980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789079147181277:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:48.674022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789079147181278:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:48.674048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789079147181279:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:48.756748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789079794742961:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:48.768400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789079794742964:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:48.798642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789076357964117:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:48.802174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789079147181338:2105] [node 5] ICP01 ready to work
2025-11-28T14:51:48.779675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789079794742965:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:48.802847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789076365648735:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:48.802545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789079873551162:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:48.800937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789076357964118:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:48.804403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789079873551164:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:48.804929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789078625461022:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:48.804979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789079794742900:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44345914ns) {Configure:104(44337240ns) {ProcessConfigured:135(10447965ns) {StartInitialHandshake:157(10444423ns) {DropHandshakes:470(2714ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(676ns)} GenerateSessionVirtualId:988(505ns) PrepareNewSessionHandshake:477(6635ns) {SwitchToState:179(4683ns)}}}}}
2025-11-28T14:51:48.804569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789079147181339:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:48.804614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789076365648736:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:48.802263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789076357964119:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:48.804920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789079873551163:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:48.807354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789078625461023:2118] [node 6] ICP01 ready to work
2025-11-28T14:51:48.811491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789076357964054:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(55025009ns) {Configure:104(55019021ns) {ProcessConfigured:135(407091ns) {StartInitialHandshake:157(405110ns) {DropHandshakes:470(1948ns) {DropIncomingHandshake:442(514ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(995ns) PrepareNewSessionHandshake:477(1473ns) {SwitchToState:179(392ns)}}}}}
2025-11-28T14:51:48.812162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789078625460941:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49710306ns) {Configure:104(49706813ns) {ProcessConfigured:135(673334ns) {StartInitialHandshake:157(671480ns) {DropHandshakes:470(1180ns) {DropIncomingHandshake:442(390ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(1592ns) {SwitchToState:179(674ns)}}}}}
2025-11-28T14:51:48.812668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789078625460942:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48981114ns) {Configure:104(48976908ns) {ProcessConfigured:135(551895ns) {StartInitialHandshake:157(550271ns) {DropHandshakes:470(1077ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(486ns) PrepareNewSessionHandshake:477(142528ns) {SwitchToState:179(925ns)}}}}}
2025-11-28T14:51:48.812108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789076357964055:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(54392648ns) {Configure:104(54388439ns) {ProcessConfigured:135(494657ns) {StartInitialHandshake:157(492373ns) {DropHandshakes:470(2969ns) {DropIncomingHandshake:442(852ns) DropOutgoingHandshake:457(462ns)} GenerateSessionVirtualId:988(1078ns) PrepareNewSessionHandshake:477(1445ns) {SwitchToState:179(472ns)}}}}}
2025-11-28T14:51:48.812892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789079794742901:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48276675ns) {Configure:104(48264043ns) {ProcessConfigured:135(773542ns) {StartInitialHandshake:157(771299ns) {DropHandshakes:470(1204ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(428ns) PrepareNewSessionHandshake:477(1796ns) {SwitchToState:179(966ns)}}}}}
2025-11-28T14:51:48.812931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789078625460943:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43232585ns) {Configure:104(43222952ns) {ProcessConfigured:135(586529ns) {StartInitialHandshake:157(581849ns) {DropHandshakes:470(1412ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(801ns) PrepareNewSessionHandshake:477(131125ns) {SwitchToState:179(1609ns)}}}}}
2025-11-28T14:51:48.813635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789079873551099:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(64461865ns) {Configure:104(64448443ns) {ProcessConfigured:135(277828ns) {StartInitialHandshake:157(276402ns) {DropHandshakes:470(883ns) {DropIncomingHandshake:442(233ns) DropOutgoingHandshake:457(163ns)} GenerateSessionVirtualId:988(380ns) PrepareNewSessionHandshake:477(1201ns) {SwitchToState:179(569ns)}}}}}
2025-11-28T14:51:48.819492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789076365648672:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16535399ns) {Configure:104(16529116ns) {ProcessConfigured:135(3900829ns) {StartInitialHandshake:157(3896057ns) {DropHandshakes:470(2816ns) {DropIncomingHandshake:442(949ns) DropOutgoingHandshake:457(490ns)} GenerateSessionVirtualId:988(2817ns) PrepareNewSessionHandshake:477(2451ns) {SwitchToState:179(1010ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.830071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789079147181353:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.831714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789079873551176:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55578
2025-11-28T14:51:48.832456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789078625461037:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35952
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.844740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076365648748:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50509
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.845389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789079147181354:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
2025-11-28T14:51:48.841510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789079794742980:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21833
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.841866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789079873551175:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55578
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.846300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076365648748:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.846760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789079794742979:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21833
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.849485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789079147181339:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21068337ns) {Configure:104(21060601ns) {ProcessConfigured:135(4351240ns) {StartInitialHandshake:157(4348212ns) {DropHandshakes:470(2183ns) {DropIncomingHandshake:442(634ns) DropOutgoingHandshake:457(453ns)} GenerateSessionVirtualId:988(1018ns) PrepareNewSessionHandshake:477(3721ns) {SwitchToState:179(1759ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.849493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789076365648736:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21999861ns) {Configure:104(21990112ns) {ProcessConfigured:135(3020579ns) {StartInitialHandshake:157(3018409ns) {DropHandshakes:470(2552ns) {DropIncomingHandshake:442(405ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:477(1661ns) {SwitchToState:179(396ns)}}}}}
2025-11-28T14:51:48.850629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789079873551175:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.846981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789079794742983:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21833
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.850628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789079147181353:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.851897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789079873551178:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55578
2025-11-28T14:51:48.854035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076357964131:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27708
2025-11-28T14:51:48.852706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789079794742981:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21833
2025-11-28T14:51:48.852754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789079147181354:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.854533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076365648744:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.853344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789079873551099:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.854523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076357964135:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27708
2025-11-28T14:51:48.853313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742979:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.854569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789076365648744:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.854543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076357964134:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27708
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.853386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789079147181338:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.863016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076365648744:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:48.853704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789079147181277:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.853361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742980:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.856000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076357964133:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27708
2025-11-28T14:51:48.854203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789079147181355:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
2025-11-28T14:51:48.853506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789079794742982:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21833
2025-11-28T14:51:48.868206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789079873551171:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.862999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789079147181355:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.860748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076357964132:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27708
2025-11-28T14:51:48.853762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742983:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.868193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789076365648735:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.868243Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789079873551171:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.867446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789079147181360:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
2025-11-28T14:51:48.854167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742971:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.868692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:48.866695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076357964131:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.872847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789079873551171:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:48.867748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789079147181278:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.867601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076357964135:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.869334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789076365648735:2110] [node 6] ICP22 created new session: [5:7577789076365648753:2110]
2025-11-28T14:51:48.854191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789079794742971:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.868689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789079147181354:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:48.871345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789076365648753:2110] [node 6] ICS09 handshake done sender: [5:7577789076365648744:2114] self: [5:7577789076365648740:0] peer: [6:7577789079147181359:0] socket: 249 qp: -1
2025-11-28T14:51:48.868075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076357964133:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.868149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076357964132:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.870816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789076357964054:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.872824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789076365648753:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:48.854276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742967:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.873373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789079147181338:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.873405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:48.854308Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789079794742967:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.875156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:48.860787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742982:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.875196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789079147181277:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789079147181353:2114] held: no
2025-11-28T14:51:48.866754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789079794742971:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:48.868548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789079794742964:2109] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:48.877893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789079873551099:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789079873551175:2114] held: no
::ffff:127.1.0.0
2025-11-28T14:51:48.875420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789079147181338:2105] [node 5] ICP22 created new session: [6:7577789079147181363:2105]
2025-11-28T14:51:48.879648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789079873551162:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.875987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789079147181363:2105] [node 5] ICS09 handshake done sender: [6:7577789079147181354:2115] self: [6:7577789079147181359:0] peer: [5:7577789076365648740:0] socket: 252 qp: -1
2025-11-28T14:51:48.879858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:48.868742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789079794742967:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:48.876196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789079147181363:2105] [node 5] ICS10 traffic start
2025-11-28T14:51:48.868811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:48.880104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789079873551162:2109] [node 6] ICP22 created new session: [2:7577789079873551185:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.877125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.884734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789079873551185:2109] [node 6] ICS09 handshake done sender: [2:7577789079873551171:2112] self: [2:7577789079873551169:0] peer: [6:7577789079147181361:0] socket: 264 qp: -1
2025-11-28T14:51:48.869898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789079794742964:2109] [node 6] ICP22 created new session: [1:7577789079794742990:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.885252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789079873551185:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:48.888862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076357964142:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27708
2025-11-28T14:51:48.871091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789079794742900:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.877166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789079147181362:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
2025-11-28T14:51:48.888991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076365648756:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50509
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.886131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789079873551182:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55578
2025-11-28T14:51:48.879654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789079147181353:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:48.872113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789079794742990:2109] [node 6] ICS09 handshake done sender: [1:7577789079794742971:2112] self: [1:7577789079794742969:0] peer: [6:7577789079147181358:0] socket: 236 qp: -1
2025-11-28T14:51:48.886323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789079873551184:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55578
2025-11-28T14:51:48.880305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789079147181338:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:48.872150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789079794742990:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:48.889994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789078625461038:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35952
2025-11-28T14:51:48.880573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789079147181277:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.886380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:48.890578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789078625461039:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35952
2025-11-28T14:51:48.872190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.888229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789079873551184:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.891313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789078625461038:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.884631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:48.873108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:48.889904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789079873551175:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.891599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789078625460941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.885244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789079147181277:2048] [node 1] ICP22 created new session: [6:7577789079147181368:2048]
2025-11-28T14:51:48.891097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789079873551099:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.873413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789079794742900:2048] [node 2] ICP22 created new session: [1:7577789079794742992:2048]
2025-11-28T14:51:48.892104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789078625461039:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.894338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789078625461040:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35952
2025-11-28T14:51:48.893484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076357964131:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:48.874769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789079794742992:2048] [node 2] ICS09 handshake done sender: [1:7577789079794742967:2110] self: [1:7577789079794742966:0] peer: [2:7577789079873551181:0] socket: 234 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.891138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:48.886805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789079147181368:2048] [node 1] ICS09 handshake done sender: [6:7577789079147181353:2114] self: [6:7577789079147181358:0] peer: [1:7577789079794742969:0] socket: 244 qp: -1
2025-11-28T14:51:48.891294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789079873551099:2048] [node 1] ICP22 created new session: [2:7577789079873551190:2048]
2025-11-28T14:51:48.887228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789079147181368:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:48.895721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789076357964054:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.875383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789079794742992:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:48.895460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076365648757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50509
2025-11-28T14:51:48.875425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:48.887279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:48.891699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789079873551190:2048] [node 1] ICS09 handshake done sender: [2:7577789079873551175:2114] self: [2:7577789079873551181:0] peer: [1:7577789079794742966:0] socket: 248 qp: -1
2025-11-28T14:51:48.876271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742970:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.896624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789078625460941:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789078625461038:2134] held: no
2025-11-28T14:51:48.876329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789079794742970:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.895875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:48.891842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789079873551190:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:48.889996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789079147181365:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.878054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789079794742970:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:48.896691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789076357964054:2048] [node 1] ICP22 created new session: [3:7577789076357964143:2048]
2025-11-28T14:51:48.879857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789079794742901:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.891587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789079147181355:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:48.891877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:48.880089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:48.898986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789078625461043:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35952
2025-11-28T14:51:48.892046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789079147181278:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.897490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789076357964143:2048] [node 1] ICS09 handshake done sender: [3:7577789076357964131:2114] self: [3:7577789076357964138:0] peer: [1:7577789079794742968:0] socket: 251 qp: -1
2025-11-28T14:51:48.880130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789079794742901:2049] [node 3] ICP22 created new session: [1:7577789079794742997:2049]
2025-11-28T14:51:48.893273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789079873551187:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55578
2025-11-28T14:51:48.898882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076365648757:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.892088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:48.897869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789076357964143:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:48.899027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789076365648672:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.884651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789079794742997:2049] [node 3] ICS09 handshake done sender: [1:7577789079794742970:2111] self: [1:7577789079794742968:0] peer: [3:7577789076357964138:0] socket: 235 qp: -1
2025-11-28T14:51:48.897951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:48.892319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789079147181278:2049] [node 2] ICP22 created new session: [6:7577789079147181370:2049]
2025-11-28T14:51:48.886199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789079794742997:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:48.893101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789079147181370:2049] [node 2] ICS09 handshake done sender: [6:7577789079147181355:2116] self: [6:7577789079147181361:0] peer: [2:7577789079873551169:0] socket: 270 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.898813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789079873551162:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:48.899789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789078625461038:2134] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:48.893425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789079147181370:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:48.886243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:48.899870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789079873551180:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.900982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789078625460941:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.892468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742973:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.894195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:48.901019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:48.899916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789079873551180:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.900910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076365648759:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50509
2025-11-28T14:51:48.892662Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789079794742973:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.902032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789078625460941:2048] [node 1] ICP22 created new session: [4:7577789078625461045:2048]
2025-11-28T14:51:48.894479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789079794742973:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:48.902082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789078625461045:2048] [node 1] ICS09 handshake done sender: [4:7577789078625461038:2134] self: [4:7577789078625461041:0] peer: [1:7577789079794742972:0] socket: 239 qp: -1
2025-11-28T14:51:48.902356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789079873551180:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:48.902285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789078625461045:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:48.902412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:48.895152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789079794742961:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.902852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076365648759:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.895212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:48.903074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789079873551163:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.896199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789079794742961:2107] [node 4] ICP22 created new session: [1:7577789079794743000:2107]
2025-11-28T14:51:48.903305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:48.903571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789079873551163:2108] [node 5] ICP22 created new session: [2:7577789079873551194:2108]
2025-11-28T14:51:48.903391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789076365648736:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.897514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789079794743000:2107] [node 4] ICS09 handshake done sender: [1:7577789079794742973:2113] self: [1:7577789079794742972:0] peer: [4:7577789078625461041:0] socket: 254 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.898258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789079794743000:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:48.903633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789079873551194:2108] [node 5] ICS09 handshake done sender: [2:7577789079873551180:2117] self: [2:7577789079873551179:0] peer: [5:7577789076365648758:0] socket: 292 qp: -1
2025-11-28T14:51:48.903677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789079873551194:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:48.898712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:48.903715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:48.903820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076365648760:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50509
2025-11-28T14:51:48.904044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789078625461033:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.903821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789076365648672:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6873) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789076365648757:2118] held: no
2025-11-28T14:51:48.904076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789078625461033:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.904199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076365648757:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:48.904989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789076365648672:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.905025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:48.905907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789078625461033:2131] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:48.905748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789076365648672:2049] [node 2] ICP22 created new session: [5:7577789076365648762:2049]
2025-11-28T14:51:48.905908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789076365648762:2049] [node 2] ICS09 handshake done sender: [5:7577789076365648757:2118] self: [5:7577789076365648758:0] peer: [2:7577789079873551179:0] socket: 293 qp: -1
2025-11-28T14:51:48.905934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789076365648762:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:48.905966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:48.906201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789078625461022:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.906248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:48.906303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789078625461022:2108] [node 5] ICP22 created new session: [4:7577789078625461048:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.906399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789078625461048:2108] [node 5] ICS09 handshake done sender: [4:7577789078625461033:2131] self: [4:7577789078625461030:0] peer: [5:7577789076365648761:0] socket: 256 qp: -1
2025-11-28T14:51:48.906405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076365648763:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50509
2025-11-28T14:51:48.906436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789078625461048:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:48.906495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.906632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076365648759:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.906791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789078625461050:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35952
2025-11-28T14:51:48.906853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789076365648736:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.906911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:48.906987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789076365648736:2108] [node 4] ICP22 created new session: [5:7577789076365648766:2108]
2025-11-28T14:51:48.906967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789079147181372:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
2025-11-28T14:51:48.907059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789076365648766:2108] [node 4] ICS09 handshake done sender: [5:7577789076365648759:2119] self: [5:7577789076365648761:0] peer: [4:7577789078625461030:0] socket: 267 qp: -1
2025-11-28T14:51:48.907095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789076365648766:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:48.907143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:48.908172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789079147181372:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.908325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789079147181339:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.908553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789078625461036:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.908587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789078625461036:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:48.909748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789078625461022:2108] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.909973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789078625461036:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:48.910027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789079147181374:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
2025-11-28T14:51:48.910895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789078625461023:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.911967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789079873551168:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.911912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789079147181372:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:48.911644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:48.911925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789078625461023:2118] [node 6] ICP22 created new session: [4:7577789078625461053:2118]
2025-11-28T14:51:48.911974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789078625461053:2118] [node 6] ICS09 handshake done sender: [4:7577789078625461036:2132] self: [4:7577789078625461035:0] peer: [6:7577789079147181373:0] socket: 297 qp: -1
2025-11-28T14:51:48.912014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789078625461053:2118] [node 6] ICS10 traffic start
2025-11-28T14:51:48.912253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789079147181339:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.912234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:48.912304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:48.917152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789079147181339:2107] [node 4] ICP22 created new session: [6:7577789079147181375:2107]
2025-11-28T14:51:48.919937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789079147181375:2107] [node 4] ICS09 handshake done sender: [6:7577789079147181372:2121] self: [6:7577789079147181373:0] peer: [4:7577789078625461035:0] socket: 299 qp: -1
2025-11-28T14:51:48.919973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789079147181375:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:48.920162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:48.920157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076357964134:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.920332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789076357964055:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.920439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789079147181339:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:48.920698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789079873551168:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.921145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789079873551168:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.921250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789076357964146:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27708
2025-11-28T14:51:48.921426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789079873551100:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.921479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:48.921520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789079873551100:2049] [node 3] ICP22 created new session: [2:7577789079873551197:2049]
2025-11-28T14:51:48.921637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789079873551197:2049] [node 3] ICS09 handshake done sender: [2:7577789079873551168:2111] self: [2:7577789079873551165:0] peer: [3:7577789076357964145:0] socket: 245 qp: -1
2025-11-28T14:51:48.921718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789079873551197:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:48.921728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076357964134:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:48.921762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:48.921933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789076357964055:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.921993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:48.922024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789076357964055:2049] [node 2] ICP22 created new session: [3:7577789076357964147:2049]
2025-11-28T14:51:48.922046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789076357964147:2049] [node 2] ICS09 handshake done sender: [3:7577789076357964134:2117] self: [3:7577789076357964145:0] peer: [2:7577789079873551165:0] socket: 268 qp: -1
2025-11-28T14:51:48.922067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789076357964147:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:48.922095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:48.923052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076357964122:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.931545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742981:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.931748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789079794742901:2049] [node 3] ICP18 handshake (actor [1:7577789079794742981:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.932903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076365648768:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50509
2025-11-28T14:51:48.935247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076357964123:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.942480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789079873551176:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.942701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789079873551100:2049] [node 3] ICP18 handshake (actor [2:7577789079873551176:2115]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.943332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789079147181377:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
2025-11-28T14:51:48.944715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789078625461028:2129] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.952403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789079873551178:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.970618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076357964126:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.978435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789078625461037:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:48.978645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789078625460943:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:48.978976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076357964126:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:48.979552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076357964126:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:48.979659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789078625461056:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35952
2025-11-28T14:51:48.979786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789076357964117:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:48.979904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:48.979919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789078625461037:2133] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:48.979971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789076357964117:2107] [node 4] ICP22 created new session: [3:7577789076357964152:2107]
2025-11-28T14:51:48.980101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789076357964152:2107] [node 4] ICS09 handshake done sender: [3:7577789076357964126:2112] self: [3:7577789076357964124:0] peer: [4:7577789078625461055:0] socket: 257 qp: -1
2025-11-28T14:51:48.980063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789078625460943:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:48.980135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:48.980235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789076357964152:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:48.980274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789078625460943:2050] [node 3] ICP22 created new session: [4:7577789078625461057:2050]
2025-11-28T14:51:48.980282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:48.980338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789078625461057:2050] [node 3] ICS09 handshake done sender: [4:7577789078625461037:2133] self: [4:7577789078625461055:0] peer: [3:7577789076357964124:0] socket: 273 qp: -1
2025-11-28T14:51:48.980388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789078625461057:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:48.980426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:48.990597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789079794742976:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:48.999746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076365648756:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.000010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789076365648671:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.000206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789079794742976:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.000702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789079794742976:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.000885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789076365648770:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50509
2025-11-28T14:51:49.000935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789079794742965:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.001064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:49.001107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789079794742965:2108] [node 5] ICP22 created new session: [1:7577789084089710299:2108]
2025-11-28T14:51:49.001167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789084089710299:2108] [node 5] ICS09 handshake done sender: [1:7577789079794742976:2114] self: [1:7577789079794742974:0] peer: [5:7577789076365648769:0] socket: 283 qp: -1
2025-11-28T14:51:49.001198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789084089710299:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:49.001235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:49.001272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076365648756:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:49.001553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789076365648671:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18773) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789076365648756:2117] held: no
2025-11-28T14:51:49.001706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789076365648671:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.001735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:49.002366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789076365648671:2048] [node 1] ICP22 created new session: [5:7577789080660616067:2048]
2025-11-28T14:51:49.002426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789080660616067:2048] [node 1] ICS09 handshake done sender: [5:7577789076365648756:2117] self: [5:7577789076365648769:0] peer: [1:7577789079794742974:0] socket: 286 qp: -1
2025-11-28T14:51:49.002471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789080660616067:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:49.002549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:49.018310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789079873551172:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.026330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789078625461040:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.026541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789078625460942:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.026773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789079873551172:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.027279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789079873551172:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.027485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789079873551164:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.027571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:49.027530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789082920428356:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35952
2025-11-28T14:51:49.027631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789079873551164:2107] [node 4] ICP22 created new session: [2:7577789084168518497:2107]
2025-11-28T14:51:49.027714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789084168518497:2107] [node 4] ICS09 handshake done sender: [2:7577789079873551172:2113] self: [2:7577789079873551170:0] peer: [4:7577789082920428355:0] socket: 290 qp: -1
2025-11-28T14:51:49.027756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789084168518497:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:49.027814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:49.027821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789078625461040:2136] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:49.028148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789078625460942:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6873) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789078625461040:2136] held: no
2025-11-28T14:51:49.028336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789078625460942:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.028407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:49.028500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789078625460942:2049] [node 2] ICP22 created new session: [4:7577789082920428357:2049]
2025-11-28T14:51:49.028555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789082920428357:2049] [node 2] ICS09 handshake done sender: [4:7577789078625461040:2136] self: [4:7577789082920428355:0] peer: [2:7577789079873551170:0] socket: 291 qp: -1
2025-11-28T14:51:49.028658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789082920428357:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:49.028728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:49.031536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076357964127:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.040071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789076365648768:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.040268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789076365648673:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.040502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076357964127:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.041103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076357964127:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.041356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789076357964118:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.041290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789080660616070:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50509
2025-11-28T14:51:49.041509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:49.041591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789076357964118:2108] [node 5] ICP22 created new session: [3:7577789080652931451:2108]
2025-11-28T14:51:49.041605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789076365648768:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:49.041716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789080652931451:2108] [node 5] ICS09 handshake done sender: [3:7577789076357964127:2113] self: [3:7577789076357964125:0] peer: [5:7577789080660616069:0] socket: 243 qp: -1
2025-11-28T14:51:49.041789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789076365648673:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.041806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789080652931451:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:49.041951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:49.041966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:49.042000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789076365648673:2050] [node 3] ICP22 created new session: [5:7577789080660616071:2050]
2025-11-28T14:51:49.042102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789080660616071:2050] [node 3] ICS09 handshake done sender: [5:7577789076365648768:2122] self: [5:7577789080660616069:0] peer: [3:7577789076357964125:0] socket: 262 qp: -1
2025-11-28T14:51:49.042208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789080660616071:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:49.042239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:49.043240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789076365648673:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:49.045142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789076357964137:2119] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.045944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789079147181377:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.046168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789079147181279:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.046427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789076357964137:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.046908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789076357964137:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:49.047151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789076357964119:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.047390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:49.047456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789076357964119:2109] [node 6] ICP22 created new session: [3:7577789080652931454:2109]
2025-11-28T14:51:49.047513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789080652931454:2109] [node 6] ICS09 handshake done sender: [3:7577789076357964137:2119] self: [3:7577789076357964136:0] peer: [6:7577789083442148674:0] socket: 246 qp: -1
2025-11-28T14:51:49.047551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789080652931454:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.047720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:49.047736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789083442148675:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54331
2025-11-28T14:51:49.048782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789079147181279:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15468) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789079147181377:2123] held: no
2025-11-28T14:51:49.048995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789079147181377:2123] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:49.049159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789079147181279:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.049226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:49.049280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789079147181279:2050] [node 3] ICP22 created new session: [6:7577789083442148676:2050]
2025-11-28T14:51:49.049330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789083442148676:2050] [node 3] ICS09 handshake done sender: [6:7577789079147181377:2123] self: [6:7577789083442148674:0] peer: [3:7577789076357964136:0] socket: 298 qp: -1
2025-11-28T14:51:49.049368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789083442148676:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:49.049404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:49.061776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:49.061943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789079873551190:2048] [node 1] ICS01 socket: 248 reason# EndOfStream
2025-11-28T14:51:49.061976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789079873551099:2048] [node 1] ICP30 unregister session Session# [2:7577789079873551190:2048] VirtualId# [2:7577789079873551181:0]
2025-11-28T14:51:49.061999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789079873551190:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.062063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.062109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.062610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:49.062807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789076357964143:2048] [node 1] ICS07 socket disconnect 251 reason# EndOfStream
2025-11-28T14:51:49.062848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789076357964143:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.062932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.062985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789076357964143:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:49.063692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789076357964054:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18773) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.063776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789076357964143:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.063816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789076357964054:2048] [node 1] ICP30 unregister session Session# [3:7577789076357964143:2048] VirtualId# [3:7577789076357964138:0]
2025-11-28T14:51:49.063873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.063902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789076357964054:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18773) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.063917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18773) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.064812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:49.065009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789078625461045:2048] [node 1] ICS01 socket: 239 reason# EndOfStream
2025-11-28T14:51:49.065051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789078625460941:2048] [node 1] ICP30 unregister session Session# [4:7577789078625461045:2048] VirtualId# [4:7577789078625461041:0]
2025-11-28T14:51:49.065081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789078625461045:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.065137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.065168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.065745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:49.065946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789080660616067:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-11-28T14:51:49.065981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789076365648671:2048] [node 1] ICP30 unregister session Session# [5:7577789080660616067:2048] VirtualId# [5:7577789076365648769:0]
2025-11-28T14:51:49.066014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789080660616067:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.066077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.066120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.066752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:49.066951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789079147181368:2048] [node 1] ICS01 socket: 244 reason# EndOfStream
2025-11-28T14:51:49.066994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789079147181277:2048] [node 1] ICP30 unregister session Session# [6:7577789079147181368:2048] VirtualId# [6:7577789079147181358:0]
2025-11-28T14:51:49.067021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789079147181368:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.067088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.067150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.073593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789076357964054:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:49.084605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:49.084852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789076357964147:2049] [node 2] ICS07 socket disconnect 268 reason# EndOfStream
2025-11-28T14:51:49.084891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789076357964147:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.085015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:49.085079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789076357964147:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:49.085834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789076357964055:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6873) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.085954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789076357964147:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.086048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789076357964055:2049] [node 2] ICP30 unregister session Session# [3:7577789076357964147:2049] VirtualId# [3:7577789076357964145:0]
2025-11-28T14:51:49.086110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:49.086150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789076357964055:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6873) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.086166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6873) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.086732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:49.086912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789082920428357:2049] [node 2] ICS01 socket: 291 reason# EndOfStream
2025-11-28T14:51:49.086959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789078625460942:2049] [node 2] ICP30 unregister session Session# [4:7577789082920428357:2049] VirtualId# [4:7577789082920428355:0]
2025-11-28T14:51:49.086988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789082920428357:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.087064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:49.087105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:49.087358Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:49.087480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789076365648762:2049] [node 2] ICS01 socket: 293 reason# EndOfStream
2025-11-28T14:51:49.087505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789076365648672:2049] [node 2] ICP30 unregister session Session# [5:7577789076365648762:2049] VirtualId# [5:7577789076365648758:0]
2025-11-28T14:51:49.087522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789076365648762:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.087580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:49.087611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:49.087940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:49.088141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789079147181370:2049] [node 2] ICS07 socket disconnect 270 reason# EndOfStream
2025-11-28T14:51:49.088179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789079147181370:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.088250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:49.088300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789079147181370:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:49.089003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789079147181278:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6873) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.089107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789079147181370:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.089151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789079147181278:2049] [node 2] ICP30 unregister session Session# [6:7577789079147181370:2049] VirtualId# [6:7577789079147181361:0]
2025-11-28T14:51:49.089197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:49.089241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789079147181278:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6873) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.089272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6873) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.097050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:49.097263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789078625461057:2050] [node 3] ICS07 socket disconnect 273 reason# EndOfStream
2025-11-28T14:51:49.097313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789078625461057:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.097404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:49.097458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789078625461057:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:49.097783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:49.097994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789080660616071:2050] [node 3] ICS01 socket: 262 reason# EndOfStream
2025-11-28T14:51:49.098048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789076365648673:2050] [node 3] ICP30 unregister session Session# [5:7577789080660616071:2050] VirtualId# [5:7577789080660616069:0]
2025-11-28T14:51:49.098083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789080660616071:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.098163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:49.098240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:49.098254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789078625460943:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15468) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.098317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789078625461057:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.098392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789078625460943:2050] [node 3] ICP30 unregister session Session# [4:7577789078625461057:2050] VirtualId# [4:7577789078625461055:0]
2025-11-28T14:51:49.098448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:49.098483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789078625460943:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.098498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.098662Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:49.098874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789083442148676:2050] [node 3] ICS01 socket: 298 reason# EndOfStream
2025-11-28T14:51:49.098916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789079147181279:2050] [node 3] ICP30 unregister session Session# [6:7577789083442148676:2050] VirtualId# [6:7577789083442148674:0]
2025-11-28T14:51:49.098962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789083442148676:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.099049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:49.099104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:49.099155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789079147181278:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:49.124557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:49.124803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789076365648766:2108] [node 4] ICS07 socket disconnect 267 reason# EndOfStream
2025-11-28T14:51:49.124837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789076365648766:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.124917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:49.124962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789076365648766:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:49.125650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:49.125621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789076365648736:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28812) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.125680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789076365648766:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.125752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789076365648736:2108] [node 4] ICP30 unregister session Session# [5:7577789076365648766:2108] VirtualId# [5:7577789076365648761:0]
2025-11-28T14:51:49.125799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:49.125837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789076365648736:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28812) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.125855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28812) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.125954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789079147181375:2107] [node 4] ICS07 socket disconnect 299 reason# EndOfStream
2025-11-28T14:51:49.125997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789079147181375:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.126068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:49.126138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789079147181375:2107] [node 4] ICS15 start handshake
2025-11-28T14:51:49.126669Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789079147181339:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28812) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.126729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789079147181375:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.126781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789079147181339:2107] [node 4] ICP30 unregister session Session# [6:7577789079147181375:2107] VirtualId# [6:7577789079147181373:0]
2025-11-28T14:51:49.126831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:49.126867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789079147181339:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28812) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.126888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28812) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.137119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789079147181339:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:51:49.168193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:49.168458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789079147181363:2105] [node 5] ICS07 socket disconnect 252 reason# EndOfStream
2025-11-28T14:51:49.168504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789079147181363:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.168616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:49.168672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789079147181363:2105] [node 5] ICS15 start handshake
2025-11-28T14:51:49.169390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789079147181338:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19909) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.169490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789079147181363:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.169540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789079147181338:2105] [node 5] ICP30 unregister session Session# [6:7577789079147181363:2105] VirtualId# [6:7577789079147181359:0]
2025-11-28T14:51:49.169582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:49.169610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789079147181338:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19909) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.169627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19909) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:49.230334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789080256104329:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:49.230343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789080256104328:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:49.249709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789080416007893:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:49.249994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789080416007894:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:49.270650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789080605590107:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:49.270768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789080605590108:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:49.293503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789084169835460:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:49.293549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789084169835461:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:49.293723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789084169835462:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:49.317584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789081423225376:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:49.317602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789081423225377:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:49.317891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789081423225378:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:49.339027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789082073009574:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:49.339066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789082073009575:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:49.339168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789082073009576:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:49.431997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789080256104391:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:49.438150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789081423225440:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:49.438364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789080416007957:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:49.436043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789080256104392:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:49.441481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789084169835523:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:49.441538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789080605590189:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:49.446671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789082073009637:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:49.441419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789080416007956:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:49.442000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789080605590190:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:49.441008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789080256104393:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:49.446646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789081423225441:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:49.441484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789084169835524:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:49.441982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789080416007958:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:49.446630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789082073009639:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:49.447314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789080605590191:2119] [node 6] ICP01 ready to work
2025-11-28T14:51:49.456203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789080256104328:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(62334210ns) {Configure:104(62324775ns) {ProcessConfigured:135(975204ns) {StartInitialHandshake:157(972925ns) {DropHandshakes:470(1018ns) {DropIncomingHandshake:442(258ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:477(1756ns) {SwitchToState:179(597ns)}}}}}
2025-11-28T14:51:49.456708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789080256104329:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(58206871ns) {Configure:104(58199364ns) {ProcessConfigured:135(409039ns) {StartInitialHandshake:157(406102ns) {DropHandshakes:470(1575ns) {DropIncomingHandshake:442(523ns) DropOutgoingHandshake:457(295ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(2320ns) {SwitchToState:179(578ns)}}}}}
2025-11-28T14:51:49.458567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789082073009576:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31524808ns) {Configure:104(31520240ns) {ProcessConfigured:135(766021ns) {StartInitialHandshake:157(763390ns) {DropHandshakes:470(1234ns) {DropIncomingHandshake:442(399ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(457ns) PrepareNewSessionHandshake:477(1723ns) {SwitchToState:179(223ns)}}}}}
2025-11-28T14:51:49.458507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789081423225377:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44422428ns) {Configure:104(44410017ns) {ProcessConfigured:135(427467ns) {StartInitialHandshake:157(425921ns) {DropHandshakes:470(959ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(325ns) PrepareNewSessionHandshake:477(1580ns) {SwitchToState:179(890ns)}}}}}
2025-11-28T14:51:49.458968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789080416007893:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38571346ns) {Configure:104(38561060ns) {ProcessConfigured:135(864767ns) {StartInitialHandshake:157(861730ns) {DropHandshakes:470(89959ns) {DropIncomingHandshake:442(610ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(669ns) PrepareNewSessionHandshake:477(1490ns) {SwitchToState:179(497ns)}}}}}
2025-11-28T14:51:49.459526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789082073009574:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18412626ns) {Configure:104(18171764ns) {ProcessConfigured:135(711364ns) {StartInitialHandshake:157(710491ns) {DropHandshakes:470(444ns) {DropIncomingHandshake:442(122ns) DropOutgoingHandshake:457(42ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(1015ns) {SwitchToState:179(80ns)}}}}}
2025-11-28T14:51:49.459906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789084169835460:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46823246ns) {Configure:104(46815703ns) {ProcessConfigured:135(1612511ns) {StartInitialHandshake:157(1611443ns) {DropHandshakes:470(756ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(406ns) PrepareNewSessionHandshake:477(1113ns) {SwitchToState:179(205ns)}}}}}
2025-11-28T14:51:49.460041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789082073009575:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17929621ns) {Configure:104(17922256ns) {ProcessConfigured:135(384159ns) {StartInitialHandshake:157(382069ns) {DropHandshakes:470(957ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(170ns)} GenerateSessionVirtualId:988(441ns) PrepareNewSessionHandshake:477(1202ns) {SwitchToState:179(589ns)}}}}}
2025-11-28T14:51:49.460552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789080416007894:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31718090ns) {Configure:104(31702459ns) {ProcessConfigured:135(912051ns) {StartInitialHandshake:157(910317ns) {DropHandshakes:470(893ns) {DropIncomingHandshake:442(223ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(1764ns) {SwitchToState:179(366ns)}}}}}
2025-11-28T14:51:49.460325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789081423225376:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(54939357ns) {Configure:104(54931190ns) {ProcessConfigured:135(830910ns) {StartInitialHandshake:157(827958ns) {DropHandshakes:470(2397ns) {DropIncomingHandshake:442(773ns) DropOutgoingHandshake:457(337ns)} GenerateSessionVirtualId:988(1242ns) PrepareNewSessionHandshake:477(2477ns) {SwitchToState:179(951ns)}}}}}
2025-11-28T14:51:49.460927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789084169835462:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26909815ns) {Configure:104(26901766ns) {ProcessConfigured:135(2026984ns) {StartInitialHandshake:157(2025171ns) {DropHandshakes:470(954ns) {DropIncomingHandshake:442(279ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:477(1610ns) {SwitchToState:179(558ns)}}}}}
2025-11-28T14:51:49.460615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789080605590107:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(62717735ns) {Configure:104(62708310ns) {ProcessConfigured:135(300008ns) {StartInitialHandshake:157(298081ns) {DropHandshakes:470(1109ns) {DropIncomingHandshake:442(255ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:477(596ns) {SwitchToState:179(145ns)}}}}}
2025-11-28T14:51:49.461244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789081423225378:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20340435ns) {Configure:104(20333458ns) {ProcessConfigured:135(1674929ns) {StartInitialHandshake:157(1672906ns) {DropHandshakes:470(875ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(2022ns) {SwitchToState:179(1217ns)}}}}}
2025-11-28T14:51:49.461191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789084169835461:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(57512211ns) {Configure:104(57507369ns) {ProcessConfigured:135(2052761ns) {StartInitialHandshake:157(2051403ns) {DropHandshakes:470(807ns) {DropIncomingHandshake:442(231ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(307ns) PrepareNewSessionHandshake:477(1288ns) {SwitchToState:179(537ns)}}}}}
2025-11-28T14:51:49.467812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789080605590108:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18694344ns) {Configure:104(18685106ns) {ProcessConfigured:135(662122ns) {StartInitialHandshake:157(660087ns) {DropHandshakes:470(1014ns) {DropIncomingHandshake:442(247ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(1426ns) {SwitchToState:179(610ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.475544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789084169835539:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52765
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.477661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009651:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.477638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789084169835538:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52765
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.480143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789080256104408:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.482379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009654:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
2025-11-28T14:51:49.482388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789080416007972:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55167
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.483781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789080605590206:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37757
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.483237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789082073009651:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.483187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789080256104407:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-28T14:51:49.484283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789084169835538:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.483766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009655:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
2025-11-28T14:51:49.483201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789081423225455:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27429
2025-11-28T14:51:49.482405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789080416007973:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55167
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.484153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789080605590204:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37757
2025-11-28T14:51:49.485812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789084169835539:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.484217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104408:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.485036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789080416007973:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.486139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789080605590207:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37757
2025-11-28T14:51:49.484224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789082073009574:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.487211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789081423225455:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.484456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789080256104409:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-28T14:51:49.485073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789080416007972:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.485915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789084169835461:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.485854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789080256104411:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-28T14:51:49.487223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789080416007965:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.485968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789084169835460:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.486521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104399:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.487769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789081423225456:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27429
2025-11-28T14:51:49.487256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789080416007965:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:49.486558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789080256104399:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:49.493615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789080416007974:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55167
2025-11-28T14:51:49.486851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789080256104410:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-28T14:51:49.487590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104401:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.487614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789080256104401:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:49.496810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789080256104401:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:49.496976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789080416007965:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:49.497031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789080256104399:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.497494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789080256104393:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.497592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789080416007958:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.497873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009657:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
2025-11-28T14:51:49.497651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:49.497791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789080256104391:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.497845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789080416007958:2107] [node 4] ICP22 created new session: [2:7577789080416007978:2107]
2025-11-28T14:51:49.498009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:49.497906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789084169835542:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52765
2025-11-28T14:51:49.498255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:49.498111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789080416007978:2107] [node 4] ICS09 handshake done sender: [2:7577789080416007965:2111] self: [2:7577789080416007959:0] peer: [4:7577789084169835540:0] socket: 242 qp: -1
No RDMA context found for GID: 2025-11-28T14:51:49.498302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789080256104391:2107] [node 4] ICP22 created new session: [1:7577789080256104417:2107]
::ffff:127.1.0.02025-11-28T14:51:49.498984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789080416007978:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:49.498439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789080256104393:2109] [node 6] ICP22 created new session: [1:7577789080256104416:2109]
2025-11-28T14:51:49.499030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:49.498450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789080256104417:2107] [node 4] ICS09 handshake done sender: [1:7577789080256104401:2112] self: [1:7577789080256104396:0] peer: [4:7577789084169835541:0] socket: 243 qp: -1
2025-11-28T14:51:49.499996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789084169835543:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52765
2025-11-28T14:51:49.498486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789080256104417:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:49.499987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789082073009651:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:49.499017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789080256104416:2109] [node 6] ICS09 handshake done sender: [1:7577789080256104399:2110] self: [1:7577789080256104394:0] peer: [6:7577789082073009656:0] socket: 240 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.500834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789082073009574:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.500838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789084169835539:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.499083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:49.502127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789084169835538:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:49.502523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789081423225458:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27429
2025-11-28T14:51:49.499388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789080256104416:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:49.503043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789084169835461:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.501683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:49.503083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.502216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789082073009574:2048] [node 1] ICP22 created new session: [6:7577789082073009659:2048]
2025-11-28T14:51:49.499445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:49.503069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789082073009659:2048] [node 1] ICS09 handshake done sender: [6:7577789082073009651:2116] self: [6:7577789082073009656:0] peer: [1:7577789080256104394:0] socket: 246 qp: -1
2025-11-28T14:51:49.503119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789084169835460:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.503149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789084169835461:2049] [node 2] ICP22 created new session: [4:7577789084169835544:2049]
2025-11-28T14:51:49.503112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789082073009659:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:49.503259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789080416007981:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55167
2025-11-28T14:51:49.503118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009658:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
2025-11-28T14:51:49.503231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789084169835544:2049] [node 2] ICS09 handshake done sender: [4:7577789084169835538:2115] self: [4:7577789084169835540:0] peer: [2:7577789080416007959:0] socket: 255 qp: -1
2025-11-28T14:51:49.503177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:49.503264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789084169835544:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:49.503354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:49.506203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789081423225458:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.503404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:49.503444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789084169835460:2048] [node 1] ICP22 created new session: [4:7577789084169835545:2048]
2025-11-28T14:51:49.506275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789080416007981:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.506842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789084169835545:2048] [node 1] ICS09 handshake done sender: [4:7577789084169835539:2116] self: [4:7577789084169835541:0] peer: [1:7577789080256104396:0] socket: 260 qp: -1
2025-11-28T14:51:49.506969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789080416007893:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.506986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789084169835545:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:49.507190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789081423225376:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.507306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104402:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.507370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:49.507343Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789080256104402:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:49.507429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104403:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.507456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789080256104403:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:49.510926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789080256104402:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:49.511062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789080256104403:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:49.511351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789080256104328:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.511599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789080256104392:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.511742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.511815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789080256104328:2048] [node 2] ICP22 created new session: [1:7577789080256104423:2048]
2025-11-28T14:51:49.511760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789080416007983:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55167
2025-11-28T14:51:49.511868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789081423225460:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27429
2025-11-28T14:51:49.511894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789080256104423:2048] [node 2] ICS09 handshake done sender: [1:7577789080256104402:2113] self: [1:7577789080256104397:0] peer: [2:7577789080416007982:0] socket: 291 qp: -1
2025-11-28T14:51:49.512159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789080416007981:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:49.511897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:49.511958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789080256104392:2108] [node 5] ICP22 created new session: [1:7577789080256104424:2108]
2025-11-28T14:51:49.511967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789080256104423:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:49.513883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789080416007893:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.513712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789081423225458:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:49.512139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789080256104424:2108] [node 5] ICS09 handshake done sender: [1:7577789080256104403:2114] self: [1:7577789080256104398:0] peer: [5:7577789081423225459:0] socket: 289 qp: -1
2025-11-28T14:51:49.513940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:49.515327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789080416007893:2048] [node 1] ICP22 created new session: [2:7577789080416007984:2048]
2025-11-28T14:51:49.512398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:49.512600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789080256104424:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:49.515763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789080416007984:2048] [node 1] ICS09 handshake done sender: [2:7577789080416007981:2118] self: [2:7577789080416007982:0] peer: [1:7577789080256104397:0] socket: 292 qp: -1
2025-11-28T14:51:49.513899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789081423225376:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789081423225458:2118] held: no
2025-11-28T14:51:49.512722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:49.515553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789081423225376:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.515813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789080416007984:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:49.515597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:49.516052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:49.515775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789081423225376:2048] [node 1] ICP22 created new session: [5:7577789081423225461:2048]
2025-11-28T14:51:49.515947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789081423225461:2048] [node 1] ICS09 handshake done sender: [5:7577789081423225458:2118] self: [5:7577789081423225459:0] peer: [1:7577789080256104398:0] socket: 290 qp: -1
2025-11-28T14:51:49.515981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789081423225461:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:49.516280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.516989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789084169835548:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52765
2025-11-28T14:51:49.518439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789084169835548:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.519779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789081423225447:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.527882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789082073009654:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.528128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789082073009637:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.528420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789081423225447:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.528936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789081423225447:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.529153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789081423225440:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.529210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009662:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
2025-11-28T14:51:49.529254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:49.529294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789081423225440:2109] [node 6] ICP22 created new session: [5:7577789081423225465:2109]
2025-11-28T14:51:49.529363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789081423225465:2109] [node 6] ICS09 handshake done sender: [5:7577789081423225447:2113] self: [5:7577789081423225444:0] peer: [6:7577789082073009661:0] socket: 256 qp: -1
2025-11-28T14:51:49.529399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789081423225465:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:49.529455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:49.529535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789082073009654:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:49.530075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789082073009637:2106] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.530394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:49.530478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789082073009637:2106] [node 5] ICP22 created new session: [6:7577789082073009663:2106]
2025-11-28T14:51:49.530548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789082073009663:2106] [node 5] ICS09 handshake done sender: [6:7577789082073009654:2117] self: [6:7577789082073009661:0] peer: [5:7577789081423225444:0] socket: 261 qp: -1
2025-11-28T14:51:49.530503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789080605590208:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37757
2025-11-28T14:51:49.530608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789082073009663:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:49.530657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:49.531496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789082073009637:2106] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.531941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789080416007966:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.532649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789080605590209:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37757
2025-11-28T14:51:49.532805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104407:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.539331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789080256104328:2048] [node 2] ICP18 handshake (actor [1:7577789080256104407:2115]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.539985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789080605590209:2138] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.539996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789081423225468:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27429
2025-11-28T14:51:49.541474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789081423225468:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.541673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789081423225377:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.541941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789080416007968:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.541985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789080416007968:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:49.544108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789080416007968:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:49.544257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789080416007967:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.544733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789080416007956:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.544812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:49.545032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789080605590206:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.544920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789080416007956:2108] [node 5] ICP22 created new session: [2:7577789080416007988:2108]
2025-11-28T14:51:49.550857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789080605590108:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:51:49.551088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789081423225377:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789081423225468:2120] held: no
::ffff:127.1.0.0
2025-11-28T14:51:49.544967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789080416007988:2108] [node 5] ICS09 handshake done sender: [2:7577789080416007968:2114] self: [2:7577789080416007963:0] peer: [5:7577789081423225469:0] socket: 247 qp: -1
2025-11-28T14:51:49.545049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789080416007988:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:49.545091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:49.551350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789081423225470:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27429
2025-11-28T14:51:49.551383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789080416007967:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.552064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789081423225468:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:49.552359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789080416007967:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:49.552207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789081423225377:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.552242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:49.552337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789081423225377:2049] [node 2] ICP22 created new session: [5:7577789081423225471:2049]
2025-11-28T14:51:49.552530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789081423225471:2049] [node 2] ICS09 handshake done sender: [5:7577789081423225468:2120] self: [5:7577789081423225469:0] peer: [2:7577789080416007963:0] socket: 302 qp: -1
2025-11-28T14:51:49.552627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789081423225471:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:49.552677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.552723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789080416007894:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.552830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:49.552885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789080416007894:2049] [node 3] ICP22 created new session: [2:7577789080416007991:2049]
2025-11-28T14:51:49.552931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789080416007991:2049] [node 3] ICS09 handshake done sender: [2:7577789080416007967:2113] self: [2:7577789080416007961:0] peer: [3:7577789080605590211:0] socket: 244 qp: -1
2025-11-28T14:51:49.552943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789080605590212:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37757
2025-11-28T14:51:49.552968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789080416007991:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:49.552994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:49.553208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789080605590206:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:49.553455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789080605590108:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.553522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:49.553571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789080605590108:2049] [node 2] ICP22 created new session: [3:7577789080605590213:2049]
2025-11-28T14:51:49.553622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789080605590213:2049] [node 2] ICS09 handshake done sender: [3:7577789080605590206:2135] self: [3:7577789080605590211:0] peer: [2:7577789080416007961:0] socket: 270 qp: -1
2025-11-28T14:51:49.553666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789080605590213:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:49.553703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.554464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789084169835550:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52765
2025-11-28T14:51:49.555606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789080256104400:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.563546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789080605590204:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.563705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789080605590107:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.563990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104400:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.564586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789080256104400:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:49.564683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789080605590217:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37757
2025-11-28T14:51:49.564809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789080256104329:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.564953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:49.565144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789080256104329:2049] [node 3] ICP22 created new session: [1:7577789080256104428:2049]
2025-11-28T14:51:49.565195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789080256104428:2049] [node 3] ICS09 handshake done sender: [1:7577789080256104400:2111] self: [1:7577789080256104395:0] peer: [3:7577789080605590216:0] socket: 241 qp: -1
2025-11-28T14:51:49.565185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789080605590204:2134] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:49.565235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789080256104428:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:49.565288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:49.565693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789080605590107:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.565752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:49.565798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789080605590107:2048] [node 1] ICP22 created new session: [3:7577789080605590218:2048]
2025-11-28T14:51:49.565845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789080605590218:2048] [node 1] ICS09 handshake done sender: [3:7577789080605590204:2134] self: [3:7577789080605590216:0] peer: [1:7577789080256104395:0] socket: 252 qp: -1
2025-11-28T14:51:49.565876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789080605590218:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:49.565925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:49.585246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789084169835529:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.593428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104409:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.593816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789080256104391:2107] [node 4] ICP18 handshake (actor [1:7577789080256104409:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.594493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009666:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
2025-11-28T14:51:49.608428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789084169835530:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.617530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789080605590207:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.636785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789082073009647:2113] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.645355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104410:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.645590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789080256104393:2109] [node 6] ICP18 handshake (actor [1:7577789080256104410:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.646539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789084169835552:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52765
2025-11-28T14:51:49.648105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789084169835552:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.651202Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789080605590197:2129] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.659106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789082073009655:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.659351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789082073009576:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.659639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789080605590197:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.660151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789080605590197:2129] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.660400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789080605590191:2119] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.660427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009669:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
2025-11-28T14:51:49.660470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:49.660533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789080605590191:2119] [node 6] ICP22 created new session: [3:7577789080605590222:2119]
2025-11-28T14:51:49.660596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789080605590222:2119] [node 6] ICS09 handshake done sender: [3:7577789080605590197:2129] self: [3:7577789080605590192:0] peer: [6:7577789082073009668:0] socket: 272 qp: -1
2025-11-28T14:51:49.660661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789080605590222:2119] [node 6] ICS10 traffic start
2025-11-28T14:51:49.660697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:49.660707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789082073009655:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:49.660878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789082073009576:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.660954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:49.661146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789082073009576:2050] [node 3] ICP22 created new session: [6:7577789082073009670:2050]
2025-11-28T14:51:49.661205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789082073009670:2050] [node 3] ICS09 handshake done sender: [6:7577789082073009655:2118] self: [6:7577789082073009668:0] peer: [3:7577789080605590192:0] socket: 275 qp: -1
2025-11-28T14:51:49.661252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789082073009670:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:49.661344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:49.662021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789082073009576:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:49.663496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789080605590199:2131] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.671900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789080256104411:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.672106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789080256104329:2049] [node 3] ICP18 handshake (actor [1:7577789080256104411:2119]) is too old
2025-11-28T14:51:49.675488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789080605590198:2130] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.682716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789081423225456:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.682913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789081423225378:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.683160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789080605590198:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.683680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789080605590198:2130] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.683900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789080605590189:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.683842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789081423225474:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27429
2025-11-28T14:51:49.683961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:49.684014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789080605590189:2109] [node 5] ICP22 created new session: [3:7577789080605590225:2109]
2025-11-28T14:51:49.684224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789080605590225:2109] [node 5] ICS09 handshake done sender: [3:7577789080605590198:2130] self: [3:7577789080605590194:0] peer: [5:7577789081423225473:0] socket: 276 qp: -1
2025-11-28T14:51:49.684285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789080605590225:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:49.684302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789081423225456:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:49.684331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:49.684514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789081423225378:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.684586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:49.684619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789081423225378:2050] [node 3] ICP22 created new session: [5:7577789081423225475:2050]
2025-11-28T14:51:49.684664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789081423225475:2050] [node 3] ICS09 handshake done sender: [5:7577789081423225456:2117] self: [5:7577789081423225473:0] peer: [3:7577789080605590194:0] socket: 278 qp: -1
2025-11-28T14:51:49.684690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789081423225475:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:49.684717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:49.686181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789084169835531:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.686981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789080416007974:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.693995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789080416007958:2107] [node 4] ICP18 handshake (actor [2:7577789080416007974:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.694563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789081423225477:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27429
2025-11-28T14:51:49.699442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789080416007964:2110] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.709737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789082073009658:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.709971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789082073009575:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.710243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789080416007964:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.710782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789080416007964:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.710989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789080416007957:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.712274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:49.712402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009673:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
2025-11-28T14:51:49.712392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789080416007957:2109] [node 6] ICP22 created new session: [2:7577789080416007994:2109]
2025-11-28T14:51:49.712474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789080416007994:2109] [node 6] ICS09 handshake done sender: [2:7577789080416007964:2110] self: [2:7577789080416007960:0] peer: [6:7577789082073009672:0] socket: 288 qp: -1
2025-11-28T14:51:49.712517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789080416007994:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:49.712552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:49.712714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789082073009658:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:49.712726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789081423225450:2114] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.720186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789082073009575:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.720851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:49.720922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789082073009575:2049] [node 2] ICP22 created new session: [6:7577789082073009674:2049]
2025-11-28T14:51:49.721889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789082073009674:2049] [node 2] ICS09 handshake done sender: [6:7577789082073009658:2120] self: [6:7577789082073009672:0] peer: [2:7577789080416007960:0] socket: 293 qp: -1
2025-11-28T14:51:49.721948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789082073009674:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:49.722002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:49.721951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789080605590208:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.722249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789080605590189:2109] [node 5] ICP18 handshake (actor [3:7577789080605590208:2137]) is too old
2025-11-28T14:51:49.722722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789082073009575:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:49.726022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789080605590200:2132] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.738152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789084169835550:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.738396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789084169835462:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.738495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789084169835534:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.745022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789080605590200:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.746733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789080605590200:2132] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:49.746968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789080605590190:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.747036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:49.747085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789080605590190:2107] [node 4] ICP22 created new session: [3:7577789080605590228:2107]
2025-11-28T14:51:49.747052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789082073009666:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.747125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789080605590228:2107] [node 4] ICS09 handshake done sender: [3:7577789080605590200:2132] self: [3:7577789080605590193:0] peer: [4:7577789084169835555:0] socket: 304 qp: -1
2025-11-28T14:51:49.747130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789084169835556:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52765
2025-11-28T14:51:49.747151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789080605590228:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:49.747300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:49.747311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789082073009639:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.747631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789084169835534:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.747677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789084169835462:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32147) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789084169835550:2120] held: no
2025-11-28T14:51:49.747757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789084169835550:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:49.747987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789084169835462:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.748156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:49.748218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789084169835462:2050] [node 3] ICP22 created new session: [4:7577789084169835558:2050]
2025-11-28T14:51:49.748264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789084169835558:2050] [node 3] ICS09 handshake done sender: [4:7577789084169835550:2120] self: [4:7577789084169835555:0] peer: [3:7577789080605590193:0] socket: 305 qp: -1
2025-11-28T14:51:49.748298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789084169835558:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:49.748348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:49.748471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789084169835534:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.748650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789082073009677:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54875
2025-11-28T14:51:49.748627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789084169835523:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.748725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:49.748751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789084169835523:2109] [node 6] ICP22 created new session: [4:7577789084169835560:2109]
2025-11-28T14:51:49.748824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789084169835560:2109] [node 6] ICS09 handshake done sender: [4:7577789084169835534:2114] self: [4:7577789084169835533:0] peer: [6:7577789082073009676:0] socket: 259 qp: -1
2025-11-28T14:51:49.748858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789084169835560:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:49.748887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:49.749002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789082073009666:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:49.749317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789082073009639:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789082073009666:2122] held: no
2025-11-28T14:51:49.749928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789082073009639:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.749976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:49.750006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789082073009639:2107] [node 4] ICP22 created new session: [6:7577789082073009678:2107]
2025-11-28T14:51:49.750044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789082073009678:2107] [node 4] ICS09 handshake done sender: [6:7577789082073009666:2122] self: [6:7577789082073009676:0] peer: [4:7577789084169835533:0] socket: 308 qp: -1
2025-11-28T14:51:49.750065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789082073009678:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:49.750100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:49.752679Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789084169835532:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:49.753376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789081423225477:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:49.753542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789081423225441:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:49.753726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789084169835532:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:49.754163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789084169835532:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:49.754235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789081423225479:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27429
2025-11-28T14:51:49.754358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789084169835524:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:49.754462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:49.754487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789081423225477:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:49.754563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789084169835524:2108] [node 5] ICP22 created new session: [4:7577789084169835563:2108]
2025-11-28T14:51:49.754636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789084169835563:2108] [node 5] ICS09 handshake done sender: [4:7577789084169835532:2113] self: [4:7577789084169835528:0] peer: [5:7577789081423225478:0] socket: 263 qp: -1
2025-11-28T14:51:49.754667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789084169835563:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:49.754716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789081423225441:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:49.754870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:49.754769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:49.754927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789081423225441:2108] [node 4] ICP22 created new session: [5:7577789081423225480:2108]
2025-11-28T14:51:49.754970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789081423225480:2108] [node 4] ICS09 handshake done sender: [5:7577789081423225477:2123] self: [5:7577789081423225478:0] peer: [4:7577789084169835528:0] socket: 274 qp: -1
2025-11-28T14:51:49.754997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789081423225480:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:49.755274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:49.755496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789081423225441:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:49.795136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:49.795412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789080416007984:2048] [node 1] ICS07 socket disconnect 292 reason# EndOfStream
2025-11-28T14:51:49.795455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789080416007984:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.795583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.795645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789080416007984:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:49.796112Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:49.796419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789080605590218:2048] [node 1] ICS01 socket: 252 reason# EndOfStream
2025-11-28T14:51:49.796462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789080605590107:2048] [node 1] ICP30 unregister session Session# [3:7577789080605590218:2048] VirtualId# [3:7577789080605590216:0]
2025-11-28T14:51:49.796497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789080605590218:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.796590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.796654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.797700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789080416007893:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:49.797744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789080416007984:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:49.798408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:49.798655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789084169835545:2048] [node 1] ICS07 socket disconnect 260 reason# EndOfStream
2025-11-28T14:51:49.798702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789084169835545:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.798753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789080416007893:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.798791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789080416007984:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.798786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.798854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789080416007893:2048] [node 1] ICP30 unregister session Session# [2:7577789080416007984:2048] VirtualId# [2:7577789080416007982:0]
2025-11-28T14:51:49.798844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789084169835545:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:49.798890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.798908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789080416007893:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.798919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.799666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:49.799684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789084169835460:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.799744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789084169835545:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.799784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789084169835460:2048] [node 1] ICP30 unregister session Session# [4:7577789084169835545:2048] VirtualId# [4:7577789084169835541:0]
2025-11-28T14:51:49.799833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.799860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789081423225461:2048] [node 1] ICS01 socket: 290 reason# EndOfStream
2025-11-28T14:51:49.799861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789084169835460:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.799891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789081423225376:2048] [node 1] ICP30 unregister session Session# [5:7577789081423225461:2048] VirtualId# [5:7577789081423225459:0]
2025-11-28T14:51:49.799877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.799910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789081423225461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.799959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.800003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.800743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:49.800929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789082073009659:2048] [node 1] ICS07 socket disconnect 246 reason# EndOfStream
2025-11-28T14:51:49.800959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789082073009659:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.801026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:49.801070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789082073009659:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:49.801691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789082073009574:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.801798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789082073009659:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.801834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789082073009574:2048] [node 1] ICP30 unregister session Session# [6:7577789082073009659:2048] VirtualId# [6:7577789082073009656:0]
2025-11-28T14:51:49.801877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:49.801907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789082073009574:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.801930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2933) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.809798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789084169835460:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:49.811865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789082073009574:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:49.821585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:49.821747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789080605590213:2049] [node 2] ICS07 socket disconnect 270 reason# EndOfStream
2025-11-28T14:51:49.821776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789080605590213:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.821876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:49.821926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789080605590213:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:49.824105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:49.824207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789080605590108:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:49.824256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789080605590213:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:49.824288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789084169835544:2049] [node 2] ICS07 socket disconnect 255 reason# EndOfStream
2025-11-28T14:51:49.824319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789084169835544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.824669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:49.824740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789084169835544:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:49.825219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789080605590108:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.825314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789080605590213:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.825392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789080605590108:2049] [node 2] ICP30 unregister session Session# [3:7577789080605590213:2049] VirtualId# [3:7577789080605590211:0]
2025-11-28T14:51:49.825353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:49.825449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:49.825487Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789080605590108:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.825507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.825499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789084169835461:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.825566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789081423225471:2049] [node 2] ICS01 socket: 302 reason# EndOfStream
2025-11-28T14:51:49.825672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789081423225377:2049] [node 2] ICP30 unregister session Session# [5:7577789081423225471:2049] VirtualId# [5:7577789081423225469:0]
2025-11-28T14:51:49.825702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789081423225471:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.825741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789084169835544:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.825790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:49.825849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789084169835461:2049] [node 2] ICP30 unregister session Session# [4:7577789084169835544:2049] VirtualId# [4:7577789084169835540:0]
2025-11-28T14:51:49.825840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:49.825929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:49.825971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789084169835461:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.825984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.826598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:49.826782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789082073009674:2049] [node 2] ICS07 socket disconnect 293 reason# EndOfStream
2025-11-28T14:51:49.826814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789082073009674:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.826874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:49.826914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789082073009674:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:49.827514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789082073009575:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.827556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789082073009674:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.827630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789082073009575:2049] [node 2] ICP30 unregister session Session# [6:7577789082073009674:2049] VirtualId# [6:7577789082073009672:0]
2025-11-28T14:51:49.827676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:49.827724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789082073009575:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.827739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.835472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789084169835461:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:49.837443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789082073009575:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:49.839051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:49.839307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789084169835558:2050] [node 3] ICS01 socket: 305 reason# EndOfStream
2025-11-28T14:51:49.839356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789084169835462:2050] [node 3] ICP30 unregister session Session# [4:7577789084169835558:2050] VirtualId# [4:7577789084169835555:0]
2025-11-28T14:51:49.839401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789084169835558:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.839507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:49.839577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:49.840123Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:49.840278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789081423225475:2050] [node 3] ICS07 socket disconnect 278 reason# EndOfStream
2025-11-28T14:51:49.840305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789081423225475:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.840427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:49.840475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789081423225475:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:49.841340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789081423225378:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32147) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.841218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:49.841459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789081423225475:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.841507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789082073009670:2050] [node 3] ICS07 socket disconnect 275 reason# EndOfStream
2025-11-28T14:51:49.841491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789081423225378:2050] [node 3] ICP30 unregister session Session# [5:7577789081423225475:2050] VirtualId# [5:7577789081423225473:0]
2025-11-28T14:51:49.841536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789082073009670:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.841695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:49.841606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:49.841764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789081423225378:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.841786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.841711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789082073009670:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:49.842632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789082073009576:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32147) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.842694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789082073009670:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.842732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789082073009576:2050] [node 3] ICP30 unregister session Session# [6:7577789082073009670:2050] VirtualId# [6:7577789082073009668:0]
2025-11-28T14:51:49.842782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:49.842816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789082073009576:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.842834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.851818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789081423225378:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:49.852853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:49.852806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789082073009576:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:49.853034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789081423225480:2108] [node 4] ICS07 socket disconnect 274 reason# EndOfStream
2025-11-28T14:51:49.853068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789081423225480:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.853151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:49.853193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789081423225480:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:49.853819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:49.853951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789082073009678:2107] [node 4] ICS01 socket: 308 reason# EndOfStream
2025-11-28T14:51:49.853982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789082073009639:2107] [node 4] ICP30 unregister session Session# [6:7577789082073009678:2107] VirtualId# [6:7577789082073009676:0]
2025-11-28T14:51:49.854004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789082073009678:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.854064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:49.854102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:49.854045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789081423225441:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7630) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.854137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789081423225480:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.854190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789081423225441:2108] [node 4] ICP30 unregister session Session# [5:7577789081423225480:2108] VirtualId# [5:7577789081423225478:0]
2025-11-28T14:51:49.854242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:49.854302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789081423225441:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7630) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.854322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7630) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.889844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:49.889989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789082073009663:2106] [node 5] ICS07 socket disconnect 261 reason# EndOfStream
2025-11-28T14:51:49.890014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789082073009663:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:49.890079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:49.890116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789082073009663:2106] [node 5] ICS15 start handshake
2025-11-28T14:51:49.890754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789082073009637:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9579) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:49.890826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789082073009663:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:49.890857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789082073009637:2106] [node 5] ICP30 unregister session Session# [6:7577789082073009663:2106] VirtualId# [6:7577789082073009661:0]
2025-11-28T14:51:49.890892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:49.890913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789082073009637:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9579) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:49.890922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9579) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:49.945202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789081945770117:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:49.945240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789081945770118:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:49.963568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789082862819178:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:49.963623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789082862819179:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:49.983486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789081542660264:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:49.983522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789081542660265:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:50.003330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789087349510883:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:50.003372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789087349510884:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:50.003396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789087349510885:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:50.022754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088081626283:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:50.022792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088081626284:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:50.022818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088081626285:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:50.043436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789085610857882:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:50.043501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789085610857883:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:50.043544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789085610857884:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:50.118462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789085610857943:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:50.125641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789087349510948:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:50.125867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789085837627650:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:50.125879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789086240737476:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:50.126242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789086240737477:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:50.126074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789087157786539:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:50.126388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789087157786540:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:50.126140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789087349510949:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:50.126317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789085837627651:2136] [node 5] ICP01 ready to work
2025-11-28T14:51:50.126320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789085610857946:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:50.126697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789085837627652:2124] [node 6] ICP01 ready to work
2025-11-28T14:51:50.126674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789086240737478:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:50.126714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789087157786541:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:50.128446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088081626366:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:50.128817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088081626367:2127] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.136608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789087349510963:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59687
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.141268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789087157786552:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17404
2025-11-28T14:51:50.143296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088081626376:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28997
2025-11-28T14:51:50.141355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789087157786553:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17404
2025-11-28T14:51:50.141389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789087157786554:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17404
2025-11-28T14:51:50.141426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789087157786555:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17404
2025-11-28T14:51:50.146144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088081626376:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.146380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086240737489:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19807
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.146625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086240737490:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19807
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.147078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086240737491:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19807
2025-11-28T14:51:50.147874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737489:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.148009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086240737494:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19807
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.148091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086240737495:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19807
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.149376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789085837627666:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63085
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.149602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789085837627667:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63085
2025-11-28T14:51:50.149636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737490:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.150006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789085837627668:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63085
2025-11-28T14:51:50.149973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789087157786552:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.150291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789087157786553:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.150320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789085837627669:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63085
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:51:50.150429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789087157786554:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.150599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789087157786559:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17404
2025-11-28T14:51:50.150602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789085837627666:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.150586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789087349510964:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59687
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.150713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789085837627667:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.150734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789087349510965:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59687
2025-11-28T14:51:50.150833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789085837627668:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.150844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789087349510963:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.150951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789087349510885:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.151045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789081542660265:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.151140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789087349510964:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.151225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789087349510884:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.151191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789087157786555:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.151250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789085837627658:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.151275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789085837627658:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.151354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789087157786547:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.151380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789087157786547:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.152302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789085837627658:2139] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.152473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789087157786543:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.152500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789087157786543:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.152729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789085837627650:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.152733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789087349510968:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59687
2025-11-28T14:51:50.152792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.153033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789085837627650:2107] [node 4] ICP22 created new session: [3:7577789085837627674:2107]
2025-11-28T14:51:50.153295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789085837627674:2107] [node 4] ICS09 handshake done sender: [3:7577789085837627658:2139] self: [3:7577789085837627657:0] peer: [4:7577789087349510966:0] socket: 252 qp: -1
2025-11-28T14:51:50.153324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789085837627674:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:50.153401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.153592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737495:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.153722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737491:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.153731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789087157786547:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:50.153790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737494:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.153875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789087349510963:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:50.154273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789087349510885:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.154366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.154344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789087157786559:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.154416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789087349510885:2050] [node 3] ICP22 created new session: [4:7577789087349510970:2050]
2025-11-28T14:51:50.154406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789082862819179:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.154463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789087349510970:2050] [node 3] ICS09 handshake done sender: [4:7577789087349510963:2117] self: [4:7577789087349510966:0] peer: [3:7577789085837627657:0] socket: 257 qp: -1
2025-11-28T14:51:50.154460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.154500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789087349510970:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:50.154501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789082862819179:2049] [node 3] ICP22 created new session: [2:7577789087157786566:2049]
2025-11-28T14:51:50.154580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789087157786566:2049] [node 3] ICS09 handshake done sender: [2:7577789087157786547:2114] self: [2:7577789087157786546:0] peer: [3:7577789085837627672:0] socket: 273 qp: -1
2025-11-28T14:51:50.154593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:50.154616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789087157786566:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:50.154648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:50.155097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789087157786543:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:50.155112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789085837627669:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.155274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789087349510965:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.156581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789085837627676:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63085
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.156729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789087157786539:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.156821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.156880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789087157786539:2107] [node 4] ICP22 created new session: [2:7577789087157786568:2107]
2025-11-28T14:51:50.156829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088081626381:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28997
2025-11-28T14:51:50.156980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789087157786568:2107] [node 4] ICS09 handshake done sender: [2:7577789087157786543:2112] self: [2:7577789087157786542:0] peer: [4:7577789087349510967:0] socket: 271 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.157022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789087157786568:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:50.157051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.157113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789087349510969:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59687
2025-11-28T14:51:50.157569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789087349510885:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:50.157801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789087349510883:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.158237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789087349510964:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:50.158316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789082862819178:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.158696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789081542660265:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64579) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789085837627668:2144] held: no
2025-11-28T14:51:50.158749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088081626382:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28997
2025-11-28T14:51:50.158873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737480:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.158923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789086240737480:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.158898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789081542660264:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.159134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789085837627668:2144] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.159990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789087349510884:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.160012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789081542660265:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.160000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737484:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.160024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857960:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
2025-11-28T14:51:50.160028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789086240737484:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.160077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:50.160063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:50.160124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789087349510884:2049] [node 2] ICP22 created new session: [4:7577789087349510974:2049]
2025-11-28T14:51:50.160129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789081542660265:2049] [node 2] ICP22 created new session: [3:7577789085837627679:2049]
2025-11-28T14:51:50.160172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789087349510974:2049] [node 2] ICS09 handshake done sender: [4:7577789087349510964:2118] self: [4:7577789087349510967:0] peer: [2:7577789087157786542:0] socket: 286 qp: -1
2025-11-28T14:51:50.160206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789087349510974:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:50.160235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:50.160230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789085837627679:2049] [node 2] ICS09 handshake done sender: [3:7577789085837627668:2144] self: [3:7577789085837627672:0] peer: [2:7577789087157786546:0] socket: 282 qp: -1
2025-11-28T14:51:50.160263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789085837627679:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:50.160310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:50.160469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789087349510884:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:50.160830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737482:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.160855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789086240737482:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.161652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088081626383:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28997
2025-11-28T14:51:50.161763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088081626382:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.161792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086240737480:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:50.162037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789088081626284:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.162446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857961:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
2025-11-28T14:51:50.162508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789081945770117:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.162648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:50.162759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789081945770117:2048] [node 2] ICP22 created new session: [1:7577789086240737505:2048]
2025-11-28T14:51:50.162809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086240737505:2048] [node 2] ICS09 handshake done sender: [1:7577789086240737480:2110] self: [1:7577789086240737479:0] peer: [2:7577789087157786571:0] socket: 278 qp: -1
2025-11-28T14:51:50.162838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086240737505:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:50.162896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:50.162920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789085610857960:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.162938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789087157786549:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.162990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789085610857883:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.162959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789087157786549:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.163653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857963:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.163610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789087157786551:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.163628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789087157786551:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.163745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088081626383:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.163827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086240737484:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:50.163866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789087349510977:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59687
2025-11-28T14:51:50.163820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088081626381:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.163984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789088081626366:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.164030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789088081626285:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.164449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789085837627682:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63085
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.164709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086240737482:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:50.164706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789087157786573:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17404
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.165474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088081626387:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28997
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.165512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789085837627660:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.166305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857964:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
2025-11-28T14:51:50.165550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789085837627660:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.165693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789087349510957:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.165721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789087349510957:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.166865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789081945770118:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.166951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.167005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789081945770118:2049] [node 3] ICP22 created new session: [1:7577789086240737507:2049]
2025-11-28T14:51:50.167054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086240737507:2049] [node 3] ICS09 handshake done sender: [1:7577789086240737484:2112] self: [1:7577789086240737483:0] peer: [3:7577789085837627678:0] socket: 280 qp: -1
2025-11-28T14:51:50.167192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086240737507:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:50.167238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:50.167463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789087157786549:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:50.167488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789085610857961:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.167498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789087349510965:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.167665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789086240737476:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.167709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789085610857946:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.168039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.168090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789086240737476:2107] [node 4] ICP22 created new session: [1:7577789086240737509:2107]
2025-11-28T14:51:50.168237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789087349510883:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.168181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086240737509:2107] [node 4] ICS09 handshake done sender: [1:7577789086240737482:2111] self: [1:7577789086240737481:0] peer: [4:7577789087349510972:0] socket: 279 qp: -1
2025-11-28T14:51:50.168283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.168246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086240737509:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:50.168261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789087157786551:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:50.168259Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789085837627669:2145] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.168315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789087349510883:2048] [node 1] ICP22 created new session: [4:7577789087349510979:2048]
2025-11-28T14:51:50.168278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.168357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789087349510979:2048] [node 1] ICS09 handshake done sender: [4:7577789087349510965:2119] self: [4:7577789087349510972:0] peer: [1:7577789086240737481:0] socket: 287 qp: -1
2025-11-28T14:51:50.168405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789087349510979:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.168480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.168765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088081626382:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:50.168784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789087157786559:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.169050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789082862819178:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24397) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789087157786559:2121] held: no
2025-11-28T14:51:50.169068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789081542660264:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.169144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789082862819178:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.169122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.169218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789081542660264:2048] [node 1] ICP22 created new session: [3:7577789085837627684:2048]
2025-11-28T14:51:50.169203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.169266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789085837627684:2048] [node 1] ICS09 handshake done sender: [3:7577789085837627669:2145] self: [3:7577789085837627678:0] peer: [1:7577789086240737483:0] socket: 283 qp: -1
2025-11-28T14:51:50.169229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789082862819178:2048] [node 1] ICP22 created new session: [2:7577789087157786576:2048]
2025-11-28T14:51:50.169268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789088081626284:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.169301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789085837627684:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.169332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:50.169342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.169256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789087157786576:2048] [node 1] ICS09 handshake done sender: [2:7577789087157786559:2121] self: [2:7577789087157786571:0] peer: [1:7577789086240737479:0] socket: 285 qp: -1
2025-11-28T14:51:50.169280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789087157786576:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.169375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088081626284:2049] [node 2] ICP22 created new session: [5:7577789088081626390:2049]
2025-11-28T14:51:50.169324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.169380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789085610857963:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.169423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088081626390:2049] [node 2] ICS09 handshake done sender: [5:7577789088081626382:2135] self: [5:7577789088081626384:0] peer: [2:7577789087157786548:0] socket: 256 qp: -1
2025-11-28T14:51:50.169450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088081626390:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:50.169436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789085610857960:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:50.169520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:50.169740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789085610857883:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.169791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:50.169708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789087157786540:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.169840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789085610857883:2049] [node 2] ICP22 created new session: [6:7577789085610857966:2049]
2025-11-28T14:51:50.169848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.169876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789085610857966:2049] [node 2] ICS09 handshake done sender: [6:7577789085610857960:2115] self: [6:7577789085610857962:0] peer: [2:7577789087157786550:0] socket: 294 qp: -1
2025-11-28T14:51:50.169886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789087157786540:2109] [node 5] ICP22 created new session: [2:7577789087157786578:2109]
2025-11-28T14:51:50.169905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789085610857966:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:50.169918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789087157786578:2109] [node 5] ICS09 handshake done sender: [2:7577789087157786549:2115] self: [2:7577789087157786548:0] peer: [5:7577789088081626384:0] socket: 251 qp: -1
2025-11-28T14:51:50.169936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:50.169937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789087349510959:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:51:50.169973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789087349510959:2116] [node 6] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-28T14:51:50.169973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789087157786578:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:50.170021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.170042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088081626388:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28997
2025-11-28T14:51:50.170825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789087349510957:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:50.171640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789087157786541:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.171673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.171692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789087157786541:2110] [node 6] ICP22 created new session: [2:7577789087157786580:2110]
2025-11-28T14:51:50.173007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789087157786580:2110] [node 6] ICS09 handshake done sender: [2:7577789087157786551:2116] self: [2:7577789087157786550:0] peer: [6:7577789085610857962:0] socket: 269 qp: -1
2025-11-28T14:51:50.173038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789087157786580:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:50.173224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789085837627660:2140] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:50.173059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:50.173332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789085610857884:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.173525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789087349510948:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.173572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.173628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789087349510948:2108] [node 5] ICP22 created new session: [4:7577789087349510982:2108]
2025-11-28T14:51:50.173681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789087349510982:2108] [node 5] ICS09 handshake done sender: [4:7577789087349510957:2115] self: [4:7577789087349510956:0] peer: [5:7577789088081626385:0] socket: 295 qp: -1
2025-11-28T14:51:50.173719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789087349510982:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:50.173752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.173776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789085837627651:2136] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.173838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.173905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789085837627651:2136] [node 5] ICP22 created new session: [3:7577789085837627686:2136]
2025-11-28T14:51:50.173953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789085837627686:2136] [node 5] ICS09 handshake done sender: [3:7577789085837627660:2140] self: [3:7577789085837627659:0] peer: [5:7577789088081626386:0] socket: 293 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.173988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789085837627686:2136] [node 5] ICS10 traffic start
2025-11-28T14:51:50.174020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.174059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088081626389:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28997
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.174807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088081626383:2136] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:50.174942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857969:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
2025-11-28T14:51:50.175024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789087349510959:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:50.175972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789085837627662:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.176007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789085837627662:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.176571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088081626381:2134] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:50.176697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789087349510949:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.177131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.177188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789088081626366:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.177285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789087349510949:2109] [node 6] ICP22 created new session: [4:7577789087349510984:2109]
2025-11-28T14:51:50.177327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789087349510984:2109] [node 6] ICS09 handshake done sender: [4:7577789087349510959:2116] self: [4:7577789087349510958:0] peer: [6:7577789085610857965:0] socket: 298 qp: -1
2025-11-28T14:51:50.177268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.177438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789087349510984:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:50.177350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088081626366:2108] [node 4] ICP22 created new session: [5:7577789088081626394:2108]
2025-11-28T14:51:50.177486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.177606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088081626394:2108] [node 4] ICS09 handshake done sender: [5:7577789088081626383:2136] self: [5:7577789088081626385:0] peer: [4:7577789087349510956:0] socket: 296 qp: -1
2025-11-28T14:51:50.177624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088081626394:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:50.177699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.178019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857971:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.178237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789085837627688:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63085
2025-11-28T14:51:50.178422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789085837627662:2141] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:50.179024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789085837627652:2124] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.179074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.179114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789085837627652:2124] [node 6] ICP22 created new session: [3:7577789085837627690:2124]
2025-11-28T14:51:50.179162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789085837627690:2124] [node 6] ICS09 handshake done sender: [3:7577789085837627662:2141] self: [3:7577789085837627661:0] peer: [6:7577789085610857968:0] socket: 304 qp: -1
2025-11-28T14:51:50.179196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789085837627690:2124] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:51:50.179231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
2025-11-28T14:51:50.179393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088081626392:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28997
2025-11-28T14:51:50.179477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789085610857969:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.179544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789085610857961:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:50.179771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789085837627688:2148] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.179925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789088081626285:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.179969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.179996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088081626285:2050] [node 3] ICP22 created new session: [5:7577789088081626396:2050]
2025-11-28T14:51:50.179997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789085610857946:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.180046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.180049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088081626396:2050] [node 3] ICS09 handshake done sender: [5:7577789088081626381:2134] self: [5:7577789088081626386:0] peer: [3:7577789085837627659:0] socket: 268 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.180077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088081626396:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:50.180062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789085837627652:2124] [node 6] ICP18 handshake (actor [3:7577789085837627688:2148]) is too old
2025-11-28T14:51:50.180087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789085610857946:2107] [node 4] ICP22 created new session: [6:7577789085610857973:2107]
2025-11-28T14:51:50.180096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:50.180136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789085610857973:2107] [node 4] ICS09 handshake done sender: [6:7577789085610857961:2116] self: [6:7577789085610857965:0] peer: [4:7577789087349510958:0] socket: 300 qp: -1
2025-11-28T14:51:50.180167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789085610857973:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:50.180147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789087349510986:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59687
2025-11-28T14:51:50.180297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.180332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789088081626285:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.181054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088081626392:2140] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.181083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857972:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
2025-11-28T14:51:50.181435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789088081626283:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.181757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737486:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.181793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789086240737486:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.182708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086240737486:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.183317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789086240737477:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.183315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088081626400:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28997
2025-11-28T14:51:50.183336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789085610857884:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28150) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789085610857963:2117] held: no
2025-11-28T14:51:50.183369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.183413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789086240737477:2108] [node 5] ICP22 created new session: [1:7577789086240737514:2108]
2025-11-28T14:51:50.183452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086240737514:2108] [node 5] ICS09 handshake done sender: [1:7577789086240737486:2113] self: [1:7577789086240737485:0] peer: [5:7577789088081626399:0] socket: 260 qp: -1
2025-11-28T14:51:50.183480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086240737514:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:50.183511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.183523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789085610857882:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.183688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088081626392:2140] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.183746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789085610857963:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:50.183837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789088081626283:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.183877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.183907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088081626283:2048] [node 1] ICP22 created new session: [5:7577789088081626401:2048]
2025-11-28T14:51:50.183949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088081626401:2048] [node 1] ICS09 handshake done sender: [5:7577789088081626392:2140] self: [5:7577789088081626399:0] peer: [1:7577789086240737485:0] socket: 248 qp: -1
2025-11-28T14:51:50.183977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088081626401:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.183995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.184211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789085610857884:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.184247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.184293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789085610857884:2050] [node 3] ICP22 created new session: [6:7577789085610857977:2050]
2025-11-28T14:51:50.184390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789085610857977:2050] [node 3] ICS09 handshake done sender: [6:7577789085610857963:2117] self: [6:7577789085610857968:0] peer: [3:7577789085837627661:0] socket: 305 qp: -1
2025-11-28T14:51:50.184434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789085610857977:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:50.184472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:50.184580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086240737488:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.184608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789086240737488:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.185826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857975:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
2025-11-28T14:51:50.186043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086240737488:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:50.186129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789085610857975:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.186360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789086240737478:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.186408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.186443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789086240737478:2109] [node 6] ICP22 created new session: [1:7577789086240737517:2109]
2025-11-28T14:51:50.186434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857979:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
2025-11-28T14:51:50.186481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086240737517:2109] [node 6] ICS09 handshake done sender: [1:7577789086240737488:2114] self: [1:7577789086240737487:0] peer: [6:7577789085610857976:0] socket: 250 qp: -1
2025-11-28T14:51:50.186515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086240737517:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:50.186542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:50.186816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789085610857943:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.186974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789085610857969:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.187096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088081626375:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.187122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789088081626375:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.187170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789085610857882:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.187518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.187594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789085610857882:2048] [node 1] ICP22 created new session: [6:7577789085610857981:2048]
2025-11-28T14:51:50.187641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789085610857981:2048] [node 1] ICS09 handshake done sender: [6:7577789085610857969:2119] self: [6:7577789085610857976:0] peer: [1:7577789086240737487:0] socket: 253 qp: -1
2025-11-28T14:51:50.187671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789085610857981:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.187710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.187751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088081626375:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:50.188002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789088081626367:2127] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.188047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.188088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088081626367:2127] [node 6] ICP22 created new session: [5:7577789088081626404:2127]
2025-11-28T14:51:50.188120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088081626404:2127] [node 6] ICS09 handshake done sender: [5:7577789088081626375:2132] self: [5:7577789088081626374:0] peer: [6:7577789085610857980:0] socket: 265 qp: -1
2025-11-28T14:51:50.188148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088081626404:2127] [node 6] ICS10 traffic start
2025-11-28T14:51:50.188165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:50.190683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789085610857943:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17777) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789085610857975:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.190878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789085610857983:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19503
2025-11-28T14:51:50.191247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789085610857975:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:50.191418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789085610857943:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.191458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.191493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789085610857943:2106] [node 5] ICP22 created new session: [6:7577789085610857984:2106]
2025-11-28T14:51:50.191538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789085610857984:2106] [node 5] ICS09 handshake done sender: [6:7577789085610857975:2122] self: [6:7577789085610857980:0] peer: [5:7577789088081626374:0] socket: 313 qp: -1
2025-11-28T14:51:50.191577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789085610857984:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:50.191614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.209414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:50.209648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789087157786576:2048] [node 1] ICS01 socket: 285 reason# EndOfStream
2025-11-28T14:51:50.209693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789082862819178:2048] [node 1] ICP30 unregister session Session# [2:7577789087157786576:2048] VirtualId# [2:7577789087157786571:0]
2025-11-28T14:51:50.209725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789087157786576:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.209834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.209892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.210469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:50.210721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789085837627684:2048] [node 1] ICS01 socket: 283 reason# EndOfStream
2025-11-28T14:51:50.210784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789081542660264:2048] [node 1] ICP30 unregister session Session# [3:7577789085837627684:2048] VirtualId# [3:7577789085837627678:0]
2025-11-28T14:51:50.210817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789085837627684:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.210922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.210981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.212407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:50.212620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789087349510979:2048] [node 1] ICS01 socket: 287 reason# EndOfStream
2025-11-28T14:51:50.212698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789087349510883:2048] [node 1] ICP30 unregister session Session# [4:7577789087349510979:2048] VirtualId# [4:7577789087349510972:0]
2025-11-28T14:51:50.212733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789087349510979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.212811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.212859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.213088Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:50.213302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789088081626401:2048] [node 1] ICS01 socket: 248 reason# EndOfStream
2025-11-28T14:51:50.213356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789088081626283:2048] [node 1] ICP30 unregister session Session# [5:7577789088081626401:2048] VirtualId# [5:7577789088081626399:0]
2025-11-28T14:51:50.213381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789088081626401:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.213535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.213584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.213875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:50.214106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789085610857981:2048] [node 1] ICS07 socket disconnect 253 reason# EndOfStream
2025-11-28T14:51:50.214146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789085610857981:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.214226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.214270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789085610857981:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:50.215093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789085610857882:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24397) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:50.215143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789085610857981:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:50.215169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789085610857882:2048] [node 1] ICP30 unregister session Session# [6:7577789085610857981:2048] VirtualId# [6:7577789085610857976:0]
2025-11-28T14:51:50.215207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.215308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789085610857882:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24397) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.215348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24397) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.225591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789085610857882:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:50.230789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:50.230994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789085837627679:2049] [node 2] ICS01 socket: 282 reason# EndOfStream
2025-11-28T14:51:50.231064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789081542660265:2049] [node 2] ICP30 unregister session Session# [3:7577789085837627679:2049] VirtualId# [3:7577789085837627672:0]
2025-11-28T14:51:50.231126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789085837627679:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.231235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:50.231294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:50.232761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:50.232993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789087349510974:2049] [node 2] ICS01 socket: 286 reason# EndOfStream
2025-11-28T14:51:50.233044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789087349510884:2049] [node 2] ICP30 unregister session Session# [4:7577789087349510974:2049] VirtualId# [4:7577789087349510967:0]
2025-11-28T14:51:50.233077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789087349510974:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.233162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:50.233217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:50.233369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:50.233573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789088081626390:2049] [node 2] ICS01 socket: 256 reason# EndOfStream
2025-11-28T14:51:50.233628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789088081626284:2049] [node 2] ICP30 unregister session Session# [5:7577789088081626390:2049] VirtualId# [5:7577789088081626384:0]
2025-11-28T14:51:50.233666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789088081626390:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.233739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:50.233786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:50.234037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:50.234179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789085610857966:2049] [node 2] ICS01 socket: 294 reason# EndOfStream
2025-11-28T14:51:50.234208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789085610857883:2049] [node 2] ICP30 unregister session Session# [6:7577789085610857966:2049] VirtualId# [6:7577789085610857962:0]
2025-11-28T14:51:50.234224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789085610857966:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.234281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:50.234321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:50.257302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:50.257544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789087349510970:2050] [node 3] ICS07 socket disconnect 257 reason# EndOfStream
2025-11-28T14:51:50.257582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789087349510970:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.257686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:50.257736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789087349510970:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:50.258382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:50.258608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789085610857977:2050] [node 3] ICS01 socket: 305 reason# EndOfStream
2025-11-28T14:51:50.258608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789087349510885:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:50.258662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789085610857884:2050] [node 3] ICP30 unregister session Session# [6:7577789085610857977:2050] VirtualId# [6:7577789085610857968:0]
2025-11-28T14:51:50.258668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789087349510970:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:50.258694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789085610857977:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.258696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789087349510885:2050] [node 3] ICP30 unregister session Session# [4:7577789087349510970:2050] VirtualId# [4:7577789087349510966:0]
2025-11-28T14:51:50.258774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:50.258748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:50.258832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:50.258788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789087349510885:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.258806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.259334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:50.259499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789088081626396:2050] [node 3] ICS01 socket: 268 reason# EndOfStream
2025-11-28T14:51:50.259534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789088081626285:2050] [node 3] ICP30 unregister session Session# [5:7577789088081626396:2050] VirtualId# [5:7577789088081626386:0]
2025-11-28T14:51:50.259561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789088081626396:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.259626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:50.259677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:50.284133Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:50.284414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789088081626394:2108] [node 4] ICS01 socket: 296 reason# EndOfStream
2025-11-28T14:51:50.284461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789088081626366:2108] [node 4] ICP30 unregister session Session# [5:7577789088081626394:2108] VirtualId# [5:7577789088081626385:0]
2025-11-28T14:51:50.284492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789088081626394:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.284575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:50.284655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:50.285218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:50.285403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789085610857973:2107] [node 4] ICS07 socket disconnect 300 reason# EndOfStream
2025-11-28T14:51:50.285437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789085610857973:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.285505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:50.285564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789085610857973:2107] [node 4] ICS15 start handshake
2025-11-28T14:51:50.286357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789085610857946:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10217) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:50.286419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789085610857973:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:50.286452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789085610857946:2107] [node 4] ICP30 unregister session Session# [6:7577789085610857973:2107] VirtualId# [6:7577789085610857965:0]
2025-11-28T14:51:50.286494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:50.286527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789085610857946:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10217) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.286548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10217) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.296221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789085610857946:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:51:50.310047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:50.310327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789085610857984:2106] [node 5] ICS01 socket: 313 reason# EndOfStream
2025-11-28T14:51:50.310380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789085610857943:2106] [node 5] ICP30 unregister session Session# [6:7577789085610857984:2106] VirtualId# [6:7577789085610857980:0]
2025-11-28T14:51:50.310415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789085610857984:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.310503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:50.310553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:50.416883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789086283391368:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:50.416937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789086283391369:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:50.438172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789085890407782:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:50.438216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789085890407783:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:50.460716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789086132566008:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:50.460753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789086132566009:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:50.481279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789085785265975:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:50.481318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789085785265976:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:50.481343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789085785265977:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:50.502944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088184511585:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:50.503010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088184511587:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:50.503023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088184511586:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:50.524680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789087461026027:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:50.524712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789087461026028:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:50.524774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789087461026029:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:50.600505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789087461026085:2104] [node 5] ICP01 ready to work
2025-11-28T14:51:50.637710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789086283391432:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:50.674497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789086283391433:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:50.693908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088184511648:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:50.694399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789085785266048:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:50.688601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789085890407845:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:50.698928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789087461026085:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(69178962ns) {Configure:104(69168559ns) {ProcessConfigured:135(84343ns) {StartInitialHandshake:157(80462ns) {DropHandshakes:470(3646ns) {DropIncomingHandshake:442(827ns) DropOutgoingHandshake:457(698ns)} GenerateSessionVirtualId:988(1276ns) PrepareNewSessionHandshake:477(1893ns) {SwitchToState:179(885ns)}}}}}
2025-11-28T14:51:50.693970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789086132566008:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(74423240ns) {Configure:104(74417322ns) {ProcessConfigured:135(117145ns) {StartInitialHandshake:157(113870ns) {DropHandshakes:470(3984ns) {DropIncomingHandshake:442(1034ns) DropOutgoingHandshake:457(862ns)} GenerateSessionVirtualId:988(1087ns) PrepareNewSessionHandshake:477(2713ns) {SwitchToState:179(1176ns)}}}}}
2025-11-28T14:51:50.695477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789085890407782:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(84308451ns) {Configure:104(84303045ns) {ProcessConfigured:135(71305ns) {StartInitialHandshake:157(68384ns) {DropHandshakes:470(2920ns) {DropIncomingHandshake:442(1174ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:477(866ns) {SwitchToState:179(221ns)}}}}}
2025-11-28T14:51:50.696947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789086283391434:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:50.701161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789088184511649:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:50.701126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789086132566073:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:50.705856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789085785266049:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:50.695798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789085890407848:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:50.709489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789087461026095:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:50.707064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789086132566074:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:50.709403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789086132566075:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:50.700630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789085890407849:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:50.715021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789088184511585:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(78115487ns) {Configure:104(78109396ns) {ProcessConfigured:135(53849ns) {StartInitialHandshake:157(51448ns) {DropHandshakes:470(2709ns) {DropIncomingHandshake:442(803ns) DropOutgoingHandshake:457(705ns)} GenerateSessionVirtualId:988(509ns) PrepareNewSessionHandshake:477(4041ns) {SwitchToState:179(313ns)}}}}}
2025-11-28T14:51:50.715385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789086283391369:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16909017ns) {Configure:104(16894690ns) {ProcessConfigured:135(1540298ns) {StartInitialHandshake:157(1536840ns) {DropHandshakes:470(2539ns) {DropIncomingHandshake:442(664ns) DropOutgoingHandshake:457(734ns)} GenerateSessionVirtualId:988(1038ns) PrepareNewSessionHandshake:477(1834ns) {SwitchToState:179(843ns)}}}}}
2025-11-28T14:51:50.716415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789085890407783:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20031255ns) {Configure:104(20027628ns) {ProcessConfigured:135(479829ns) {StartInitialHandshake:157(477317ns) {DropHandshakes:470(1217ns) {DropIncomingHandshake:442(371ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(1867ns) {SwitchToState:179(1086ns)}}}}}
2025-11-28T14:51:50.719380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789086283391368:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26171305ns) {Configure:104(26159581ns) {ProcessConfigured:135(1970073ns) {StartInitialHandshake:157(1967477ns) {DropHandshakes:470(905ns) {DropIncomingHandshake:442(205ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:477(3487ns) {SwitchToState:179(1103ns)}}}}}
2025-11-28T14:51:50.719453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789086132566009:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(89693401ns) {Configure:104(89677242ns) {ProcessConfigured:135(375552ns) {StartInitialHandshake:157(373099ns) {DropHandshakes:470(2175ns) {DropIncomingHandshake:442(739ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(590ns) PrepareNewSessionHandshake:477(1584ns) {SwitchToState:179(914ns)}}}}}
2025-11-28T14:51:50.721036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789085785265975:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(45828857ns) {Configure:104(45823512ns) {ProcessConfigured:135(715787ns) {StartInitialHandshake:157(713705ns) {DropHandshakes:470(11046ns) {DropIncomingHandshake:442(441ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(632ns) PrepareNewSessionHandshake:477(1880ns) {SwitchToState:179(733ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.735536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789085890407861:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38738
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.730972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088184511660:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8968
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.736022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789085890407862:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38738
2025-11-28T14:51:50.734932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086283391448:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13129
2025-11-28T14:51:50.736096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789085890407863:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38738
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.740861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086283391449:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13129
2025-11-28T14:51:50.743083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789086132566087:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-11-28T14:51:50.743857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789086132566088:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.743940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789086132566089:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-11-28T14:51:50.736168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789085890407864:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38738
2025-11-28T14:51:50.744214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789086132566087:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.739793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789085890407861:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.745461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789086132566009:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.746178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789086132566089:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.745800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789085890407862:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.745811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088184511660:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.745499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391449:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:50.746456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789085890407851:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:51:50.746215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086283391450:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13129
2025-11-28T14:51:50.746493Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789085890407851:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.747688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086283391451:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13129
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.747364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789085890407864:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.748934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789085785266058:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30988
2025-11-28T14:51:50.749865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789085785266059:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30988
2025-11-28T14:51:50.748645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391450:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.749697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789085890407851:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:50.749058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391448:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.750793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088184511665:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8968
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.751815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789085785266058:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.749287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391451:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.751915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789085785266062:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30988
2025-11-28T14:51:50.752859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789085890407783:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.751926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789085785266059:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.752108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789085785265976:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.752928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.753997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789085890407783:2049] [node 3] ICP22 created new session: [2:7577789085890407869:2049]
No RDMA context found for GID: 2025-11-28T14:51:50.754234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789085890407869:2049] [node 3] ICS09 handshake done sender: [2:7577789085890407851:2111] self: [2:7577789085890407850:0] peer: [3:7577789086132566090:0] socket: 256 qp: -1
::ffff:127.1.0.0
2025-11-28T14:51:50.754363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789085890407869:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:50.753516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789085785265977:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.756568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789086132566079:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.754425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:50.754564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789085890407853:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.756612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789086132566079:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.754587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789085890407853:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.759221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789086132566092:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.759396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789086132566093:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-11-28T14:51:50.760052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789085785266068:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30988
2025-11-28T14:51:50.765420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789085890407853:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:50.766393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789086132566079:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:50.768721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789085890407845:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.768777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.769129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789085890407845:2107] [node 4] ICP22 created new session: [2:7577789085890407873:2107]
2025-11-28T14:51:50.769209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789085890407873:2107] [node 4] ICS09 handshake done sender: [2:7577789085890407853:2112] self: [2:7577789085890407852:0] peer: [4:7577789085785266066:0] socket: 257 qp: -1
2025-11-28T14:51:50.769257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789085890407873:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.769305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.769385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026105:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
2025-11-28T14:51:50.769183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789086132566009:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21143) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789086132566087:2115] held: no
2025-11-28T14:51:50.769261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789086132566073:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.769352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.769382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789086132566087:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:50.769434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789086132566073:2107] [node 4] ICP22 created new session: [3:7577789086132566095:2107]
2025-11-28T14:51:50.769484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789086132566095:2107] [node 4] ICS09 handshake done sender: [3:7577789086132566079:2112] self: [3:7577789086132566078:0] peer: [4:7577789085785266067:0] socket: 266 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.769569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789086132566095:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:50.770021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789087461026105:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.769603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.769999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789086132566009:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.770035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:50.770093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789086132566009:2049] [node 2] ICP22 created new session: [3:7577789086132566097:2049]
2025-11-28T14:51:50.770136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789086132566097:2049] [node 2] ICS09 handshake done sender: [3:7577789086132566087:2115] self: [3:7577789086132566090:0] peer: [2:7577789085890407850:0] socket: 267 qp: -1
2025-11-28T14:51:50.770162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789086132566097:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:50.770195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:50.770306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789087461026028:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.770152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088184511666:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8968
2025-11-28T14:51:50.770607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789085890407855:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.770640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789085890407855:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.772272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789085890407855:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:50.772299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088184511667:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8968
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.772385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026107:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
2025-11-28T14:51:50.772594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789085890407848:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.772929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026108:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
2025-11-28T14:51:50.772821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088184511667:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.772823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.772993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789085890407848:2109] [node 6] ICP22 created new session: [2:7577789085890407877:2109]
2025-11-28T14:51:50.773277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789085890407877:2109] [node 6] ICS09 handshake done sender: [2:7577789085890407855:2113] self: [2:7577789085890407854:0] peer: [6:7577789087461026106:0] socket: 271 qp: -1
2025-11-28T14:51:50.773188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789088184511587:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.773323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789085890407877:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:50.773556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789086132566081:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.773366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:50.773705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789087461026105:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:50.773596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789086132566081:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.774359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088184511666:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.774900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789088184511586:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.775136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789087461026028:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21143) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789087461026105:2115] held: no
2025-11-28T14:51:50.775263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789085890407857:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.775023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789086132566081:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:50.775293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789085890407857:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.775266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789087461026028:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.775319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789086132566074:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.775556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088184511670:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8968
2025-11-28T14:51:50.775303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:50.775368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.775517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789086132566074:2108] [node 5] ICP22 created new session: [3:7577789086132566102:2108]
2025-11-28T14:51:50.775456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789087461026028:2049] [node 2] ICP22 created new session: [6:7577789087461026109:2049]
2025-11-28T14:51:50.775917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789086132566102:2108] [node 5] ICS09 handshake done sender: [3:7577789086132566081:2113] self: [3:7577789086132566080:0] peer: [5:7577789088184511668:0] socket: 302 qp: -1
2025-11-28T14:51:50.775926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789087461026109:2049] [node 2] ICS09 handshake done sender: [6:7577789087461026105:2115] self: [6:7577789087461026106:0] peer: [2:7577789085890407854:0] socket: 299 qp: -1
2025-11-28T14:51:50.776016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789086132566102:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:50.776041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789087461026109:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:50.776054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.776072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:50.776594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789085890407857:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:50.776100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789087461026107:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.776952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789087461026029:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.777065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088184511667:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:50.777194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789085890407849:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.777250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.777288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789085890407849:2108] [node 5] ICP22 created new session: [2:7577789085890407880:2108]
2025-11-28T14:51:50.777319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789085890407880:2108] [node 5] ICS09 handshake done sender: [2:7577789085890407857:2114] self: [2:7577789085890407856:0] peer: [5:7577789088184511669:0] socket: 263 qp: -1
2025-11-28T14:51:50.777401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789085890407880:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:50.777664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789088184511587:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.777429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.777711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.777751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088184511587:2050] [node 3] ICP22 created new session: [5:7577789088184511672:2050]
2025-11-28T14:51:50.777754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789086132566083:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.777784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789086132566083:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:50.777794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088184511672:2050] [node 3] ICS09 handshake done sender: [5:7577789088184511667:2118] self: [5:7577789088184511668:0] peer: [3:7577789086132566080:0] socket: 303 qp: -1
2025-11-28T14:51:50.778055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088184511672:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.778270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:50.779235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789086132566104:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-11-28T14:51:50.779257Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789086132566083:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.779340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789088184511587:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.779531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088184511671:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8968
2025-11-28T14:51:50.779557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789086132566075:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.779577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026113:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
2025-11-28T14:51:50.779601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.779644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789086132566075:2109] [node 6] ICP22 created new session: [3:7577789086132566106:2109]
2025-11-28T14:51:50.779719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789086132566106:2109] [node 6] ICS09 handshake done sender: [3:7577789086132566083:2114] self: [3:7577789086132566082:0] peer: [6:7577789087461026112:0] socket: 304 qp: -1
2025-11-28T14:51:50.779757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789086132566106:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:50.779756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789086132566104:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.779771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789087461026107:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:50.779783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:50.780202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789087461026029:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.780274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.780396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789088184511586:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21143) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789088184511666:2117] held: no
2025-11-28T14:51:50.780435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789087461026029:2050] [node 3] ICP22 created new session: [6:7577789087461026114:2050]
2025-11-28T14:51:50.780497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789087461026114:2050] [node 3] ICS09 handshake done sender: [6:7577789087461026107:2116] self: [6:7577789087461026112:0] peer: [3:7577789086132566082:0] socket: 305 qp: -1
2025-11-28T14:51:50.780527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789087461026114:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:50.780552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:50.781163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088184511666:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:50.781102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789086132566075:2109] [node 6] ICP18 handshake (actor [3:7577789086132566104:2120]) is too old
2025-11-28T14:51:50.781937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789088184511586:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.783176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:50.783251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088184511586:2049] [node 2] ICP22 created new session: [5:7577789088184511675:2049]
2025-11-28T14:51:50.783296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088184511675:2049] [node 2] ICS09 handshake done sender: [5:7577789088184511666:2117] self: [5:7577789088184511669:0] peer: [2:7577789085890407856:0] socket: 301 qp: -1
2025-11-28T14:51:50.783333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088184511675:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:50.783402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.783831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026117:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
2025-11-28T14:51:50.785400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789087461026117:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.785550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789087461026085:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.785769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088184511659:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.785799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789088184511659:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.786519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088184511659:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:50.786607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026119:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
2025-11-28T14:51:50.786759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789088184511649:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.786836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789087461026117:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:50.786853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.786914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088184511649:2109] [node 6] ICP22 created new session: [5:7577789088184511679:2109]
2025-11-28T14:51:50.787144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088184511679:2109] [node 6] ICS09 handshake done sender: [5:7577789088184511659:2114] self: [5:7577789088184511658:0] peer: [6:7577789087461026118:0] socket: 310 qp: -1
2025-11-28T14:51:50.787181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088184511679:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:50.787148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789087461026085:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.787217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:50.787195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.787296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789087461026085:2104] [node 5] ICP22 created new session: [6:7577789087461026120:2104]
2025-11-28T14:51:50.787414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789087461026120:2104] [node 5] ICS09 handshake done sender: [6:7577789087461026117:2119] self: [6:7577789087461026118:0] peer: [5:7577789088184511658:0] socket: 311 qp: -1
2025-11-28T14:51:50.787467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789087461026120:2104] [node 5] ICS10 traffic start
2025-11-28T14:51:50.787508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.791210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789086283391438:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:50.791944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789085890407863:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.792120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789085890407782:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.813527Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789086283391436:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.822652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789086283391454:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13129
2025-11-28T14:51:50.822792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789086132566088:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.822891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391438:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.823036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789086132566008:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.823236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391436:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.823472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086283391438:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.823604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789085890407883:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38738
2025-11-28T14:51:50.823936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789085890407863:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.824116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789086283391368:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.824193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:50.824235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789085890407782:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.824244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789086283391368:2048] [node 2] ICP22 created new session: [1:7577789086283391459:2048]
2025-11-28T14:51:50.824334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086283391459:2048] [node 2] ICS09 handshake done sender: [1:7577789086283391438:2112] self: [1:7577789086283391437:0] peer: [2:7577789085890407882:0] socket: 270 qp: -1
2025-11-28T14:51:50.824348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.824412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086283391459:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:50.824414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789085890407782:2048] [node 1] ICP22 created new session: [2:7577789085890407884:2048]
2025-11-28T14:51:50.824453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:50.824453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789085890407884:2048] [node 1] ICS09 handshake done sender: [2:7577789085890407863:2117] self: [2:7577789085890407882:0] peer: [1:7577789086283391437:0] socket: 274 qp: -1
2025-11-28T14:51:50.824480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789085890407884:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.824506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789086132566109:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-11-28T14:51:50.824527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.826490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086283391436:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:50.826721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789086132566088:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.826918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789086132566008:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.826968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.827017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789086132566008:2048] [node 1] ICP22 created new session: [3:7577789086132566110:2048]
2025-11-28T14:51:50.827021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789086283391369:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.827066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789086132566110:2048] [node 1] ICS09 handshake done sender: [3:7577789086132566088:2116] self: [3:7577789086132566108:0] peer: [1:7577789086283391435:0] socket: 277 qp: -1
2025-11-28T14:51:50.827077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.827095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789086132566110:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.827127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789086283391369:2049] [node 3] ICP22 created new session: [1:7577789086283391462:2049]
2025-11-28T14:51:50.827154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.827181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086283391462:2049] [node 3] ICS09 handshake done sender: [1:7577789086283391436:2111] self: [1:7577789086283391435:0] peer: [3:7577789086132566108:0] socket: 269 qp: -1
2025-11-28T14:51:50.827217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086283391462:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:50.827247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.832564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026122:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.833347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088184511681:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8968
2025-11-28T14:51:50.840252Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789085785266053:2121] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:50.848697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789086132566093:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.848906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789086132566073:2107] [node 4] ICP18 handshake (actor [3:7577789086132566093:2119]) is too old
2025-11-28T14:51:50.862871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789085785266056:2122] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:50.871302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088184511665:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.871515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789088184511648:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.872090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789085785266059:2125] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.872417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789085785266069:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30988
2025-11-28T14:51:50.872887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789085785265977:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24064) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789085785266059:2125] held: no
2025-11-28T14:51:50.872990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789085785265977:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.873041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:50.873089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789085785265977:2050] [node 3] ICP22 created new session: [4:7577789085785266072:2050]
2025-11-28T14:51:50.873130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789085785266072:2050] [node 3] ICS09 handshake done sender: [4:7577789085785266059:2125] self: [4:7577789085785266067:0] peer: [3:7577789086132566078:0] socket: 282 qp: -1
2025-11-28T14:51:50.873177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789085785266072:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:50.873224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:50.873721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789085785266056:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.873867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789085785266070:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30988
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.874006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789085785266071:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30988
2025-11-28T14:51:50.874283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789085785266058:2124] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:50.874785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789085785266070:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.874912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789085785266071:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.874952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789085785265976:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.874991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.875031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789085785265976:2049] [node 2] ICP22 created new session: [4:7577789085785266076:2049]
2025-11-28T14:51:50.875071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789085785266076:2049] [node 2] ICS09 handshake done sender: [4:7577789085785266058:2124] self: [4:7577789085785266066:0] peer: [2:7577789085890407852:0] socket: 280 qp: -1
2025-11-28T14:51:50.875103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789085785266076:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:50.875136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:50.875086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088184511683:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8968
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.876178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789085785266056:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:50.876221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789085890407886:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38738
2025-11-28T14:51:50.876332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088184511665:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:50.876617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789088184511648:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.876766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.876812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088184511648:2108] [node 4] ICP22 created new session: [5:7577789088184511684:2108]
2025-11-28T14:51:50.876855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088184511684:2108] [node 4] ICS09 handshake done sender: [5:7577789088184511665:2116] self: [5:7577789088184511682:0] peer: [4:7577789085785266054:0] socket: 293 qp: -1
2025-11-28T14:51:50.876885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088184511684:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:50.876928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.876981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789085785266048:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.877015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.877051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789085785266048:2108] [node 5] ICP22 created new session: [4:7577789085785266079:2108]
2025-11-28T14:51:50.877133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789085785266079:2108] [node 5] ICS09 handshake done sender: [4:7577789085785266056:2122] self: [4:7577789085785266054:0] peer: [5:7577789088184511682:0] socket: 291 qp: -1
2025-11-28T14:51:50.877171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789085785266079:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:50.877214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.878308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789085785266048:2108] [node 5] ICP18 handshake (actor [4:7577789085785266071:2131]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.879835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026123:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
2025-11-28T14:51:50.889399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789086283391440:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:50.916201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789086283391442:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:50.924874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789087461026122:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.925152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789087461026027:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.925402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391442:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.925745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086283391442:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.926717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789086283391433:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.926733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026125:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
2025-11-28T14:51:50.926797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.926857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789086283391433:2109] [node 6] ICP22 created new session: [1:7577789086283391466:2109]
2025-11-28T14:51:50.926908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086283391466:2109] [node 6] ICS09 handshake done sender: [1:7577789086283391442:2114] self: [1:7577789086283391441:0] peer: [6:7577789087461026124:0] socket: 255 qp: -1
2025-11-28T14:51:50.926952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086283391466:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:50.926994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:50.927232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789087461026122:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.927887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789087461026027:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.927956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.928023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789087461026027:2048] [node 1] ICP22 created new session: [6:7577789087461026126:2048]
2025-11-28T14:51:50.928088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789087461026126:2048] [node 1] ICS09 handshake done sender: [6:7577789087461026122:2121] self: [6:7577789087461026124:0] peer: [1:7577789086283391441:0] socket: 258 qp: -1
2025-11-28T14:51:50.928140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789087461026126:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.928178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.939580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789085785266051:2120] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:50.946910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789085785266062:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.946931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391454:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.947074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789085785265975:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.947286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391440:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.947707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086283391440:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.948002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789085785266083:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30988
2025-11-28T14:51:50.948066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789086283391432:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.948131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.948187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789086283391432:2107] [node 4] ICP22 created new session: [1:7577789086283391470:2107]
2025-11-28T14:51:50.948242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086283391470:2107] [node 4] ICS09 handshake done sender: [1:7577789086283391440:2113] self: [1:7577789086283391439:0] peer: [4:7577789085785266082:0] socket: 273 qp: -1
2025-11-28T14:51:50.948285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086283391470:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:50.948324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.948522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789085785266062:2127] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.948763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789085785265975:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.948819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.948865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789085785265975:2048] [node 1] ICP22 created new session: [4:7577789085785266084:2048]
2025-11-28T14:51:50.948914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789085785266084:2048] [node 1] ICS09 handshake done sender: [4:7577789085785266062:2127] self: [4:7577789085785266082:0] peer: [1:7577789086283391439:0] socket: 283 qp: -1
2025-11-28T14:51:50.948964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789085785266084:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.948998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.951555Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789085785266061:2126] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:50.960114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789087461026123:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.960353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789087461026095:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.960622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789085785266061:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.961101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789085785266061:2126] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.961341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789085785266049:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.961525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:50.961608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789085785266049:2110] [node 6] ICP22 created new session: [4:7577789085785266087:2110]
2025-11-28T14:51:50.961601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789087461026129:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21596
2025-11-28T14:51:50.961666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789085785266087:2110] [node 6] ICS09 handshake done sender: [4:7577789085785266061:2126] self: [4:7577789085785266060:0] peer: [6:7577789087461026128:0] socket: 279 qp: -1
2025-11-28T14:51:50.961718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789085785266087:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:50.961759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:50.961858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789087461026123:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:50.962370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789087461026095:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.962443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:50.962487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789087461026095:2108] [node 4] ICP22 created new session: [6:7577789087461026130:2108]
2025-11-28T14:51:50.962559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789087461026130:2108] [node 4] ICS09 handshake done sender: [6:7577789087461026123:2122] self: [6:7577789087461026128:0] peer: [4:7577789085785266060:0] socket: 292 qp: -1
2025-11-28T14:51:50.962603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789087461026130:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:50.962691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:50.962974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789087461026095:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:50.964225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789086283391446:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:50.964861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789088184511681:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.965056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789088184511585:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:50.965294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789086283391446:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:50.965712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789086283391446:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:50.965929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789086283391434:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:50.965902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789088184511687:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8968
2025-11-28T14:51:50.965992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:50.966043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789086283391434:2108] [node 5] ICP22 created new session: [1:7577789086283391473:2108]
2025-11-28T14:51:50.966108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789086283391473:2108] [node 5] ICS09 handshake done sender: [1:7577789086283391446:2115] self: [1:7577789086283391443:0] peer: [5:7577789088184511686:0] socket: 264 qp: -1
2025-11-28T14:51:50.966138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789086283391473:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:50.966159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:50.966166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789088184511681:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:50.966442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789088184511585:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:50.966484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:50.966514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789088184511585:2048] [node 1] ICP22 created new session: [5:7577789088184511688:2048]
2025-11-28T14:51:50.966594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789088184511688:2048] [node 1] ICS09 handshake done sender: [5:7577789088184511681:2121] self: [5:7577789088184511686:0] peer: [1:7577789086283391443:0] socket: 268 qp: -1
2025-11-28T14:51:50.966619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789088184511688:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:50.966648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:50.970396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:50.970631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789085890407884:2048] [node 1] ICS01 socket: 274 reason# EndOfStream
2025-11-28T14:51:50.970674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789085890407782:2048] [node 1] ICP30 unregister session Session# [2:7577789085890407884:2048] VirtualId# [2:7577789085890407882:0]
2025-11-28T14:51:50.970708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789085890407884:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.970788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.970847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.971587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:50.971785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789086132566110:2048] [node 1] ICS01 socket: 277 reason# EndOfStream
2025-11-28T14:51:50.971823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789086132566008:2048] [node 1] ICP30 unregister session Session# [3:7577789086132566110:2048] VirtualId# [3:7577789086132566108:0]
2025-11-28T14:51:50.971851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789086132566110:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.971929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.971970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.973800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:50.974042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789085785266084:2048] [node 1] ICS01 socket: 283 reason# EndOfStream
2025-11-28T14:51:50.974097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789085785265975:2048] [node 1] ICP30 unregister session Session# [4:7577789085785266084:2048] VirtualId# [4:7577789085785266082:0]
2025-11-28T14:51:50.974161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789085785266084:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.974291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.974344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.974688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:50.974921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789088184511688:2048] [node 1] ICS07 socket disconnect 268 reason# EndOfStream
2025-11-28T14:51:50.974957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789088184511688:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.975030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.975085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789088184511688:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:50.975633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:50.975793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789088184511585:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18739) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:50.975842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789087461026126:2048] [node 1] ICS07 socket disconnect 258 reason# EndOfStream
2025-11-28T14:51:50.975836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789088184511688:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:50.975921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789088184511585:2048] [node 1] ICP30 unregister session Session# [5:7577789088184511688:2048] VirtualId# [5:7577789088184511686:0]
2025-11-28T14:51:50.975973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.975881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789087461026126:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:50.976004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789088184511585:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18739) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.975958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:50.976018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18739) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.976061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789087461026126:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:50.976667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789087461026027:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18739) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:50.976718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789087461026126:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:50.976750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789087461026027:2048] [node 1] ICP30 unregister session Session# [6:7577789087461026126:2048] VirtualId# [6:7577789087461026124:0]
2025-11-28T14:51:50.976805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:50.976844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789087461026027:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18739) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.976857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18739) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:50.985485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789088184511585:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:50.986380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789087461026027:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:50.999624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:50.999834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789086132566097:2049] [node 2] ICS01 socket: 267 reason# EndOfStream
2025-11-28T14:51:50.999893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789086132566009:2049] [node 2] ICP30 unregister session Session# [3:7577789086132566097:2049] VirtualId# [3:7577789086132566090:0]
2025-11-28T14:51:50.999931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789086132566097:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.000019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:51.000094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:51.002536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:51.002774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789085785266076:2049] [node 2] ICS07 socket disconnect 280 reason# EndOfStream
2025-11-28T14:51:51.002815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789085785266076:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.002921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:51.002994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789085785266076:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:51.003519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:51.003757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789088184511675:2049] [node 2] ICS01 socket: 301 reason# EndOfStream
2025-11-28T14:51:51.003859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789088184511586:2049] [node 2] ICP30 unregister session Session# [5:7577789088184511675:2049] VirtualId# [5:7577789088184511669:0]
2025-11-28T14:51:51.003783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789085785265976:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21143) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.003892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789088184511675:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.003911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789085785266076:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.003971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:51.003948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789085785265976:2049] [node 2] ICP30 unregister session Session# [4:7577789085785266076:2049] VirtualId# [4:7577789085785266066:0]
2025-11-28T14:51:51.003998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:51.004036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:51.004041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789085785265976:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21143) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.004061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21143) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.004506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:51.004725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789087461026109:2049] [node 2] ICS01 socket: 299 reason# EndOfStream
2025-11-28T14:51:51.004766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789087461026028:2049] [node 2] ICP30 unregister session Session# [6:7577789087461026109:2049] VirtualId# [6:7577789087461026106:0]
2025-11-28T14:51:51.004794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789087461026109:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.004860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:51.004952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:51.014075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789085785265976:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:51.030706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:51.030995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789085785266072:2050] [node 3] ICS01 socket: 282 reason# EndOfStream
2025-11-28T14:51:51.031040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789085785265977:2050] [node 3] ICP30 unregister session Session# [4:7577789085785266072:2050] VirtualId# [4:7577789085785266067:0]
2025-11-28T14:51:51.031072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789085785266072:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.031269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:51.031371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:51.031994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:51.032128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789088184511672:2050] [node 3] ICS07 socket disconnect 303 reason# EndOfStream
2025-11-28T14:51:51.032149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789088184511672:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.032221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:51.032264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789088184511672:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:51.032989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:51.033025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789088184511587:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.033094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789088184511672:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.033132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789088184511587:2050] [node 3] ICP30 unregister session Session# [5:7577789088184511672:2050] VirtualId# [5:7577789088184511668:0]
2025-11-28T14:51:51.033182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:51.033216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789088184511587:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.033372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789087461026114:2050] [node 3] ICS07 socket disconnect 305 reason# EndOfStream
2025-11-28T14:51:51.033231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.033418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789087461026114:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.033490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:51.033590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789087461026114:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:51.034170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789087461026029:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.034205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789087461026114:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.034230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789087461026029:2050] [node 3] ICP30 unregister session Session# [6:7577789087461026114:2050] VirtualId# [6:7577789087461026112:0]
2025-11-28T14:51:51.034256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:51.034279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789087461026029:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.034288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.040660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:51.040876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789088184511684:2108] [node 4] ICS01 socket: 293 reason# EndOfStream
2025-11-28T14:51:51.040916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789088184511648:2108] [node 4] ICP30 unregister session Session# [5:7577789088184511684:2108] VirtualId# [5:7577789088184511682:0]
2025-11-28T14:51:51.040940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789088184511684:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.041042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:51.041082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:51.041381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:51.041526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789087461026130:2108] [node 4] ICS07 socket disconnect 292 reason# EndOfStream
2025-11-28T14:51:51.041561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789087461026130:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.041627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:51.041673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789087461026130:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:51.042500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789087461026095:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3193) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.042554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789087461026130:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.042589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789087461026095:2108] [node 4] ICP30 unregister session Session# [6:7577789087461026130:2108] VirtualId# [6:7577789087461026128:0]
2025-11-28T14:51:51.042645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:51.042675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789087461026095:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3193) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.042689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3193) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.042865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789088184511587:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:51.043723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789087461026029:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:51.053059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789087461026095:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:51.095987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:51.096285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789087461026120:2104] [node 5] ICS07 socket disconnect 311 reason# EndOfStream
2025-11-28T14:51:51.096326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789087461026120:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.096482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:51.096553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789087461026120:2104] [node 5] ICS15 start handshake
2025-11-28T14:51:51.097376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789087461026085:2104] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2083) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.100730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789087461026120:2104] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.100813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789087461026085:2104] [node 5] ICP30 unregister session Session# [6:7577789087461026120:2104] VirtualId# [6:7577789087461026118:0]
2025-11-28T14:51:51.100925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:51.101026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789087461026085:2104] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2083) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.101058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2083) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:51.187002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789092477658589:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:51.187036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789092477658590:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:51.214780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789090570057356:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:51.215010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789090570057355:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:51.237289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789090637048111:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:51.237333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789090637048112:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:51.259557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789091816341422:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:51.259619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789091816341423:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:51.259653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789091816341424:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:51.282806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789092452088014:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:51.282851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789092452088015:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:51.282882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789092452088016:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:51.305074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789092024931158:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:51.305118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789092024931159:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:51.305151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789092024931160:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:51.416906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789092024931219:2105] [node 5] ICP01 ready to work
2025-11-28T14:51:51.417086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789092477658662:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:51.525199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789090570057355:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(112437391ns) {Configure:104(112428283ns) {ProcessConfigured:135(85927ns) {StartInitialHandshake:157(82226ns) {DropHandshakes:470(2350ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(329ns)} GenerateSessionVirtualId:988(1307ns) PrepareNewSessionHandshake:477(1760ns) {SwitchToState:179(860ns)}}}}}
2025-11-28T14:51:51.525677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789090570057425:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:51.514972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789092477658670:2120] [node 6] ICP01 ready to work
2025-11-28T14:51:51.528753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789091816341494:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:51.526916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789092024931222:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:51.531717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789092477658669:2112] [node 5] ICP01 ready to work
2025-11-28T14:51:51.536703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789090570057426:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:51.541855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789090637048174:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:51.541937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789091816341495:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:51.541231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789090570057427:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:51.544330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789092452088096:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:51.543717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789090637048175:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:51.544765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789092452088098:2123] [node 6] ICP01 ready to work
2025-11-28T14:51:51.548572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789092452088015:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(86319182ns) {Configure:104(86314165ns) {ProcessConfigured:135(3551593ns) {StartInitialHandshake:157(3547172ns) {DropHandshakes:470(4157ns) {DropIncomingHandshake:442(1029ns) DropOutgoingHandshake:457(770ns)} GenerateSessionVirtualId:988(1688ns) PrepareNewSessionHandshake:477(2381ns) {SwitchToState:179(817ns)}}}}}
2025-11-28T14:51:51.550128Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789092452088016:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(88790206ns) {Configure:104(88772205ns) {ProcessConfigured:135(1475248ns) {StartInitialHandshake:157(1472309ns) {DropHandshakes:470(2398ns) {DropIncomingHandshake:442(664ns) DropOutgoingHandshake:457(504ns)} GenerateSessionVirtualId:988(1107ns) PrepareNewSessionHandshake:477(1859ns) {SwitchToState:179(178ns)}}}}}
2025-11-28T14:51:51.549765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789090637048176:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:51.557137Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789090570057356:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25320386ns) {Configure:104(20287400ns) {ProcessConfigured:135(64072ns) {StartInitialHandshake:157(61075ns) {DropHandshakes:470(1724ns) {DropIncomingHandshake:442(597ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(469ns) PrepareNewSessionHandshake:477(1206ns) {SwitchToState:179(328ns)}}}}}
2025-11-28T14:51:51.571354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789092024931158:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50170838ns) {Configure:104(50162150ns) {ProcessConfigured:135(1009665ns) {StartInitialHandshake:157(1006767ns) {DropHandshakes:470(2085ns) {DropIncomingHandshake:442(514ns) DropOutgoingHandshake:457(375ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:477(5302ns) {SwitchToState:179(4007ns)}}}}}
2025-11-28T14:51:51.572309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789092477658589:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(151357025ns) {Configure:104(151348316ns) {ProcessConfigured:135(4484062ns) {StartInitialHandshake:157(4478378ns) {DropHandshakes:470(2726ns) {DropIncomingHandshake:442(563ns) DropOutgoingHandshake:457(477ns)} GenerateSessionVirtualId:988(721ns) PrepareNewSessionHandshake:477(2958ns) {SwitchToState:179(1381ns)}}}}}
2025-11-28T14:51:51.573068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789092477658590:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33708116ns) {Configure:104(33701409ns) {ProcessConfigured:135(5959127ns) {StartInitialHandshake:157(5956576ns) {DropHandshakes:470(3622501ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(2625ns)} GenerateSessionVirtualId:988(3029ns) PrepareNewSessionHandshake:477(2799ns) {SwitchToState:179(1464ns)}}}}}
2025-11-28T14:51:51.573015Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789092024931160:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(50396806ns) {Configure:104(50391369ns) {ProcessConfigured:135(4741981ns) {StartInitialHandshake:157(4738585ns) {DropHandshakes:470(2573ns) {DropIncomingHandshake:442(693ns) DropOutgoingHandshake:457(407ns)} GenerateSessionVirtualId:988(910ns) PrepareNewSessionHandshake:477(2526ns) {SwitchToState:179(1002ns)}}}}}
2025-11-28T14:51:51.574894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789090570057426:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26890159ns) {Configure:104(26880255ns) {ProcessConfigured:135(4500253ns) {StartInitialHandshake:157(3271787ns) {DropHandshakes:470(3022ns) {DropIncomingHandshake:442(547ns) DropOutgoingHandshake:457(386ns)} GenerateSessionVirtualId:988(1596ns) PrepareNewSessionHandshake:477(2624ns) {SwitchToState:179(461ns)}}}}}
2025-11-28T14:51:51.574249Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789092024931219:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(40906651ns) {Configure:104(40898829ns) {ProcessConfigured:135(3075706ns) {StartInitialHandshake:157(3074127ns) {DropHandshakes:470(2276ns) {DropIncomingHandshake:442(469ns) DropOutgoingHandshake:457(366ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(2193ns) {SwitchToState:179(525ns)}}}}}
2025-11-28T14:51:51.574926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789090570057425:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17443140ns) {Configure:104(17438146ns) {ProcessConfigured:135(1623962ns) {StartInitialHandshake:157(1326766ns) {DropHandshakes:470(1944ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(1343ns) PrepareNewSessionHandshake:477(1574ns) {SwitchToState:179(699ns)}}}}}
2025-11-28T14:51:51.573741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789092477658662:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29133787ns) {Configure:104(29126307ns) {ProcessConfigured:135(5636975ns) {StartInitialHandshake:157(5634200ns) {DropHandshakes:470(2547ns) {DropIncomingHandshake:442(518ns) DropOutgoingHandshake:457(387ns)} GenerateSessionVirtualId:988(841ns) PrepareNewSessionHandshake:477(4229ns) {SwitchToState:179(1676ns)}}}}}
2025-11-28T14:51:51.576354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789091816341423:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31725162ns) {Configure:104(31717685ns) {ProcessConfigured:135(1281951ns) {StartInitialHandshake:157(1279846ns) {DropHandshakes:470(1303ns) {DropIncomingHandshake:442(366ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(577ns) PrepareNewSessionHandshake:477(2004ns) {SwitchToState:179(989ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.579231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931229:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
2025-11-28T14:51:51.577675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789091816341424:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(54160688ns) {Configure:104(54149715ns) {ProcessConfigured:135(1830804ns) {StartInitialHandshake:157(1828480ns) {DropHandshakes:470(1050ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:477(2422ns) {SwitchToState:179(1169ns)}}}}}
2025-11-28T14:51:51.582276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789090637048174:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27418572ns) {Configure:104(27414861ns) {ProcessConfigured:135(700148ns) {StartInitialHandshake:157(693057ns) {DropHandshakes:470(1642ns) {DropIncomingHandshake:442(471ns) DropOutgoingHandshake:457(318ns)} GenerateSessionVirtualId:988(760ns) PrepareNewSessionHandshake:477(1349ns) {SwitchToState:179(624ns)}}}}}
2025-11-28T14:51:51.578563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789091816341422:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33774766ns) {Configure:104(33767339ns) {ProcessConfigured:135(1993606ns) {StartInitialHandshake:157(1990999ns) {DropHandshakes:470(2097ns) {DropIncomingHandshake:442(602ns) DropOutgoingHandshake:457(366ns)} GenerateSessionVirtualId:988(10543ns) PrepareNewSessionHandshake:477(2390ns) {SwitchToState:179(834ns)}}}}}
2025-11-28T14:51:51.582377Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789090637048112:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28361460ns) {Configure:104(28354671ns) {ProcessConfigured:135(726607ns) {StartInitialHandshake:157(723760ns) {DropHandshakes:470(3139ns) {DropIncomingHandshake:442(996ns) DropOutgoingHandshake:457(753ns)} GenerateSessionVirtualId:988(1211ns) PrepareNewSessionHandshake:477(2160ns) {SwitchToState:179(975ns)}}}}}
2025-11-28T14:51:51.583178Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789090637048111:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35289751ns) {Configure:104(34490711ns) {ProcessConfigured:135(883889ns) {StartInitialHandshake:157(881963ns) {DropHandshakes:470(4429ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(3230ns)} GenerateSessionVirtualId:988(870ns) PrepareNewSessionHandshake:477(1203ns) {SwitchToState:179(562ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.583901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789092024931229:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.585718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789090570057427:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25291633ns) {Configure:104(25285507ns) {ProcessConfigured:135(1507162ns) {StartInitialHandshake:157(1504806ns) {DropHandshakes:470(1850ns) {DropIncomingHandshake:442(438ns) DropOutgoingHandshake:457(532ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:477(2016ns) {SwitchToState:179(636ns)}}}}}
2025-11-28T14:51:51.587091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789092452088110:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.588334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789090570057434:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42345
2025-11-28T14:51:51.590217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789090570057434:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.601293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789090637048183:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51769
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.591326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789092024931219:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.613259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789091816341508:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35449
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.601519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789090637048184:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51769
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.617020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789092477658678:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30528
2025-11-28T14:51:51.602080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789090637048187:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51769
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.615745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789092452088105:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.602137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789090637048188:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51769
2025-11-28T14:51:51.616193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789091816341509:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35449
2025-11-28T14:51:51.615802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789092452088105:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:51.619761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789092477658680:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30528
2025-11-28T14:51:51.619819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789092452088105:2131] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.621702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658678:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.623659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789091816341508:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.623801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789091816341423:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.615771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789090637048176:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25697861ns) {Configure:104(25692687ns) {ProcessConfigured:135(17283298ns) {StartInitialHandshake:157(17279809ns) {DropHandshakes:470(1499ns) {DropIncomingHandshake:442(455ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(1450ns) PrepareNewSessionHandshake:477(2542ns) {SwitchToState:179(1240ns)}}}}}
2025-11-28T14:51:51.620065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789092452088110:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.619170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789090637048175:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(30736164ns) {Configure:104(30731330ns) {ProcessConfigured:135(17031714ns) {StartInitialHandshake:157(17027924ns) {DropHandshakes:470(2479ns) {DropIncomingHandshake:442(401ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(1357ns) PrepareNewSessionHandshake:477(2367ns) {SwitchToState:179(1234ns)}}}}}
2025-11-28T14:51:51.626253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789092477658686:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30528
2025-11-28T14:51:51.620339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789090637048183:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.626905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789090570057441:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42345
2025-11-28T14:51:51.622224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789092452088098:2123] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.627370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789090637048184:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.627359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789092024931219:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10642) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789092024931229:2113] held: no
2025-11-28T14:51:51.626944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789090570057442:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42345
2025-11-28T14:51:51.622294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:51.631621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789090570057431:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.631480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931238:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.622373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789092452088098:2123] [node 6] ICP22 created new session: [5:7577789092452088114:2123]
2025-11-28T14:51:51.632689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789092477658687:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30528
No RDMA context found for GID: 2025-11-28T14:51:51.631661Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789090570057431:2117] [node 4] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-28T14:51:51.622537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789092452088114:2123] [node 6] ICS09 handshake done sender: [5:7577789092452088105:2131] self: [5:7577789092452088104:0] peer: [6:7577789092024931237:0] socket: 260 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.644787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789090570057443:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42345
2025-11-28T14:51:51.644578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789090637048194:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51769
2025-11-28T14:51:51.644576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789092024931229:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:51.645358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789090570057442:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.645833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789092024931219:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.622577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789092452088114:2123] [node 6] ICS10 traffic start
2025-11-28T14:51:51.644968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789091816341512:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35449
2025-11-28T14:51:51.645975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:51.646213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789091816341512:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.646138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789092024931219:2105] [node 5] ICP22 created new session: [6:7577789092024931239:2105]
2025-11-28T14:51:51.622625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:51.646193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789092024931239:2105] [node 5] ICS09 handshake done sender: [6:7577789092024931229:2113] self: [6:7577789092024931237:0] peer: [5:7577789092452088104:0] socket: 262 qp: -1
2025-11-28T14:51:51.625937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789092452088111:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37417
2025-11-28T14:51:51.646304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789092024931239:2105] [node 5] ICS10 traffic start
2025-11-28T14:51:51.646349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:51.646457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789090570057431:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.646712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789091816341513:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35449
2025-11-28T14:51:51.647087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789091816341508:2124] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:51.647440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789091816341423:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.647495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:51.647543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789091816341423:2049] [node 2] ICP22 created new session: [4:7577789091816341515:2049]
2025-11-28T14:51:51.647756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789091816341515:2049] [node 2] ICS09 handshake done sender: [4:7577789091816341508:2124] self: [4:7577789091816341511:0] peer: [2:7577789090570057430:0] socket: 278 qp: -1
2025-11-28T14:51:51.647808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789091816341515:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:51.647849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.649937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789090570057426:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.650005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:51.650057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789090570057426:2107] [node 4] ICP22 created new session: [2:7577789090570057447:2107]
2025-11-28T14:51:51.650035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931242:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
2025-11-28T14:51:51.650106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789090570057447:2107] [node 4] ICS09 handshake done sender: [2:7577789090570057431:2117] self: [2:7577789090570057430:0] peer: [4:7577789091816341511:0] socket: 275 qp: -1
2025-11-28T14:51:51.650158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789090570057447:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:51.650200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:51.653028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789092024931242:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.653454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789092024931222:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.653716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789091816341504:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.653752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789091816341504:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:51.653281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789090570057429:2116] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.653670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789090570057446:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42345
2025-11-28T14:51:51.654640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789090570057446:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.654937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931244:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
2025-11-28T14:51:51.655065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931245:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
2025-11-28T14:51:51.654915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789091816341504:2122] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:51.655191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789091816341495:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.655233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:51.655287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789091816341495:2109] [node 6] ICP22 created new session: [4:7577789091816341519:2109]
2025-11-28T14:51:51.655341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789091816341519:2109] [node 6] ICS09 handshake done sender: [4:7577789091816341504:2122] self: [4:7577789091816341502:0] peer: [6:7577789092024931243:0] socket: 281 qp: -1
2025-11-28T14:51:51.655382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789091816341519:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:51.655421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.667454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789092024931242:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:51.667892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789092024931222:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.668084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789092452088117:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37417
2025-11-28T14:51:51.668670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789092452088117:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.668867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789092452088096:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.669203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789091816341505:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.667968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:51.669251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789091816341505:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:51.668008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789092024931222:2108] [node 4] ICP22 created new session: [6:7577789092024931246:2108]
2025-11-28T14:51:51.670209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789091816341505:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:51.670436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789091816341494:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.672550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:51.672685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789091816341494:2108] [node 5] ICP22 created new session: [4:7577789091816341523:2108]
2025-11-28T14:51:51.672784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789091816341523:2108] [node 5] ICS09 handshake done sender: [4:7577789091816341505:2123] self: [4:7577789091816341503:0] peer: [5:7577789092452088118:0] socket: 310 qp: -1
2025-11-28T14:51:51.672837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789091816341523:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.672707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789092024931246:2108] [node 4] ICS09 handshake done sender: [6:7577789092024931242:2117] self: [6:7577789092024931243:0] peer: [4:7577789091816341502:0] socket: 305 qp: -1
2025-11-28T14:51:51.672926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:51.672791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789092024931246:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:51.673346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789092452088119:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37417
2025-11-28T14:51:51.672901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:51.674116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789092452088096:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31114) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789092452088117:2135] held: no
2025-11-28T14:51:51.674342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789092452088117:2135] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:51.674521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789092452088096:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.674759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:51.674835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789092452088096:2108] [node 4] ICP22 created new session: [5:7577789092452088120:2108]
2025-11-28T14:51:51.674905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789092452088120:2108] [node 4] ICS09 handshake done sender: [5:7577789092452088117:2135] self: [5:7577789092452088118:0] peer: [4:7577789091816341503:0] socket: 311 qp: -1
2025-11-28T14:51:51.675014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789092452088120:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:51.675062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:51.677005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789090637048181:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.685991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789090637048194:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.686037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789092452088111:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.686140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789090637048187:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.686312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789092452088016:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.686384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789090637048112:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.686633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789090637048181:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.686753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789090570057429:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.687036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789090637048181:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:51.687142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789090570057429:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.687584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789092452088124:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37417
2025-11-28T14:51:51.687634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789090570057356:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.687687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:51.687766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789090637048174:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.687751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789090570057356:2049] [node 3] ICP22 created new session: [2:7577789090570057452:2049]
2025-11-28T14:51:51.687833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789090570057452:2049] [node 3] ICS09 handshake done sender: [2:7577789090570057429:2116] self: [2:7577789090570057428:0] peer: [3:7577789090637048198:0] socket: 274 qp: -1
2025-11-28T14:51:51.687841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:51.687867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789090570057452:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:51.687892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789090637048174:2108] [node 5] ICP22 created new session: [3:7577789090637048201:2108]
2025-11-28T14:51:51.687913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:51.687890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789092452088111:2134] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:51.687945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789090637048201:2108] [node 5] ICS09 handshake done sender: [3:7577789090637048181:2111] self: [3:7577789090637048178:0] peer: [5:7577789092452088123:0] socket: 286 qp: -1
2025-11-28T14:51:51.688001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789090637048201:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:51.688035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789092452088016:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.688047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:51.688089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:51.688595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789092452088016:2050] [node 3] ICP22 created new session: [5:7577789092452088125:2050]
2025-11-28T14:51:51.688709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789092452088125:2050] [node 3] ICS09 handshake done sender: [5:7577789092452088111:2134] self: [5:7577789092452088123:0] peer: [3:7577789090637048178:0] socket: 289 qp: -1
2025-11-28T14:51:51.688768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789092452088125:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:51.689592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.690504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789092452088127:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.692814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789090637048200:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51769
2025-11-28T14:51:51.693569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789092452088127:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.694127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789090570057424:2115] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.694191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789092452088015:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.701872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789090570057437:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.701916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789090570057437:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:51.701939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789090637048187:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.703149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789090637048112:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.703221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:51.703207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931248:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
2025-11-28T14:51:51.703266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789090637048112:2049] [node 2] ICP22 created new session: [3:7577789090637048204:2049]
2025-11-28T14:51:51.703326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789090637048204:2049] [node 2] ICS09 handshake done sender: [3:7577789090637048187:2115] self: [3:7577789090637048198:0] peer: [2:7577789090570057428:0] socket: 283 qp: -1
2025-11-28T14:51:51.703371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789090637048204:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:51.703409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:51.703516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789090570057437:2120] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.703788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789092452088129:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37417
2025-11-28T14:51:51.703868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789090570057427:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.703938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:51.703991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789090570057427:2108] [node 5] ICP22 created new session: [2:7577789090570057456:2108]
2025-11-28T14:51:51.704049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789090570057456:2108] [node 5] ICS09 handshake done sender: [2:7577789090570057437:2120] self: [2:7577789090570057435:0] peer: [5:7577789092452088128:0] socket: 259 qp: -1
2025-11-28T14:51:51.704097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789090570057456:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:51.704141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:51.704381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789092452088127:2138] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:51.705213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789092452088015:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.705262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:51.705349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789092452088015:2049] [node 2] ICP22 created new session: [5:7577789092452088130:2049]
2025-11-28T14:51:51.705397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789092452088130:2049] [node 2] ICS09 handshake done sender: [5:7577789092452088127:2138] self: [5:7577789092452088128:0] peer: [2:7577789090570057435:0] socket: 317 qp: -1
2025-11-28T14:51:51.705426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789092452088130:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:51.705464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:51.706722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789092452088015:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:51.708095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789090637048182:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.736482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789092477658676:2128] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.745743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789091816341509:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.745723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658680:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.745862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658686:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.746043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789091816341422:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.746194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789092477658689:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30528
2025-11-28T14:51:51.746473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658676:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.746971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658689:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.747129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789092477658676:2128] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:51.747521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789092477658662:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.747388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789091816341526:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35449
2025-11-28T14:51:51.747579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:51.747636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789092477658662:2107] [node 4] ICP22 created new session: [1:7577789092477658693:2107]
2025-11-28T14:51:51.747691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789092477658693:2107] [node 4] ICS09 handshake done sender: [1:7577789092477658676:2128] self: [1:7577789092477658675:0] peer: [4:7577789091816341525:0] socket: 282 qp: -1
2025-11-28T14:51:51.747735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789092477658693:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:51.747773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:51.747867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789091816341509:2125] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:51.748396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789091816341422:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.748446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:51.748489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789091816341422:2048] [node 1] ICP22 created new session: [4:7577789091816341527:2048]
2025-11-28T14:51:51.748540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789091816341527:2048] [node 1] ICS09 handshake done sender: [4:7577789091816341509:2125] self: [4:7577789091816341525:0] peer: [1:7577789092477658675:0] socket: 293 qp: -1
2025-11-28T14:51:51.748572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789091816341527:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:51.748603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.749119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789092452088132:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37417
2025-11-28T14:51:51.758888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789091816341500:2120] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.789403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789092477658674:2127] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.801137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658687:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.801297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789092477658662:2107] [node 4] ICP18 handshake (actor [1:7577789092477658687:2134]) is too old
2025-11-28T14:51:51.801605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789090637048188:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.801966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789090637048111:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.802213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658674:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.802716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789092477658674:2127] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.802986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789090637048207:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51769
2025-11-28T14:51:51.803067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789092477658590:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.803141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:51.803197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789092477658590:2049] [node 3] ICP22 created new session: [1:7577789092477658698:2049]
2025-11-28T14:51:51.803251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789092477658698:2049] [node 3] ICS09 handshake done sender: [1:7577789092477658674:2127] self: [1:7577789092477658673:0] peer: [3:7577789090637048206:0] socket: 280 qp: -1
2025-11-28T14:51:51.803303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789092477658698:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:51.803344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:51.803487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789090637048188:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:51.804171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789090637048111:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.804233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:51.804291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789090637048111:2048] [node 1] ICP22 created new session: [3:7577789090637048208:2048]
2025-11-28T14:51:51.804332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789090637048208:2048] [node 1] ICS09 handshake done sender: [3:7577789090637048188:2116] self: [3:7577789090637048206:0] peer: [1:7577789092477658673:0] socket: 284 qp: -1
2025-11-28T14:51:51.804379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789090637048208:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:51.804416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:51.806339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789092477658672:2126] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.807175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789090570057441:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.808986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931249:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
2025-11-28T14:51:51.807323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789090570057355:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.808842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658672:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.809443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789092477658672:2126] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:51.823293Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789090570057355:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16503) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789090570057441:2121] held: no
2025-11-28T14:51:51.825620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789090637048180:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.809681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789092477658589:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.840782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789090570057433:2118] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.809734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:51.809780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789092477658589:2048] [node 2] ICP22 created new session: [1:7577789092477658702:2048]
2025-11-28T14:51:51.843967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789090570057443:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.809824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789092477658702:2048] [node 2] ICS09 handshake done sender: [1:7577789092477658672:2126] self: [1:7577789092477658671:0] peer: [2:7577789090570057458:0] socket: 277 qp: -1
2025-11-28T14:51:51.809853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789092477658702:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:51.809888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:51.862332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789092024931248:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.849939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789090570057356:2049] [node 3] ICP18 handshake (actor [2:7577789090570057443:2123]) is too old
2025-11-28T14:51:51.854030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789091816341529:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35449
2025-11-28T14:51:51.862646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789092024931160:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.854981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789090637048192:2117] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.862697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789092024931244:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.867650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789092024931159:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.867754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789090637048192:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.868198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789090637048192:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.870379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931253:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
2025-11-28T14:51:51.862782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789090570057459:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42345
2025-11-28T14:51:51.870588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931252:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
2025-11-28T14:51:51.868845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789090570057433:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.869645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789090637048176:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.869498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789090570057441:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:51.869740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:51.870677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789090570057355:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.870645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789090637048176:2109] [node 6] ICP22 created new session: [3:7577789090637048212:2109]
2025-11-28T14:51:51.872605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789092024931248:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:51.872760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789092024931244:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:51.870729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:51.871519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789090637048212:2109] [node 6] ICS09 handshake done sender: [3:7577789090637048192:2117] self: [3:7577789090637048185:0] peer: [6:7577789092024931250:0] socket: 268 qp: -1
2025-11-28T14:51:51.870872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789090570057433:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:51.880250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789092024931160:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.871976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789090637048212:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:51.871372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789090570057355:2048] [node 1] ICP22 created new session: [2:7577789090570057461:2048]
2025-11-28T14:51:51.872708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789090570057461:2048] [node 1] ICS09 handshake done sender: [2:7577789090570057441:2121] self: [2:7577789090570057458:0] peer: [1:7577789092477658671:0] socket: 279 qp: -1
2025-11-28T14:51:51.880335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:51.872019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:51.873166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789090570057425:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.880399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789092024931160:2050] [node 3] ICP22 created new session: [6:7577789092024931254:2050]
2025-11-28T14:51:51.873393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789090570057461:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:51.880446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789092024931254:2050] [node 3] ICS09 handshake done sender: [6:7577789092024931248:2120] self: [6:7577789092024931250:0] peer: [3:7577789090637048185:0] socket: 318 qp: -1
2025-11-28T14:51:51.880490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789092024931254:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:51.873414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:51.880529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:51.873435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:51.881113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789092024931159:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.881157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:51.874369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789090570057425:2109] [node 6] ICP22 created new session: [2:7577789090570057462:2109]
2025-11-28T14:51:51.886134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789090637048176:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16099906ns) {HandleHandshakeStatus:329(16096798ns) {DropIncomingHandshake:442(23023ns) DropOutgoingHandshake:457(520ns) SwitchToState:179(707ns) ProcessPendingSessionEvents:495(2107947ns) {ForwardSessionEventToSession:612(1401729ns) ForwardSessionEventToSession:612(5194ns) ForwardSessionEventToSession:612(47064ns) ForwardSessionEventToSession:612(3429ns) ForwardSessionEventToSession:612(110199ns) ForwardSessionEventToSession:612(3052ns) ForwardSessionEventToSession:612(2737ns) ForwardSessionEventToSession:612(516860ns) ForwardSessionEventToSession:612(3334ns) ForwardSessionEventToSession:612(2763ns)}}}
2025-11-28T14:51:51.884448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789090570057462:2109] [node 6] ICS09 handshake done sender: [2:7577789090570057433:2118] self: [2:7577789090570057432:0] peer: [6:7577789092024931251:0] socket: 290 qp: -1
2025-11-28T14:51:51.881195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789092024931159:2049] [node 2] ICP22 created new session: [6:7577789092024931256:2049]
2025-11-28T14:51:51.884743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789090570057462:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:51.881238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789092024931256:2049] [node 2] ICS09 handshake done sender: [6:7577789092024931244:2118] self: [6:7577789092024931251:0] peer: [2:7577789090570057432:0] socket: 307 qp: -1
2025-11-28T14:51:51.884779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:51.881270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789092024931256:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:51.881300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:51.885913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789092024931160:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:51.888144Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789092477658679:2130] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.896046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789092452088132:2140] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.896387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789092452088014:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.896678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658679:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.897396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789092477658679:2130] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:51.897528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789092452088134:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37417
2025-11-28T14:51:51.897678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789092477658669:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.897828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:51.898093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789092452088132:2140] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:51.898080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789092477658669:2112] [node 5] ICP22 created new session: [1:7577789092477658705:2112]
2025-11-28T14:51:51.898112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789092477658705:2112] [node 5] ICS09 handshake done sender: [1:7577789092477658679:2130] self: [1:7577789092477658677:0] peer: [5:7577789092452088133:0] socket: 322 qp: -1
2025-11-28T14:51:51.898241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789092452088014:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.898139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789092477658705:2112] [node 5] ICS10 traffic start
2025-11-28T14:51:51.898164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:51.898373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:51.898430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789092452088014:2048] [node 1] ICP22 created new session: [5:7577789092452088135:2048]
2025-11-28T14:51:51.898548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789092452088135:2048] [node 1] ICS09 handshake done sender: [5:7577789092452088132:2140] self: [5:7577789092452088133:0] peer: [1:7577789092477658677:0] socket: 323 qp: -1
2025-11-28T14:51:51.898586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789092452088135:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:51.898770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:51.899797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789092477658682:2132] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.907409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789092024931249:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.907660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789092024931158:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.907979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789092477658682:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.908447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789092477658682:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:51.909084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789092477658670:2120] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:51.909170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:51.909231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789092477658670:2120] [node 6] ICP22 created new session: [1:7577789092477658708:2120]
2025-11-28T14:51:51.909310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789092477658708:2120] [node 6] ICS09 handshake done sender: [1:7577789092477658682:2132] self: [1:7577789092477658681:0] peer: [6:7577789092024931258:0] socket: 324 qp: -1
2025-11-28T14:51:51.909285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789092024931259:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36154
2025-11-28T14:51:51.909357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789092477658708:2120] [node 6] ICS10 traffic start
2025-11-28T14:51:51.909446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:51.909738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789092024931249:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:51.910740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789092024931158:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16503) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789092024931249:2121] held: no
2025-11-28T14:51:51.910939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789092024931158:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.910992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:51.911037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789092024931158:2048] [node 1] ICP22 created new session: [6:7577789092024931260:2048]
2025-11-28T14:51:51.911080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789092024931260:2048] [node 1] ICS09 handshake done sender: [6:7577789092024931249:2121] self: [6:7577789092024931258:0] peer: [1:7577789092477658681:0] socket: 325 qp: -1
2025-11-28T14:51:51.911116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789092024931260:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:51.911169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:51.913339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789090637048193:2118] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:51.914046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789091816341529:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.914229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789091816341424:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:51.914440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789090637048193:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:51.914911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789090637048193:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:51.915106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789090637048175:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:51.915357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789091816341531:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35449
2025-11-28T14:51:51.915165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:51.915191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789090637048175:2107] [node 4] ICP22 created new session: [3:7577789090637048215:2107]
2025-11-28T14:51:51.915315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789090637048215:2107] [node 4] ICS09 handshake done sender: [3:7577789090637048193:2118] self: [3:7577789090637048186:0] peer: [4:7577789091816341530:0] socket: 287 qp: -1
2025-11-28T14:51:51.915658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789091816341529:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:51.915361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789090637048215:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:51.915394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:51.916166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789091816341424:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:51.916207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:51.916262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789091816341424:2050] [node 3] ICP22 created new session: [4:7577789091816341532:2050]
2025-11-28T14:51:51.916327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789091816341532:2050] [node 3] ICS09 handshake done sender: [4:7577789091816341529:2129] self: [4:7577789091816341530:0] peer: [3:7577789090637048186:0] socket: 295 qp: -1
2025-11-28T14:51:51.916386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789091816341532:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:51.916432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:51.916656Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789091816341424:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:51.919627Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:51.919858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789090570057461:2048] [node 1] ICS01 socket: 279 reason# EndOfStream
2025-11-28T14:51:51.919924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789090570057355:2048] [node 1] ICP30 unregister session Session# [2:7577789090570057461:2048] VirtualId# [2:7577789090570057458:0]
2025-11-28T14:51:51.919959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789090570057461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.920056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:51.920128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:51.920844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:51.920971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789090637048208:2048] [node 1] ICS01 socket: 284 reason# EndOfStream
2025-11-28T14:51:51.921001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789090637048111:2048] [node 1] ICP30 unregister session Session# [3:7577789090637048208:2048] VirtualId# [3:7577789090637048206:0]
2025-11-28T14:51:51.921025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789090637048208:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.921076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:51.921135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:51.923041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:51.923234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789091816341527:2048] [node 1] ICS07 socket disconnect 293 reason# EndOfStream
2025-11-28T14:51:51.923268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789091816341527:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.923348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:51.923397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789091816341527:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:51.923912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:51.924067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789092452088135:2048] [node 1] ICS07 socket disconnect 323 reason# EndOfStream
2025-11-28T14:51:51.924084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789092452088135:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.924140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789091816341422:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16503) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.924148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:51.924300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789091816341527:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.924331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789091816341422:2048] [node 1] ICP30 unregister session Session# [4:7577789091816341527:2048] VirtualId# [4:7577789091816341525:0]
2025-11-28T14:51:51.924393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:51.924198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789092452088135:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:51.924421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789091816341422:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16503) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.924438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16503) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.925110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789092452088014:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16503) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.925244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:51.925154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789092452088135:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.925426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789092024931260:2048] [node 1] ICS01 socket: 325 reason# EndOfStream
2025-11-28T14:51:51.925215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789092452088014:2048] [node 1] ICP30 unregister session Session# [5:7577789092452088135:2048] VirtualId# [5:7577789092452088133:0]
2025-11-28T14:51:51.925456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789092024931158:2048] [node 1] ICP30 unregister session Session# [6:7577789092024931260:2048] VirtualId# [6:7577789092024931258:0]
2025-11-28T14:51:51.925473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789092024931260:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.925537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:51.925590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:51.925256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:51.925327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789092452088014:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16503) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.925355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16503) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.934638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789091816341422:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:51.935619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789092452088014:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:51.950326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:51.950554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789090637048204:2049] [node 2] ICS07 socket disconnect 283 reason# EndOfStream
2025-11-28T14:51:51.950578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789090637048204:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.950696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:51.950742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789090637048204:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:51.953374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789090637048112:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27045) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:51.953420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789090637048204:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:51.953539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:51.953854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789091816341515:2049] [node 2] ICS01 socket: 278 reason# EndOfStream
2025-11-28T14:51:51.953901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789091816341423:2049] [node 2] ICP30 unregister session Session# [4:7577789091816341515:2049] VirtualId# [4:7577789091816341511:0]
2025-11-28T14:51:51.953975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789091816341515:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.954075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:51.954138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:51.954525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789090637048112:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27045) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.954602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789090637048204:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.954642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789090637048112:2049] [node 2] ICP30 unregister session Session# [3:7577789090637048204:2049] VirtualId# [3:7577789090637048198:0]
2025-11-28T14:51:51.954690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:51.954719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789090637048112:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.954731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.955050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:51.955243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789092452088130:2049] [node 2] ICS07 socket disconnect 317 reason# EndOfStream
2025-11-28T14:51:51.955284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789092452088130:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.955385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:51.955438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789092452088130:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:51.955983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:51.956161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789092024931256:2049] [node 2] ICS01 socket: 307 reason# EndOfStream
2025-11-28T14:51:51.956202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789092024931159:2049] [node 2] ICP30 unregister session Session# [6:7577789092024931256:2049] VirtualId# [6:7577789092024931251:0]
2025-11-28T14:51:51.956258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789092024931256:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.956335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:51.956390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:51.956488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789092452088015:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27045) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.956584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789092452088130:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.956617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789092452088015:2049] [node 2] ICP30 unregister session Session# [5:7577789092452088130:2049] VirtualId# [5:7577789092452088128:0]
2025-11-28T14:51:51.956676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:51.956711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789092452088015:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.956741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.965701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:51.965940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789091816341532:2050] [node 3] ICS07 socket disconnect 295 reason# EndOfStream
2025-11-28T14:51:51.965987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789091816341532:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.966102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:51.966151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789091816341532:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:51.966364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789092452088015:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:51.966368Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:51.966492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789092452088125:2050] [node 3] ICS07 socket disconnect 289 reason# EndOfStream
2025-11-28T14:51:51.966511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789092452088125:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.966574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:51.966618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789092452088125:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:51.967014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789091816341424:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14794) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.967116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789091816341532:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.967164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789091816341424:2050] [node 3] ICP30 unregister session Session# [4:7577789091816341532:2050] VirtualId# [4:7577789091816341530:0]
2025-11-28T14:51:51.967120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:51.967155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789092452088016:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14794) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:51.967210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789092452088125:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:51.967224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:51.967269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789092452088016:2050] [node 3] ICP30 unregister session Session# [5:7577789092452088125:2050] VirtualId# [5:7577789092452088123:0]
2025-11-28T14:51:51.967273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789091816341424:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14794) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.967320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:51.967302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14794) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.967381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789092024931254:2050] [node 3] ICS01 socket: 318 reason# EndOfStream
2025-11-28T14:51:51.967338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789092452088016:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14794) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.967385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14794) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:51.967428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789092024931160:2050] [node 3] ICP30 unregister session Session# [6:7577789092024931254:2050] VirtualId# [6:7577789092024931250:0]
2025-11-28T14:51:51.967464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789092024931254:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.967525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:51.967568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:51.977656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789092452088016:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:51.996111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:51.996353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789092452088120:2108] [node 4] ICS01 socket: 311 reason# EndOfStream
2025-11-28T14:51:51.996425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789092452088096:2108] [node 4] ICP30 unregister session Session# [5:7577789092452088120:2108] VirtualId# [5:7577789092452088118:0]
2025-11-28T14:51:51.996470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789092452088120:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.996598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:51.996686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:51.997134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:51.997340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789092024931246:2108] [node 4] ICS07 socket disconnect 305 reason# EndOfStream
2025-11-28T14:51:51.997375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789092024931246:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:51.997465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:51.997516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789092024931246:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:52.000604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789092024931222:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31114) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.000686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789092024931246:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.000778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789092024931222:2108] [node 4] ICP30 unregister session Session# [6:7577789092024931246:2108] VirtualId# [6:7577789092024931243:0]
2025-11-28T14:51:52.000838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:52.000896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789092024931222:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31114) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.000927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31114) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.010444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789092024931222:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:52.042873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:52.043160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789092024931239:2105] [node 5] ICS01 socket: 262 reason# EndOfStream
2025-11-28T14:51:52.043224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789092024931219:2105] [node 5] ICP30 unregister session Session# [6:7577789092024931239:2105] VirtualId# [6:7577789092024931237:0]
2025-11-28T14:51:52.043269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789092024931239:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.043398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:52.043469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:52.113844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789096340201131:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:52.113843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789096340201130:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:52.133406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789093783845664:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:52.133407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789093783845663:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:52.155608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789094692816866:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:52.156420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789094692816865:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:52.181953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789094158941397:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:52.181993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789094158941399:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:52.182028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789094158941398:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:52.205588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789093419886120:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:52.205624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789093419886121:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:52.205672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789093419886122:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:52.230997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789095363798843:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:52.231070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789095363798844:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:52.231100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789095363798845:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:52.295288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789095363798906:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:52.330790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789096340201193:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:52.337911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789096340201194:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:52.337897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789094692816928:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:52.338086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789096340201195:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:52.337928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789094158941460:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:52.338269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789094692816929:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:52.337954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789093419886183:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:52.337954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789093783845753:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:52.338203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789093419886184:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:52.338826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789094692816930:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:52.338887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789093783845755:2122] [node 6] ICP01 ready to work
2025-11-28T14:51:52.338807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789095363798912:2110] [node 4] ICP01 ready to work
2025-11-28T14:51:52.339111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789094158941461:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:52.338964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789093783845754:2114] [node 5] ICP01 ready to work
2025-11-28T14:51:52.338898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789095363798906:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(42964539ns) {Configure:104(42949185ns) {ProcessConfigured:135(3607454ns) {StartInitialHandshake:157(3603063ns) {DropHandshakes:470(2164401ns) {DropIncomingHandshake:442(839ns) DropOutgoingHandshake:457(1702ns)} GenerateSessionVirtualId:988(1739ns) PrepareNewSessionHandshake:477(2005ns) {SwitchToState:179(999ns)}}}}}
2025-11-28T14:51:52.339621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789096340201131:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45946333ns) {Configure:104(45944142ns) {ProcessConfigured:135(366312ns) {StartInitialHandshake:157(363620ns) {DropHandshakes:470(1380ns) {DropIncomingHandshake:442(411ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(997ns) PrepareNewSessionHandshake:477(1367ns) {SwitchToState:179(311ns)}}}}}
2025-11-28T14:51:52.339707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789096340201130:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46414911ns) {Configure:104(46409449ns) {ProcessConfigured:135(335214ns) {StartInitialHandshake:157(333338ns) {DropHandshakes:470(4656ns) {DropIncomingHandshake:442(604ns) DropOutgoingHandshake:457(311ns)} GenerateSessionVirtualId:988(559ns) PrepareNewSessionHandshake:477(1106ns) {SwitchToState:179(512ns)}}}}}
2025-11-28T14:51:52.345954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789094158941398:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40048731ns) {Configure:104(40039526ns) {ProcessConfigured:135(779900ns) {StartInitialHandshake:157(777388ns) {DropHandshakes:470(1193ns) {DropIncomingHandshake:442(441ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(736ns) PrepareNewSessionHandshake:477(1656ns) {SwitchToState:179(739ns)}}}}}
2025-11-28T14:51:52.348242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789094692816865:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(45198296ns) {Configure:104(45188915ns) {ProcessConfigured:135(249579ns) {StartInitialHandshake:157(248226ns) {DropHandshakes:470(1284ns) {DropIncomingHandshake:442(402ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(484ns) PrepareNewSessionHandshake:477(1246ns) {SwitchToState:179(184ns)}}}}}
2025-11-28T14:51:52.348133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789094158941397:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35818168ns) {Configure:104(35808874ns) {ProcessConfigured:135(1119065ns) {StartInitialHandshake:157(1117272ns) {DropHandshakes:470(578ns) {DropIncomingHandshake:442(207ns) DropOutgoingHandshake:457(125ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(1854ns) {SwitchToState:179(819ns)}}}}}
2025-11-28T14:51:52.349819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789094692816866:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45813207ns) {Configure:104(42645709ns) {ProcessConfigured:135(1852530ns) {StartInitialHandshake:157(1696005ns) {DropHandshakes:470(1963ns) {DropIncomingHandshake:442(257ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(1029ns) PrepareNewSessionHandshake:477(1827ns) {SwitchToState:179(589ns)}}}}}
2025-11-28T14:51:52.352792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789093783845663:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52670685ns) {Configure:104(52666999ns) {ProcessConfigured:135(1195684ns) {StartInitialHandshake:157(1193034ns) {DropHandshakes:470(1731ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(872ns) PrepareNewSessionHandshake:477(5484ns) {SwitchToState:179(1301ns)}}}}}
2025-11-28T14:51:52.353263Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789093783845664:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52366901ns) {Configure:104(46007175ns) {ProcessConfigured:135(678605ns) {StartInitialHandshake:157(676149ns) {DropHandshakes:470(1434ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(499ns)} GenerateSessionVirtualId:988(541ns) PrepareNewSessionHandshake:477(1188ns) {SwitchToState:179(590ns)}}}}}
2025-11-28T14:51:52.354130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789093419886121:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16085559ns) {Configure:104(16080677ns) {ProcessConfigured:135(835227ns) {StartInitialHandshake:157(833291ns) {DropHandshakes:470(1208ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:477(1627ns) {SwitchToState:179(696ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.355348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789094692816937:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.356208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789094692816940:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.356381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789096340201209:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51981
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.358471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789093783845766:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61206
2025-11-28T14:51:52.359095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789094692816941:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-28T14:51:52.358639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789093419886193:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53575
2025-11-28T14:51:52.359133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789093419886194:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53575
2025-11-28T14:51:52.360425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789093783845764:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61206
2025-11-28T14:51:52.360684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789094692816937:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.362294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789093419886194:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.360986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789096340201210:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51981
2025-11-28T14:51:52.361067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789094692816865:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.362391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789093419886193:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.364066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789093419886120:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.360537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789093783845765:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61206
2025-11-28T14:51:52.365722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789094158941475:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38721
2025-11-28T14:51:52.364579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789093419886198:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53575
2025-11-28T14:51:52.362416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789093783845765:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.361153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789094692816940:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.362624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789096340201197:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.364057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789093783845766:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.363495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789094692816941:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.362660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789096340201197:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.365206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789096340201209:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.365134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789093783845764:2141] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.373314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798924:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.373158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789094692816948:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-28T14:51:52.366684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789096340201204:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.366682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789093783845663:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.375548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789094158941476:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38721
2025-11-28T14:51:52.366709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789096340201204:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:52.376163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789093783845772:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61206
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.367634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789096340201211:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51981
2025-11-28T14:51:52.378788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789094692816951:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-28T14:51:52.368199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789096340201197:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:52.379028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789093419886203:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53575
2025-11-28T14:51:52.373697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789096340201199:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.373741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789096340201199:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:52.379657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789093783845776:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61206
2025-11-28T14:51:52.376774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789096340201212:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51981
2025-11-28T14:51:52.377624Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789096340201204:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:52.378207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789096340201199:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:52.378319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789096340201131:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.380214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789093419886194:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:52.378368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:52.380427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789094692816937:2112] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:52.378508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789096340201131:2049] [node 3] ICP22 created new session: [1:7577789096340201217:2049]
2025-11-28T14:51:52.378886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789096340201193:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.378949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789096340201130:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.382216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789093419886120:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.378965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.378977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789096340201217:2049] [node 3] ICS09 handshake done sender: [1:7577789096340201197:2110] self: [1:7577789096340201196:0] peer: [3:7577789094692816947:0] socket: 264 qp: -1
2025-11-28T14:51:52.382283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:52.382274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789093783845764:2141] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:52.379023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-28T14:51:52.382335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789093419886120:2048] [node 1] ICP22 created new session: [5:7577789093419886204:2048]
2025-11-28T14:51:52.382382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789093419886204:2048] [node 1] ICS09 handshake done sender: [5:7577789093419886194:2115] self: [5:7577789093419886201:0] peer: [1:7577789096340201203:0] socket: 275 qp: -1
2025-11-28T14:51:52.382393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798925:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.379206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789096340201217:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:52.382246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789094692816865:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.382342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:52.382419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789093419886204:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:52.379213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789096340201130:2048] [node 2] ICP22 created new session: [1:7577789096340201219:2048]
2025-11-28T14:51:52.382720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789094158941477:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38721
2025-11-28T14:51:52.382384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789094692816865:2048] [node 1] ICP22 created new session: [3:7577789094692816952:2048]
2025-11-28T14:51:52.382455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:52.379242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:52.382553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789094692816952:2048] [node 1] ICS09 handshake done sender: [3:7577789094692816937:2112] self: [3:7577789094692816947:0] peer: [1:7577789096340201196:0] socket: 272 qp: -1
2025-11-28T14:51:52.383082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789093783845663:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.383203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789095363798925:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.382751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789094692816952:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:52.383137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:52.379289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789096340201193:2108] [node 5] ICP22 created new session: [1:7577789096340201218:2108]
2025-11-28T14:51:52.382802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:52.379399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789096340201219:2048] [node 2] ICS09 handshake done sender: [1:7577789096340201199:2111] self: [1:7577789096340201198:0] peer: [2:7577789093783845775:0] socket: 265 qp: -1
2025-11-28T14:51:52.379566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789096340201219:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:52.379578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789096340201218:2108] [node 5] ICS09 handshake done sender: [1:7577789096340201204:2112] self: [1:7577789096340201203:0] peer: [5:7577789093419886201:0] socket: 269 qp: -1
2025-11-28T14:51:52.383287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789093783845663:2048] [node 1] ICP22 created new session: [2:7577789093783845777:2048]
2025-11-28T14:51:52.379602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:52.383263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789094158941477:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.383335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789093783845777:2048] [node 1] ICS09 handshake done sender: [2:7577789093783845764:2141] self: [2:7577789093783845775:0] peer: [1:7577789096340201198:0] socket: 276 qp: -1
2025-11-28T14:51:52.379621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789096340201218:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:52.383471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789094158941397:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.383435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789093783845777:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:52.383483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:52.379658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:52.383423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789095363798843:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.383879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789096340201205:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.383910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789096340201205:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:52.384465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789096340201207:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.384498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789096340201207:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:52.385549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789096340201205:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.385719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789096340201195:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.385759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:52.386019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798927:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.385845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789096340201207:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:52.385926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789096340201195:2109] [node 6] ICP22 created new session: [1:7577789096340201227:2109]
2025-11-28T14:51:52.386092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789096340201227:2109] [node 6] ICS09 handshake done sender: [1:7577789096340201205:2113] self: [1:7577789096340201202:0] peer: [6:7577789095363798926:0] socket: 312 qp: -1
No RDMA context found for GID: 2025-11-28T14:51:52.386234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789096340201227:2109] [node 6] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:51:52.386275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789096340201194:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.386327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:52.386699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789095363798925:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:52.386704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789094158941479:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38721
2025-11-28T14:51:52.386384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:52.387116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789095363798843:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.386534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789096340201194:2107] [node 4] ICP22 created new session: [1:7577789096340201229:2107]
2025-11-28T14:51:52.387644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789094158941397:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789094158941477:2117] held: no
2025-11-28T14:51:52.387179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:52.386610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789096340201229:2107] [node 4] ICS09 handshake done sender: [1:7577789096340201207:2114] self: [1:7577789096340201206:0] peer: [4:7577789094158941478:0] socket: 314 qp: -1
2025-11-28T14:51:52.386649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789096340201229:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:52.387792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789095363798843:2048] [node 1] ICP22 created new session: [6:7577789095363798928:2048]
2025-11-28T14:51:52.388352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789094158941477:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:52.386692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:52.387897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789095363798928:2048] [node 1] ICS09 handshake done sender: [6:7577789095363798925:2119] self: [6:7577789095363798926:0] peer: [1:7577789096340201202:0] socket: 313 qp: -1
2025-11-28T14:51:52.388716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789094158941397:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.388860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:52.388064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789095363798928:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:52.388113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:52.389182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789094158941397:2048] [node 1] ICP22 created new session: [4:7577789094158941481:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.389387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789094158941481:2048] [node 1] ICS09 handshake done sender: [4:7577789094158941477:2117] self: [4:7577789094158941478:0] peer: [1:7577789096340201206:0] socket: 315 qp: -1
2025-11-28T14:51:52.389425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789094158941481:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:52.389498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:52.389874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789094158941480:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38721
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.390833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798931:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.392105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789095363798931:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.392649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789095363798912:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.392848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789094158941469:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.392876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789094158941469:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:52.393837Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789094692816938:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.401333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789093419886198:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.401529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789093419886122:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.401888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789094692816938:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.402213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789094158941469:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.402413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789094158941460:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.402483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798933:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.402468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:52.402397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789094692816938:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.402569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789094158941460:2109] [node 6] ICP22 created new session: [4:7577789094158941485:2109]
2025-11-28T14:51:52.402705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789094692816928:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.402647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789094158941485:2109] [node 6] ICS09 handshake done sender: [4:7577789094158941469:2113] self: [4:7577789094158941468:0] peer: [6:7577789095363798932:0] socket: 279 qp: -1
2025-11-28T14:51:52.402765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:52.402953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789093419886207:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53575
2025-11-28T14:51:52.402872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789095363798931:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:52.402688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789094158941485:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:52.403401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789095363798912:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.402817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789094692816928:2108] [node 5] ICP22 created new session: [3:7577789094692816955:2108]
2025-11-28T14:51:52.402803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:52.403480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:52.403031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789094692816955:2108] [node 5] ICS09 handshake done sender: [3:7577789094692816938:2113] self: [3:7577789094692816936:0] peer: [5:7577789093419886206:0] socket: 288 qp: -1
2025-11-28T14:51:52.403522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789095363798912:2110] [node 4] ICP22 created new session: [6:7577789095363798934:2110]
2025-11-28T14:51:52.403160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789094692816955:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:52.403234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:52.403576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789095363798934:2110] [node 4] ICS09 handshake done sender: [6:7577789095363798931:2121] self: [6:7577789095363798932:0] peer: [4:7577789094158941468:0] socket: 321 qp: -1
2025-11-28T14:51:52.403613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789095363798934:2110] [node 4] ICS10 traffic start
2025-11-28T14:51:52.403650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.404079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789093419886198:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:52.404252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789093783845779:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61206
2025-11-28T14:51:52.404541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789093419886122:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.404714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:52.404879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789093419886122:2050] [node 3] ICP22 created new session: [5:7577789093419886209:2050]
2025-11-28T14:51:52.405032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789093419886209:2050] [node 3] ICS09 handshake done sender: [5:7577789093419886198:2116] self: [5:7577789093419886206:0] peer: [3:7577789094692816936:0] socket: 290 qp: -1
2025-11-28T14:51:52.405070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789093419886209:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:52.405109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.405738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789093419886208:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53575
2025-11-28T14:51:52.406255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789093419886208:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.406504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789094692816934:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.406702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789093419886184:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.413464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789094158941488:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38721
2025-11-28T14:51:52.413522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789094158941474:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.413544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789094158941474:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:52.413687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789096340201210:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.413883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789096340201131:2049] [node 3] ICP18 handshake (actor [1:7577789096340201210:2116]) is too old
2025-11-28T14:51:52.415466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789094158941474:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:52.416089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789094158941461:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:52.416162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
::ffff:127.1.0.0
2025-11-28T14:51:52.416319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789094158941461:2108] [node 5] ICP22 created new session: [4:7577789094158941490:2108]
2025-11-28T14:51:52.420780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789094158941490:2108] [node 5] ICS09 handshake done sender: [4:7577789094158941474:2114] self: [4:7577789094158941473:0] peer: [5:7577789093419886212:0] socket: 326 qp: -1
2025-11-28T14:51:52.420843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789094158941490:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:52.420879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798936:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.420941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:52.424546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789094692816933:2110] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.432245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789094158941475:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.432492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789094158941399:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.432755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789094692816933:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.433250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789094692816933:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.433449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789094692816929:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.433518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:52.433569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789094692816929:2107] [node 4] ICP22 created new session: [3:7577789094692816960:2107]
2025-11-28T14:51:52.433499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789094158941493:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38721
2025-11-28T14:51:52.433627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789094692816960:2107] [node 4] ICS09 handshake done sender: [3:7577789094692816933:2110] self: [3:7577789094692816932:0] peer: [4:7577789094158941492:0] socket: 287 qp: -1
2025-11-28T14:51:52.433741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789094158941475:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:52.433883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789094158941399:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.433666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789094692816960:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:52.433701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:52.434264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:52.434354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789094158941399:2050] [node 3] ICP22 created new session: [4:7577789094158941494:2050]
2025-11-28T14:51:52.434428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789094158941494:2050] [node 3] ICS09 handshake done sender: [4:7577789094158941475:2115] self: [4:7577789094158941492:0] peer: [3:7577789094692816932:0] socket: 289 qp: -1
2025-11-28T14:51:52.434466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789094158941494:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:52.434509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:52.434820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789094158941399:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:52.435371Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789093419886190:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.443660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789093783845772:2145] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.443652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789093419886213:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53575
2025-11-28T14:51:52.444090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789093419886208:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:52.444289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789093419886184:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.444392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:52.444425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789093419886184:2108] [node 4] ICP22 created new session: [5:7577789093419886214:2108]
2025-11-28T14:51:52.444464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789093419886214:2108] [node 4] ICS09 handshake done sender: [5:7577789093419886208:2120] self: [5:7577789093419886212:0] peer: [4:7577789094158941473:0] socket: 329 qp: -1
2025-11-28T14:51:52.444493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789093419886214:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:52.444538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:52.455847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789093419886192:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.475305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789095363798924:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.475533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789095363798906:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.475875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789093419886192:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.476395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789093419886192:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:52.479335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789093419886183:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.479470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:52.479549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789093419886183:2109] [node 6] ICP22 created new session: [5:7577789093419886217:2109]
2025-11-28T14:51:52.479526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798938:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.479717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789093419886217:2109] [node 6] ICS09 handshake done sender: [5:7577789093419886192:2113] self: [5:7577789093419886187:0] peer: [6:7577789095363798937:0] socket: 295 qp: -1
2025-11-28T14:51:52.479786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789093419886217:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:52.479961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789095363798924:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:52.480132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:52.480230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789095363798906:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789095363798924:2118] held: no
2025-11-28T14:51:52.480498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789095363798906:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.480541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:52.480593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789095363798906:2108] [node 5] ICP22 created new session: [6:7577789095363798939:2108]
2025-11-28T14:51:52.480882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789095363798939:2108] [node 5] ICS09 handshake done sender: [6:7577789095363798924:2118] self: [6:7577789095363798937:0] peer: [5:7577789093419886187:0] socket: 297 qp: -1
2025-11-28T14:51:52.480917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789095363798939:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:52.480983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:52.496183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789093419886189:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.504300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789096340201211:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.504599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789096340201193:2108] [node 5] ICP18 handshake (actor [1:7577789096340201211:2117]) is too old
2025-11-28T14:51:52.525392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789093783845759:2137] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.532544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789096340201212:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.532875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789096340201130:2048] [node 2] ICP18 handshake (actor [1:7577789096340201212:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.533589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798941:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.537610Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789093783845760:2138] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.545862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789094692816948:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.546109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789094692816866:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.546350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789093783845760:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.546854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789093783845760:2138] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.547071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789093783845664:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.547131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:52.547049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789094692816963:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-28T14:51:52.547172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789093783845664:2049] [node 3] ICP22 created new session: [2:7577789093783845783:2049]
2025-11-28T14:51:52.547225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789093783845783:2049] [node 3] ICS09 handshake done sender: [2:7577789093783845760:2138] self: [2:7577789093783845758:0] peer: [3:7577789094692816962:0] socket: 300 qp: -1
2025-11-28T14:51:52.547361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789094692816948:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:52.547350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789093783845783:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:52.547389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:52.547671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789094692816866:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.547724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:52.547767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789094692816866:2049] [node 2] ICP22 created new session: [3:7577789094692816964:2049]
2025-11-28T14:51:52.547799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789094692816964:2049] [node 2] ICS09 handshake done sender: [3:7577789094692816948:2118] self: [3:7577789094692816962:0] peer: [2:7577789093783845758:0] socket: 301 qp: -1
2025-11-28T14:51:52.547825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789094692816964:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:52.547863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:52.548782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789093783845763:2140] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.557121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789093419886219:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53575
2025-11-28T14:51:52.557638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789094158941476:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.557834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789094158941398:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.558092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789093783845763:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.558541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789093783845763:2140] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.558699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789094158941497:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38721
2025-11-28T14:51:52.558779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789093783845753:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.558855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:52.558901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789093783845753:2107] [node 4] ICP22 created new session: [2:7577789093783845787:2107]
2025-11-28T14:51:52.558957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789093783845787:2107] [node 4] ICS09 handshake done sender: [2:7577789093783845763:2140] self: [2:7577789093783845761:0] peer: [4:7577789094158941496:0] socket: 296 qp: -1
2025-11-28T14:51:52.558992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789094158941476:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:52.558996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789093783845787:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:52.559081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:52.559176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789094158941398:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.559418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:52.559506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789094158941398:2049] [node 2] ICP22 created new session: [4:7577789094158941498:2049]
2025-11-28T14:51:52.559564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789094158941498:2049] [node 2] ICS09 handshake done sender: [4:7577789094158941476:2116] self: [4:7577789094158941496:0] peer: [2:7577789093783845761:0] socket: 298 qp: -1
2025-11-28T14:51:52.559598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789094158941498:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:52.559642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:52.560163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789094158941398:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:52.562569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789095363798923:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.571140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789094158941480:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.571325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789094158941460:2109] [node 6] ICP18 handshake (actor [4:7577789094158941480:2119]) is too old
2025-11-28T14:51:52.575174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789094692816939:2114] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.601642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789093783845779:2147] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.601812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789093783845664:2049] [node 3] ICP18 handshake (actor [2:7577789093783845779:2147]) is too old
2025-11-28T14:51:52.605942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789093419886200:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.615047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789094158941488:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.615242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789094158941461:2108] [node 5] ICP18 handshake (actor [4:7577789094158941488:2120]) is too old
2025-11-28T14:51:52.620188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789094692816943:2117] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.628313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789095363798936:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.628564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789095363798845:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.628900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789094692816943:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.629400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789094692816943:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.629651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789094692816930:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.629844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798943:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.629888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:52.629962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789094692816930:2109] [node 6] ICP22 created new session: [3:7577789094692816967:2109]
2025-11-28T14:51:52.630013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789094692816967:2109] [node 6] ICS09 handshake done sender: [3:7577789094692816943:2117] self: [3:7577789094692816942:0] peer: [6:7577789095363798942:0] socket: 285 qp: -1
2025-11-28T14:51:52.630266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789095363798936:2123] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:52.630047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789094692816967:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:52.630302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789095363798845:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789095363798936:2123] held: no
2025-11-28T14:51:52.630193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:52.631964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789095363798845:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.632060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:52.632107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789095363798845:2050] [node 3] ICP22 created new session: [6:7577789095363798944:2050]
2025-11-28T14:51:52.632155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789095363798944:2050] [node 3] ICS09 handshake done sender: [6:7577789095363798936:2123] self: [6:7577789095363798942:0] peer: [3:7577789094692816942:0] socket: 332 qp: -1
2025-11-28T14:51:52.632181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789095363798944:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:52.632212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:52.633699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789093783845762:2139] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.642808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789095363798941:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.643059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789095363798844:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.643350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789093783845762:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.643899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789093783845762:2139] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.644116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789093783845755:2122] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.644097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789095363798947:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45109
2025-11-28T14:51:52.644272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:52.644331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789093783845755:2122] [node 6] ICP22 created new session: [2:7577789093783845790:2122]
2025-11-28T14:51:52.644410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789093783845790:2122] [node 6] ICS09 handshake done sender: [2:7577789093783845762:2139] self: [2:7577789093783845757:0] peer: [6:7577789095363798946:0] socket: 266 qp: -1
2025-11-28T14:51:52.644450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789093783845790:2122] [node 6] ICS10 traffic start
2025-11-28T14:51:52.644496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:52.644479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789095363798941:2125] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:52.644920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789095363798844:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789095363798941:2125] held: no
2025-11-28T14:51:52.645712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789095363798844:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.645852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:52.645990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789095363798844:2049] [node 2] ICP22 created new session: [6:7577789095363798948:2049]
2025-11-28T14:51:52.646042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789095363798948:2049] [node 2] ICS09 handshake done sender: [6:7577789095363798941:2125] self: [6:7577789095363798946:0] peer: [2:7577789093783845757:0] socket: 274 qp: -1
2025-11-28T14:51:52.646075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789095363798948:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:52.646117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:52.648165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789093783845771:2144] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:52.648973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789093419886219:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:52.649155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789093419886121:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:52.649355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789093783845771:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:52.649806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789093783845771:2144] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:52.649931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789093419886221:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53575
2025-11-28T14:51:52.650049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789093783845754:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:52.650196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:52.650248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789093419886219:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:52.650259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789093783845754:2114] [node 5] ICP22 created new session: [2:7577789093783845793:2114]
2025-11-28T14:51:52.650315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789093783845793:2114] [node 5] ICS09 handshake done sender: [2:7577789093783845771:2144] self: [2:7577789093783845767:0] peer: [5:7577789093419886220:0] socket: 299 qp: -1
2025-11-28T14:51:52.650352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789093783845793:2114] [node 5] ICS10 traffic start
2025-11-28T14:51:52.650515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789093419886121:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:52.650458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:52.650644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:52.650694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789093419886121:2049] [node 2] ICP22 created new session: [5:7577789093419886222:2049]
2025-11-28T14:51:52.650828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789093419886222:2049] [node 2] ICS09 handshake done sender: [5:7577789093419886219:2122] self: [5:7577789093419886220:0] peer: [2:7577789093783845767:0] socket: 302 qp: -1
2025-11-28T14:51:52.650872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789093419886222:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:52.650994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:52.697062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:52.697314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789093783845777:2048] [node 1] ICS07 socket disconnect 276 reason# EndOfStream
2025-11-28T14:51:52.697356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789093783845777:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.697521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:52.697606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789093783845777:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:52.698225Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:52.698435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789094692816952:2048] [node 1] ICS07 socket disconnect 272 reason# EndOfStream
2025-11-28T14:51:52.698467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789094692816952:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.698564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:52.698632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789094692816952:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:52.699590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789094692816865:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.699862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789094692816952:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.699887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789094692816865:2048] [node 1] ICP30 unregister session Session# [3:7577789094692816952:2048] VirtualId# [3:7577789094692816947:0]
2025-11-28T14:51:52.699923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:52.699952Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789094692816865:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.699961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.700083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789093783845663:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:52.700144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789093783845777:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:52.701068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:52.701273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789094158941481:2048] [node 1] ICS01 socket: 315 reason# EndOfStream
2025-11-28T14:51:52.701306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789094158941397:2048] [node 1] ICP30 unregister session Session# [4:7577789094158941481:2048] VirtualId# [4:7577789094158941478:0]
2025-11-28T14:51:52.701328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789094158941481:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.701377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:52.701405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:52.701777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789093783845663:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.701890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789093783845777:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.701934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789093783845663:2048] [node 1] ICP30 unregister session Session# [2:7577789093783845777:2048] VirtualId# [2:7577789093783845775:0]
2025-11-28T14:51:52.701993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:52.702040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789093783845663:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.702056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.702080Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:52.702315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789093419886204:2048] [node 1] ICS07 socket disconnect 275 reason# EndOfStream
2025-11-28T14:51:52.702349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789093419886204:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.702458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:52.702523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789093419886204:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:52.703094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:52.703209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789093419886120:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.703257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789093419886204:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.703326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789093419886120:2048] [node 1] ICP30 unregister session Session# [5:7577789093419886204:2048] VirtualId# [5:7577789093419886201:0]
2025-11-28T14:51:52.703374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:52.703340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789095363798928:2048] [node 1] ICS07 socket disconnect 313 reason# EndOfStream
2025-11-28T14:51:52.703476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789093419886120:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.703374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789095363798928:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.703495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.703453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:52.703503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789095363798928:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:52.704205Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789095363798843:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.704302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789095363798928:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.704342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789095363798843:2048] [node 1] ICP30 unregister session Session# [6:7577789095363798928:2048] VirtualId# [6:7577789095363798926:0]
2025-11-28T14:51:52.704413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:52.704451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789095363798843:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.704464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.709808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789094692816865:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:52.710536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:52.710742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789094692816964:2049] [node 2] ICS07 socket disconnect 301 reason# EndOfStream
2025-11-28T14:51:52.710780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789094692816964:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.710879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:52.710947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789094692816964:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:52.712742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789094692816866:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:52.712853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789093419886120:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:52.712808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789094692816964:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:52.713404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:52.713575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789094158941498:2049] [node 2] ICS07 socket disconnect 298 reason# EndOfStream
2025-11-28T14:51:52.713598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789094158941498:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.713662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:52.713701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789094158941498:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:52.713879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789095363798843:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:52.714088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789094692816866:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.714127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789094692816964:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.714152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789094692816866:2049] [node 2] ICP30 unregister session Session# [3:7577789094692816964:2049] VirtualId# [3:7577789094692816962:0]
2025-11-28T14:51:52.714236Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:52.714211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:52.714259Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789094692816866:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.714273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.714375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789093419886222:2049] [node 2] ICS07 socket disconnect 302 reason# EndOfStream
2025-11-28T14:51:52.714401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789093419886222:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.714451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:52.714490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789093419886222:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:52.714777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789094158941398:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.714912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789094158941498:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.714966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789094158941398:2049] [node 2] ICP30 unregister session Session# [4:7577789094158941498:2049] VirtualId# [4:7577789094158941496:0]
2025-11-28T14:51:52.715008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:52.715020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:52.715083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789094158941398:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.715097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.715235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789095363798948:2049] [node 2] ICS01 socket: 274 reason# EndOfStream
2025-11-28T14:51:52.715306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789095363798844:2049] [node 2] ICP30 unregister session Session# [6:7577789095363798948:2049] VirtualId# [6:7577789095363798946:0]
2025-11-28T14:51:52.715343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789095363798948:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.715420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789093419886121:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.715438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:52.715457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789093419886222:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.715551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:52.715480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789093419886121:2049] [node 2] ICP30 unregister session Session# [5:7577789093419886222:2049] VirtualId# [5:7577789093419886220:0]
2025-11-28T14:51:52.715516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:52.715537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789093419886121:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.715568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5871) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.723333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:52.723519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789094158941494:2050] [node 3] ICS07 socket disconnect 289 reason# EndOfStream
2025-11-28T14:51:52.723542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789094158941494:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.723609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:52.723637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789094158941494:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:52.724301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789094158941399:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15462) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.724320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:52.724411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789094158941494:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.724472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789094158941399:2050] [node 3] ICP30 unregister session Session# [4:7577789094158941494:2050] VirtualId# [4:7577789094158941492:0]
2025-11-28T14:51:52.724516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:52.724588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789094158941399:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.724603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789093419886209:2050] [node 3] ICS07 socket disconnect 290 reason# EndOfStream
2025-11-28T14:51:52.724626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.724661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789093419886209:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.724774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:52.724828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789093419886209:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:52.725113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789094158941398:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:52.725124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789093419886121:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:52.725243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:52.725435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789095363798944:2050] [node 3] ICS01 socket: 332 reason# EndOfStream
2025-11-28T14:51:52.725478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789095363798845:2050] [node 3] ICP30 unregister session Session# [6:7577789095363798944:2050] VirtualId# [6:7577789095363798942:0]
2025-11-28T14:51:52.725506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789095363798944:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.725555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789093419886122:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15462) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.725593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:52.725657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:52.725619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789093419886209:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.725708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789093419886122:2050] [node 3] ICP30 unregister session Session# [5:7577789093419886209:2050] VirtualId# [5:7577789093419886206:0]
2025-11-28T14:51:52.725822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:52.725858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789093419886122:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.725873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.735399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789093419886122:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:52.763425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:52.763662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789093419886214:2108] [node 4] ICS07 socket disconnect 329 reason# EndOfStream
2025-11-28T14:51:52.763729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789093419886214:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.763856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:52.763909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789093419886214:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:52.764197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:52.764410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789095363798934:2110] [node 4] ICS07 socket disconnect 321 reason# EndOfStream
2025-11-28T14:51:52.764448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789095363798934:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.764536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:52.764595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789095363798934:2110] [node 4] ICS15 start handshake
2025-11-28T14:51:52.764889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789093419886184:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16791) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.765018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789093419886214:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.765072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789093419886184:2108] [node 4] ICP30 unregister session Session# [5:7577789093419886214:2108] VirtualId# [5:7577789093419886212:0]
2025-11-28T14:51:52.765119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:52.765223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789093419886184:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16791) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.765267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16791) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.765506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789095363798912:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16791) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:52.765621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789095363798934:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:52.765688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789095363798912:2110] [node 4] ICP30 unregister session Session# [6:7577789095363798934:2110] VirtualId# [6:7577789095363798932:0]
2025-11-28T14:51:52.765748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:52.765792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789095363798912:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16791) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.765833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16791) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:52.774120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:52.774382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789095363798939:2108] [node 5] ICS01 socket: 297 reason# EndOfStream
2025-11-28T14:51:52.774437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789095363798906:2108] [node 5] ICP30 unregister session Session# [6:7577789095363798939:2108] VirtualId# [6:7577789095363798937:0]
2025-11-28T14:51:52.774488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789095363798939:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:52.774605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:52.774667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:52.775310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789095363798912:2110] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:52.835189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789096164585263:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:52.835252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789096164585264:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:52.858442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789095158659429:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:52.858954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789095158659430:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:52.883227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789093618200753:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:52.883253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789093618200752:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:52.907546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789093415488434:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:52.907603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789093415488435:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:52.907775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789093415488433:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:52.933548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789094028566188:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:52.933625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789094028566189:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:52.933788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789094028566190:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:52.957209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789094958508698:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:52.957863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789094958508697:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:52.958025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789094958508696:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:53.011931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789100459552620:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:53.042233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789099253476054:2105] [node 5] ICP01 ready to work
2025-11-28T14:51:53.052227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789097913168111:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:53.053666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789098323533565:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:53.053490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789100459552624:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:53.053669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789100459552625:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:53.053273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789099453626789:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:53.053725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789099453626790:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:53.054072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789097913168112:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:53.054088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789099253476055:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:53.054674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789097913168113:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:53.054325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789099453626791:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:53.055201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789098323533566:2126] [node 6] ICP01 ready to work
2025-11-28T14:51:53.053645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789097710455792:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:53.054329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789097710455793:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:53.061797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789094958508696:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42935547ns) {Configure:104(42929987ns) {ProcessConfigured:135(642123ns) {StartInitialHandshake:157(640130ns) {DropHandshakes:470(1730ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:477(1824ns) {SwitchToState:179(585ns)}}}}}
2025-11-28T14:51:53.061802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789094958508698:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39899688ns) {Configure:104(39893230ns) {ProcessConfigured:135(624042ns) {StartInitialHandshake:157(620890ns) {DropHandshakes:470(1355ns) {DropIncomingHandshake:442(402ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(617ns) PrepareNewSessionHandshake:477(1922ns) {SwitchToState:179(758ns)}}}}}
2025-11-28T14:51:53.062580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789096164585263:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51915873ns) {Configure:104(50503425ns) {ProcessConfigured:135(719746ns) {StartInitialHandshake:157(716682ns) {DropHandshakes:470(1471ns) {DropIncomingHandshake:442(445ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(524ns) PrepareNewSessionHandshake:477(1749ns) {SwitchToState:179(619ns)}}}}}
2025-11-28T14:51:53.062531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789094958508697:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18777597ns) {Configure:104(18768813ns) {ProcessConfigured:135(774361ns) {StartInitialHandshake:157(772582ns) {DropHandshakes:470(1004ns) {DropIncomingHandshake:442(256ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(388ns) PrepareNewSessionHandshake:477(772ns) {SwitchToState:179(383ns)}}}}}
2025-11-28T14:51:53.062593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789100459552620:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24857457ns) {Configure:104(19035996ns) {ProcessConfigured:135(685594ns) {StartInitialHandshake:157(683169ns) {DropHandshakes:470(1263ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(536ns) PrepareNewSessionHandshake:477(1643ns) {SwitchToState:179(629ns)}}}}}
2025-11-28T14:51:53.064238Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789093618200752:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49215427ns) {Configure:104(49211225ns) {ProcessConfigured:135(775356ns) {StartInitialHandshake:157(772871ns) {DropHandshakes:470(1241ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:477(1005ns) {SwitchToState:179(428ns)}}}}}
2025-11-28T14:51:53.063794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789096164585264:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52330653ns) {Configure:104(52320837ns) {ProcessConfigured:135(780677ns) {StartInitialHandshake:157(778272ns) {DropHandshakes:470(4825ns) {DropIncomingHandshake:442(519ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(1025ns) PrepareNewSessionHandshake:477(1513ns) {SwitchToState:179(686ns)}}}}}
2025-11-28T14:51:53.064978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789095158659429:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43178126ns) {Configure:104(43174005ns) {ProcessConfigured:135(46372ns) {StartInitialHandshake:157(46085ns) {DropHandshakes:470(8654ns) {DropIncomingHandshake:442(8101ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:477(635ns) {SwitchToState:179(289ns)}}}}}
2025-11-28T14:51:53.065274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789094028566189:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46273631ns) {Configure:104(42578081ns) {ProcessConfigured:135(929256ns) {StartInitialHandshake:157(927720ns) {DropHandshakes:470(1071ns) {DropIncomingHandshake:442(296ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(1443ns) {SwitchToState:179(286ns)}}}}}
2025-11-28T14:51:53.065127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789093415488434:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43122978ns) {Configure:104(43113285ns) {ProcessConfigured:135(760368ns) {StartInitialHandshake:157(757912ns) {DropHandshakes:470(1133ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:477(1789ns) {SwitchToState:179(692ns)}}}}}
2025-11-28T14:51:53.064328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789093618200753:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48187877ns) {Configure:104(48184075ns) {ProcessConfigured:135(990924ns) {StartInitialHandshake:157(988211ns) {DropHandshakes:470(1096ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(590ns) PrepareNewSessionHandshake:477(1956ns) {SwitchToState:179(791ns)}}}}}
2025-11-28T14:51:53.065547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789094028566190:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(46153280ns) {Configure:104(46147016ns) {ProcessConfigured:135(1041701ns) {StartInitialHandshake:157(1039684ns) {DropHandshakes:470(401382ns) {DropIncomingHandshake:442(1066ns) DropOutgoingHandshake:457(299ns)} GenerateSessionVirtualId:988(1531ns) PrepareNewSessionHandshake:477(1581ns) {SwitchToState:179(765ns)}}}}}
2025-11-28T14:51:53.066175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789094028566188:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46785163ns) {Configure:104(46781879ns) {ProcessConfigured:135(667715ns) {StartInitialHandshake:157(377649ns) {DropHandshakes:470(1759ns) {DropIncomingHandshake:442(688ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(1211ns) PrepareNewSessionHandshake:477(1491ns) {SwitchToState:179(779ns)}}}}}
2025-11-28T14:51:53.067029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789095158659430:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(38749701ns) {Configure:104(28700846ns) {ProcessConfigured:135(734182ns) {StartInitialHandshake:157(731861ns) {DropHandshakes:470(1397ns) {DropIncomingHandshake:442(354ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(793ns) PrepareNewSessionHandshake:477(1502ns) {SwitchToState:179(670ns)}}}}}
2025-11-28T14:51:53.067350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789093415488433:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44924648ns) {Configure:104(44917247ns) {ProcessConfigured:135(1999725ns) {StartInitialHandshake:157(1999420ns) {DropHandshakes:470(155176ns) {DropIncomingHandshake:442(245ns) DropOutgoingHandshake:457(470ns)} GenerateSessionVirtualId:988(484ns) PrepareNewSessionHandshake:477(1514ns) {SwitchToState:179(1187ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.071208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789100459552638:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17680
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.075878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789097710455804:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37906
2025-11-28T14:51:53.075846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789099453626806:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52090
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.076412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789097913168127:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10794
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.078106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789099453626807:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52090
2025-11-28T14:51:53.077788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098323533581:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.078178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789097710455804:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:51:53.077327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476070:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.078133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789100459552641:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17680
2025-11-28T14:51:53.079837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789097913168128:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10794
2025-11-28T14:51:53.080825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789093415488433:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.080754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476071:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
2025-11-28T14:51:53.080070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789100459552640:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17680
2025-11-28T14:51:53.080140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789099453626806:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.077791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098323533582:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-28T14:51:53.080140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789097913168127:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.080793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789100459552638:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.080970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789099453626805:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52090
2025-11-28T14:51:53.078578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098323533580:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-28T14:51:53.080772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789099253476070:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.081413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789097710455807:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37906
2025-11-28T14:51:53.080997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098323533581:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.082320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789099453626808:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52090
2025-11-28T14:51:53.081522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789100459552631:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.089611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789097913168116:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.081554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789094028566188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.089639Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789097913168116:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.089678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789097710455807:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.081554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789100459552631:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.081931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098323533583:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-28T14:51:53.080852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476072:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
2025-11-28T14:51:53.089595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789100459552634:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.082582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098323533580:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.081506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789094958508698:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.089621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789100459552634:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.088928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098323533582:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.090850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789099253476071:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.090842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789094028566189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.090948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789099453626799:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.090855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789093415488435:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.091063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789094958508697:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.090985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789099453626799:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.091465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789097913168116:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:53.091836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789100459552634:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:53.091858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789099453626798:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.091855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789100459552631:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:53.092501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789097913168121:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.091894Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789099453626798:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.092535Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789097913168121:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.092551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789099453626799:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:53.092608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789097913168113:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.093070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098323533587:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-28T14:51:53.092931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.093965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789097710455811:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37906
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.093072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789097913168113:2109] [node 6] ICP22 created new session: [3:7577789097913168131:2109]
2025-11-28T14:51:53.093684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789100459552620:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.094866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789094958508698:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10794) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789099253476070:2115] held: no
2025-11-28T14:51:53.093725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.095384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789099453626798:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:53.095134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789097913168131:2109] [node 6] ICS09 handshake done sender: [3:7577789097913168116:2110] self: [3:7577789097913168115:0] peer: [6:7577789099253476073:0] socket: 282 qp: -1
2025-11-28T14:51:53.095725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476075:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.095434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789099453626790:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.095192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789097913168131:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:53.094037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789100459552620:2107] [node 4] ICP22 created new session: [1:7577789100459552645:2107]
2025-11-28T14:51:53.096806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789097710455804:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.097124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098323533581:2134] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.095466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.094725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789100459552624:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.095375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789097913168121:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:53.097885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789099253476070:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.097884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098323533588:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-28T14:51:53.095892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:53.095186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.095875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789099453626790:2108] [node 5] ICP22 created new session: [2:7577789099453626812:2108]
2025-11-28T14:51:53.099265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789093415488433:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.098806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789094958508698:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.096542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789099453626791:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.099712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789094028566188:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.095374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789100459552624:2108] [node 5] ICP22 created new session: [1:7577789100459552647:2108]
2025-11-28T14:51:53.097142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789097913168111:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.099795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:53.099092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:53.100110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.096750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789099453626812:2108] [node 5] ICS09 handshake done sender: [2:7577789099453626799:2113] self: [2:7577789099453626795:0] peer: [5:7577789098323533586:0] socket: 297 qp: -1
2025-11-28T14:51:53.097181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.095703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789100459552645:2107] [node 4] ICS09 handshake done sender: [1:7577789100459552631:2111] self: [1:7577789100459552629:0] peer: [4:7577789097710455809:0] socket: 272 qp: -1
2025-11-28T14:51:53.100193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789093415488433:2048] [node 1] ICP22 created new session: [4:7577789097710455813:2048]
2025-11-28T14:51:53.100749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789094028566188:2048] [node 1] ICP22 created new session: [5:7577789098323533589:2048]
2025-11-28T14:51:53.099809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789094958508698:2050] [node 3] ICP22 created new session: [6:7577789099253476079:2050]
2025-11-28T14:51:53.095876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789100459552647:2108] [node 5] ICS09 handshake done sender: [1:7577789100459552634:2114] self: [1:7577789100459552630:0] peer: [5:7577789098323533584:0] socket: 277 qp: -1
No RDMA context found for GID: 2025-11-28T14:51:53.098321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789097913168111:2107] [node 4] ICP22 created new session: [3:7577789097913168135:2107]
::ffff:127.1.0.0
2025-11-28T14:51:53.101368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789097710455813:2048] [node 1] ICS09 handshake done sender: [4:7577789097710455804:2115] self: [4:7577789097710455809:0] peer: [1:7577789100459552629:0] socket: 278 qp: -1
2025-11-28T14:51:53.096792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789099453626812:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:53.100153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476076:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.102421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098323533589:2048] [node 1] ICS09 handshake done sender: [5:7577789098323533581:2134] self: [5:7577789098323533584:0] peer: [1:7577789100459552630:0] socket: 295 qp: -1
2025-11-28T14:51:53.102155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789097710455813:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:53.099786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789097913168135:2107] [node 4] ICS09 handshake done sender: [3:7577789097913168121:2112] self: [3:7577789097913168117:0] peer: [4:7577789097710455810:0] socket: 289 qp: -1
2025-11-28T14:51:53.096115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789100459552647:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:53.102587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098323533582:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:53.101623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789099253476079:2050] [node 3] ICS09 handshake done sender: [6:7577789099253476070:2115] self: [6:7577789099253476073:0] peer: [3:7577789097913168115:0] socket: 284 qp: -1
2025-11-28T14:51:53.096797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.096234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789100459552645:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:53.102567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:53.103194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098323533589:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:53.100303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789097913168135:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:53.101666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789099253476079:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:53.096846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.102396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.103237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:53.096273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:53.102597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789099253476071:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:53.097092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789099453626791:2109] [node 6] ICP22 created new session: [2:7577789099453626813:2109]
2025-11-28T14:51:53.096881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.100836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:53.098219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789099453626813:2109] [node 6] ICS09 handshake done sender: [2:7577789099453626798:2112] self: [2:7577789099453626794:0] peer: [6:7577789099253476074:0] socket: 296 qp: -1
2025-11-28T14:51:53.103531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476077:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
2025-11-28T14:51:53.104774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789097913168137:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10794
2025-11-28T14:51:53.101739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789100459552646:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17680
2025-11-28T14:51:53.104013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789094958508697:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.105219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789097913168136:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10794
2025-11-28T14:51:53.098537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789099453626813:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:53.104061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:53.104238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789100459552646:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.105773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789094028566189:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.104303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789094958508697:2049] [node 2] ICP22 created new session: [6:7577789099253476081:2049]
2025-11-28T14:51:53.098760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:53.105823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789097913168136:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.105818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789100459552635:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.104761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789099253476077:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.105997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:53.105839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789100459552635:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.106456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789094028566189:2049] [node 2] ICP22 created new session: [5:7577789098323533592:2049]
2025-11-28T14:51:53.105069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789099253476081:2049] [node 2] ICS09 handshake done sender: [6:7577789099253476071:2116] self: [6:7577789099253476074:0] peer: [2:7577789099453626794:0] socket: 301 qp: -1
2025-11-28T14:51:53.106517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789093618200752:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.105251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789099253476081:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:53.106504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098323533592:2049] [node 2] ICS09 handshake done sender: [5:7577789098323533582:2135] self: [5:7577789098323533586:0] peer: [2:7577789099453626795:0] socket: 298 qp: -1
2025-11-28T14:51:53.105288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789094958508696:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.106677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098323533592:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:53.105291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:53.106714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.106873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789100459552633:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.106904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789100459552633:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.108294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789097710455812:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37906
2025-11-28T14:51:53.107818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789100459552635:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.108445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789100459552625:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.108622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098323533590:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-28T14:51:53.108510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.108643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476084:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
2025-11-28T14:51:53.108560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789100459552625:2109] [node 6] ICP22 created new session: [1:7577789100459552655:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.108596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789100459552633:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:53.108669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789100459552655:2109] [node 6] ICS09 handshake done sender: [1:7577789100459552635:2115] self: [1:7577789100459552626:0] peer: [6:7577789099253476082:0] socket: 324 qp: -1
2025-11-28T14:51:53.109169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789097710455815:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37906
2025-11-28T14:51:53.109147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789100459552655:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:53.109300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789097710455816:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37906
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.109475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789094958508696:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789099253476077:2120] held: no
2025-11-28T14:51:53.109316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:53.109409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789097710455807:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:53.109408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789096164585264:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.109411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098323533590:2139] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.110660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789097913168123:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.110695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789097913168123:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.110664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789093415488435:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.110023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789094028566190:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.110616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:53.110676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789099253476077:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.110651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789096164585264:2049] [node 3] ICP22 created new session: [1:7577789100459552657:2049]
2025-11-28T14:51:53.110696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:53.110901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789097913168141:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10794
2025-11-28T14:51:53.110923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789100459552657:2049] [node 3] ICS09 handshake done sender: [1:7577789100459552633:2113] self: [1:7577789100459552627:0] peer: [3:7577789097913168140:0] socket: 273 qp: -1
2025-11-28T14:51:53.110716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789093415488435:2050] [node 3] ICP22 created new session: [4:7577789097710455818:2050]
2025-11-28T14:51:53.111722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789100459552657:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:53.110825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789097710455815:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.110883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789097710455818:2050] [node 3] ICS09 handshake done sender: [4:7577789097710455807:2116] self: [4:7577789097710455810:0] peer: [3:7577789097913168117:0] socket: 300 qp: -1
2025-11-28T14:51:53.111760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.113136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789093618200752:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789097913168136:2117] held: no
2025-11-28T14:51:53.111729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789097710455818:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:53.111933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789094958508696:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.112002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:53.111767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.113395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789097913168123:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:53.113154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789094958508696:2048] [node 1] ICP22 created new session: [6:7577789099253476085:2048]
2025-11-28T14:51:53.113846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789097913168112:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.113873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.113394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789099253476085:2048] [node 1] ICS09 handshake done sender: [6:7577789099253476077:2120] self: [6:7577789099253476082:0] peer: [1:7577789100459552626:0] socket: 326 qp: -1
2025-11-28T14:51:53.114021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789097913168136:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.113520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789099253476085:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.114095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789097913168112:2108] [node 5] ICP22 created new session: [3:7577789097913168144:2108]
2025-11-28T14:51:53.113636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:53.114127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789097913168142:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10794
2025-11-28T14:51:53.114166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789093618200752:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.114348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098323533595:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-28T14:51:53.114189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789097913168144:2108] [node 5] ICS09 handshake done sender: [3:7577789097913168123:2114] self: [3:7577789097913168120:0] peer: [5:7577789098323533594:0] socket: 332 qp: -1
2025-11-28T14:51:53.114209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789097913168144:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:53.114332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.114339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:53.114406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789093618200752:2048] [node 1] ICP22 created new session: [3:7577789097913168145:2048]
2025-11-28T14:51:53.114467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789097913168145:2048] [node 1] ICS09 handshake done sender: [3:7577789097913168136:2117] self: [3:7577789097913168140:0] peer: [1:7577789100459552627:0] socket: 327 qp: -1
2025-11-28T14:51:53.114528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789097913168145:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:53.114559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:53.114743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098323533590:2139] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:53.115133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789094028566190:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.115186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:53.115297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789094028566190:2050] [node 3] ICP22 created new session: [5:7577789098323533596:2050]
2025-11-28T14:51:53.115361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098323533596:2050] [node 3] ICS09 handshake done sender: [5:7577789098323533590:2139] self: [5:7577789098323533594:0] peer: [3:7577789097913168120:0] socket: 333 qp: -1
2025-11-28T14:51:53.115388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098323533596:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:53.115426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.118756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789098323533572:2128] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.119735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789099453626807:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.119951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789099453626790:2108] [node 5] ICP18 handshake (actor [2:7577789099453626807:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.120584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476087:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
2025-11-28T14:51:53.130979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789098323533573:2129] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.141987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789097913168128:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.142243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789097913168112:2108] [node 5] ICP18 handshake (actor [3:7577789097913168128:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.142839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789097710455821:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37906
2025-11-28T14:51:53.146156Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789097710455801:2112] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.154131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789099253476072:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.154311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789099253476055:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.154476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789097710455801:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.154972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789097710455801:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.155123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476089:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
2025-11-28T14:51:53.155193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789097710455793:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.155263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.155309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789097710455793:2109] [node 6] ICP22 created new session: [4:7577789097710455823:2109]
2025-11-28T14:51:53.155400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789097710455823:2109] [node 6] ICS09 handshake done sender: [4:7577789097710455801:2112] self: [4:7577789097710455796:0] peer: [6:7577789099253476088:0] socket: 306 qp: -1
2025-11-28T14:51:53.155443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789099253476072:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:53.155448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789097710455823:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:53.155490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:53.155713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789099253476055:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.155945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.156437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789099253476055:2108] [node 4] ICP22 created new session: [6:7577789099253476090:2108]
2025-11-28T14:51:53.156474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789099253476090:2108] [node 4] ICS09 handshake done sender: [6:7577789099253476072:2117] self: [6:7577789099253476088:0] peer: [4:7577789097710455796:0] socket: 311 qp: -1
2025-11-28T14:51:53.156502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789099253476090:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:53.156549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:53.156791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789099253476055:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:53.158418Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789100459552632:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.166729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789099453626805:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.166947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789095158659429:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.167195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789100459552632:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.167704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789100459552632:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.167948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789096164585263:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.167944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789099453626819:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52090
2025-11-28T14:51:53.168023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:53.168092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789096164585263:2048] [node 2] ICP22 created new session: [1:7577789100459552660:2048]
2025-11-28T14:51:53.168143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789100459552660:2048] [node 2] ICS09 handshake done sender: [1:7577789100459552632:2112] self: [1:7577789100459552628:0] peer: [2:7577789099453626818:0] socket: 274 qp: -1
2025-11-28T14:51:53.168199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789100459552660:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:53.168230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789099453626805:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.168271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:53.168556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789095158659429:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.168616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:53.168723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789095158659429:2048] [node 1] ICP22 created new session: [2:7577789099453626820:2048]
2025-11-28T14:51:53.168791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789099453626820:2048] [node 1] ICS09 handshake done sender: [2:7577789099453626805:2116] self: [2:7577789099453626818:0] peer: [1:7577789100459552628:0] socket: 283 qp: -1
2025-11-28T14:51:53.168830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789099453626820:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:53.168877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:53.190285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789099453626793:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.211976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789100459552640:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.212193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789096164585263:2048] [node 2] ICP18 handshake (actor [1:7577789100459552640:2117]) is too old
2025-11-28T14:51:53.223582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789097710455797:2110] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.245203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098323533583:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.245517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789098323533565:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.245701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789097710455797:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.246157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789097710455797:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.246365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789097710455792:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.246409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.246463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789097710455792:2108] [node 5] ICP22 created new session: [4:7577789097710455826:2108]
2025-11-28T14:51:53.246495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789097710455826:2108] [node 5] ICS09 handshake done sender: [4:7577789097710455797:2110] self: [4:7577789097710455794:0] peer: [5:7577789098323533600:0] socket: 305 qp: -1
2025-11-28T14:51:53.246502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098323533601:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-28T14:51:53.246525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789097710455826:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:53.246559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.246734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098323533583:2136] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:53.246986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789098323533565:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.247045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.247094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789098323533565:2108] [node 4] ICP22 created new session: [5:7577789098323533602:2108]
2025-11-28T14:51:53.247143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098323533602:2108] [node 4] ICS09 handshake done sender: [5:7577789098323533583:2136] self: [5:7577789098323533600:0] peer: [4:7577789097710455794:0] socket: 308 qp: -1
2025-11-28T14:51:53.247178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098323533602:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:53.247222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:53.257940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789098323533574:2130] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.265661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789100459552641:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.265940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789100459552624:2108] [node 5] ICP18 handshake (actor [1:7577789100459552641:2118]) is too old
2025-11-28T14:51:53.269528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789097710455799:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.291135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789099453626808:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.293678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789099453626800:2114] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.301624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789097913168137:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.301848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789093618200753:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.302127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789099453626800:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.302581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789099453626800:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.302792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789095158659430:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.302830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789097913168149:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10794
2025-11-28T14:51:53.302932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:53.302993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789095158659430:2049] [node 3] ICP22 created new session: [2:7577789099453626824:2049]
2025-11-28T14:51:53.303053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789099453626824:2049] [node 3] ICS09 handshake done sender: [2:7577789099453626800:2114] self: [2:7577789099453626796:0] peer: [3:7577789097913168148:0] socket: 330 qp: -1
2025-11-28T14:51:53.303081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789097913168137:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:53.303114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789099453626824:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:53.303186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.303226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789093618200753:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.303432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:53.303470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789093618200753:2049] [node 2] ICP22 created new session: [3:7577789097913168150:2049]
2025-11-28T14:51:53.303664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789097913168150:2049] [node 2] ICS09 handshake done sender: [3:7577789097913168137:2118] self: [3:7577789097913168148:0] peer: [2:7577789099453626796:0] socket: 329 qp: -1
2025-11-28T14:51:53.304400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789097913168150:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:53.304484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:53.305676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789099453626801:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.306260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789097710455816:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.306436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789093415488434:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.306630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789099453626801:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.311819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789099453626801:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.313539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789099453626789:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.313617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.313633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789097710455829:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37906
2025-11-28T14:51:53.313695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789099453626789:2107] [node 4] ICP22 created new session: [2:7577789099453626827:2107]
2025-11-28T14:51:53.313777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789099453626827:2107] [node 4] ICS09 handshake done sender: [2:7577789099453626801:2115] self: [2:7577789099453626797:0] peer: [4:7577789097710455828:0] socket: 331 qp: -1
2025-11-28T14:51:53.313825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789099453626827:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:53.313863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:53.313917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789097710455816:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:53.314438Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789093415488434:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789097710455816:2120] held: no
2025-11-28T14:51:53.314627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789093415488434:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.314781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:53.314889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789093415488434:2049] [node 2] ICP22 created new session: [4:7577789097710455830:2049]
2025-11-28T14:51:53.315318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789097710455830:2049] [node 2] ICS09 handshake done sender: [4:7577789097710455816:2120] self: [4:7577789097710455828:0] peer: [2:7577789099453626797:0] socket: 335 qp: -1
2025-11-28T14:51:53.315361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789097710455830:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:53.315411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:53.318994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789098323533575:2131] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.327433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789099253476087:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.327680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789099253476054:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.327952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098323533575:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.328556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098323533575:2131] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.328712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789099253476093:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33621
2025-11-28T14:51:53.328806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789098323533566:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.328896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.328936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789098323533566:2126] [node 6] ICP22 created new session: [5:7577789098323533605:2126]
2025-11-28T14:51:53.328979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098323533605:2126] [node 6] ICS09 handshake done sender: [5:7577789098323533575:2131] self: [5:7577789098323533568:0] peer: [6:7577789099253476092:0] socket: 271 qp: -1
2025-11-28T14:51:53.329016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098323533605:2126] [node 6] ICS10 traffic start
2025-11-28T14:51:53.329043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:53.329003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789099253476087:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:53.329129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789099253476054:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.329202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.329317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789099253476054:2105] [node 5] ICP22 created new session: [6:7577789099253476094:2105]
2025-11-28T14:51:53.329428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789099253476094:2105] [node 5] ICS09 handshake done sender: [6:7577789099253476087:2122] self: [6:7577789099253476092:0] peer: [5:7577789098323533568:0] socket: 280 qp: -1
2025-11-28T14:51:53.329524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789099253476094:2105] [node 5] ICS10 traffic start
2025-11-28T14:51:53.329559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.334102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789098323533576:2132] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:53.334855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789097710455821:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.335033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789097710455792:2108] [node 5] ICP18 handshake (actor [4:7577789097710455821:2121]) is too old
2025-11-28T14:51:53.350272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:53.350481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789099453626820:2048] [node 1] ICS07 socket disconnect 283 reason# EndOfStream
2025-11-28T14:51:53.350523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789099453626820:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.350649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.350723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789099453626820:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:53.351598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:53.351836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789097913168145:2048] [node 1] ICS01 socket: 327 reason# EndOfStream
2025-11-28T14:51:53.351899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789093618200752:2048] [node 1] ICP30 unregister session Session# [3:7577789097913168145:2048] VirtualId# [3:7577789097913168140:0]
2025-11-28T14:51:53.351935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789097913168145:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.352061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.352134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.353147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789095158659429:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:53.353195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789099453626820:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:53.354269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789095158659429:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.354390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789099453626820:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.354437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789095158659429:2048] [node 1] ICP30 unregister session Session# [2:7577789099453626820:2048] VirtualId# [2:7577789099453626818:0]
2025-11-28T14:51:53.354489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.354528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789095158659429:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.354564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.354734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:53.354962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789097710455813:2048] [node 1] ICS07 socket disconnect 278 reason# EndOfStream
2025-11-28T14:51:53.354997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789097710455813:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.355095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.355142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789097710455813:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:53.355782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789093415488433:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.355780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:53.355823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789097710455813:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.355844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789093415488433:2048] [node 1] ICP30 unregister session Session# [4:7577789097710455813:2048] VirtualId# [4:7577789097710455809:0]
2025-11-28T14:51:53.355889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.355907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789093415488433:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.355969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.356129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789098323533589:2048] [node 1] ICS07 socket disconnect 295 reason# EndOfStream
2025-11-28T14:51:53.356166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789098323533589:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.356241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.356287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789098323533589:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:53.357067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789094028566188:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.357114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:53.357165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789098323533589:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.357199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789094028566188:2048] [node 1] ICP30 unregister session Session# [5:7577789098323533589:2048] VirtualId# [5:7577789098323533584:0]
2025-11-28T14:51:53.357248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.357280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789094028566188:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.357295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.357382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789099253476085:2048] [node 1] ICS01 socket: 326 reason# EndOfStream
2025-11-28T14:51:53.357428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789094958508696:2048] [node 1] ICP30 unregister session Session# [6:7577789099253476085:2048] VirtualId# [6:7577789099253476082:0]
2025-11-28T14:51:53.357462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789099253476085:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.357537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.357584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.366178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789093415488433:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:53.367196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789094028566188:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:53.381935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:53.382189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789097913168150:2049] [node 2] ICS07 socket disconnect 329 reason# EndOfStream
2025-11-28T14:51:53.382225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789097913168150:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.382371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:53.382441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789097913168150:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:53.383328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789093618200753:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.383390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789097913168150:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.383463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789093618200753:2049] [node 2] ICP30 unregister session Session# [3:7577789097913168150:2049] VirtualId# [3:7577789097913168148:0]
2025-11-28T14:51:53.383694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:53.383734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789093618200753:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.383752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.384918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:53.385108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789097710455830:2049] [node 2] ICS01 socket: 335 reason# EndOfStream
2025-11-28T14:51:53.385148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789093415488434:2049] [node 2] ICP30 unregister session Session# [4:7577789097710455830:2049] VirtualId# [4:7577789097710455828:0]
2025-11-28T14:51:53.385177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789097710455830:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.385283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:53.385344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:53.385915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:53.386094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789098323533592:2049] [node 2] ICS07 socket disconnect 298 reason# EndOfStream
2025-11-28T14:51:53.386126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789098323533592:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.386198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:53.386248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789098323533592:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:53.386732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:53.386842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789094028566189:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.386874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789098323533592:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.386894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789094028566189:2049] [node 2] ICP30 unregister session Session# [5:7577789098323533592:2049] VirtualId# [5:7577789098323533586:0]
2025-11-28T14:51:53.386965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:53.386956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789099253476081:2049] [node 2] ICS07 socket disconnect 301 reason# EndOfStream
2025-11-28T14:51:53.386993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789094028566189:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.387003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789099253476081:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.387005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.387099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:53.387159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789099253476081:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:53.388141Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789094958508697:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.388229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789099253476081:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.388277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789094958508697:2049] [node 2] ICP30 unregister session Session# [6:7577789099253476081:2049] VirtualId# [6:7577789099253476074:0]
2025-11-28T14:51:53.388321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:53.388399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789094958508697:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.388414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.396909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789094028566189:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:53.397961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789094958508697:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:53.432973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:53.433210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789097710455818:2050] [node 3] ICS07 socket disconnect 300 reason# EndOfStream
2025-11-28T14:51:53.433257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789097710455818:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.433348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:53.433411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789097710455818:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:53.434050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:53.434304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789098323533596:2050] [node 3] ICS07 socket disconnect 333 reason# EndOfStream
2025-11-28T14:51:53.434444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789093415488435:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10794) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.434517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789097710455818:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.434553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789093415488435:2050] [node 3] ICP30 unregister session Session# [4:7577789097710455818:2050] VirtualId# [4:7577789097710455810:0]
2025-11-28T14:51:53.434402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789098323533596:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.434603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:53.434485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:53.434647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789093415488435:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10794) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.434533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789098323533596:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:53.434661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10794) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.435523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789094028566190:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10794) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.435630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789098323533596:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.435697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789094028566190:2050] [node 3] ICP30 unregister session Session# [5:7577789098323533596:2050] VirtualId# [5:7577789098323533594:0]
2025-11-28T14:51:53.435632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:53.435744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:53.435805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789099253476079:2050] [node 3] ICS01 socket: 284 reason# EndOfStream
2025-11-28T14:51:53.435826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789094028566190:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10794) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.435875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10794) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.435859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789094958508698:2050] [node 3] ICP30 unregister session Session# [6:7577789099253476079:2050] VirtualId# [6:7577789099253476073:0]
2025-11-28T14:51:53.435886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789099253476079:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.435957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:53.436012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:53.446080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789094028566190:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:53.446169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:53.446318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789098323533602:2108] [node 4] ICS07 socket disconnect 308 reason# EndOfStream
2025-11-28T14:51:53.446345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789098323533602:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.446435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:53.446490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789098323533602:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:53.447466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:53.447652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789099253476090:2108] [node 4] ICS07 socket disconnect 311 reason# EndOfStream
2025-11-28T14:51:53.447684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789099253476090:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.447760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:53.447814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789099253476090:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:53.448503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789099253476055:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4756) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.448541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789099253476090:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.448625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789099253476055:2108] [node 4] ICP30 unregister session Session# [6:7577789099253476090:2108] VirtualId# [6:7577789099253476088:0]
2025-11-28T14:51:53.448697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:53.448718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789099253476055:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.448742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.447317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789098323533565:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4756) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.447401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789098323533602:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.447459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789098323533565:2108] [node 4] ICP30 unregister session Session# [5:7577789098323533602:2108] VirtualId# [5:7577789098323533600:0]
2025-11-28T14:51:53.447507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:53.447543Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789098323533565:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.447558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.459314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789099253476055:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:53.462767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:53.462990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789099253476094:2105] [node 5] ICS07 socket disconnect 280 reason# EndOfStream
2025-11-28T14:51:53.463029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789099253476094:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.463146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:53.463204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789099253476094:2105] [node 5] ICS15 start handshake
2025-11-28T14:51:53.464052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789099253476054:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13597) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.464150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789099253476094:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.464210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789099253476054:2105] [node 5] ICP30 unregister session Session# [6:7577789099253476094:2105] VirtualId# [6:7577789099253476092:0]
2025-11-28T14:51:53.464263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:53.464297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789099253476054:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13597) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.464310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13597) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:53.537762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789098645094195:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:53.537821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789098645094196:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:53.556394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789097642670482:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:53.556432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789097642670483:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:53.573575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789098765572807:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:53.573610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789098765572808:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:53.594438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789098779016439:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:53.594477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789098779016440:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:53.594508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789098779016441:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:53.616541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789098182921644:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:53.616623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789098182921645:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:53.616679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789098182921646:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:53.637424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789098700411404:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:53.637503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789098700411405:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:53.637542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789098700411406:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:53.699032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789098700411465:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:53.786261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789098645094259:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:53.790792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789098182921717:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:53.791058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789097642670549:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:53.796551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789097642670550:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:53.797368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789098645094260:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:53.799701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789098700411465:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(30798383ns) {Configure:104(30780508ns) {ProcessConfigured:135(78831ns) {StartInitialHandshake:157(75750ns) {DropHandshakes:470(2374ns) {DropIncomingHandshake:442(746ns) DropOutgoingHandshake:457(370ns)} GenerateSessionVirtualId:988(1343ns) PrepareNewSessionHandshake:477(1474ns) {SwitchToState:179(438ns)}}}}}
2025-11-28T14:51:53.792578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789098765572871:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:53.797934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789098182921718:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:53.793346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789098765572872:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:53.812682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789098779016439:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(59419925ns) {Configure:104(59416419ns) {ProcessConfigured:135(41487ns) {StartInitialHandshake:157(40050ns) {DropHandshakes:470(1148ns) {DropIncomingHandshake:442(231ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:477(670ns) {SwitchToState:179(180ns)}}}}}
2025-11-28T14:51:53.800982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789098645094261:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:53.793626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789098765572873:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:53.805256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789097642670551:2111] [node 6] ICP01 ready to work
2025-11-28T14:51:53.813109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789098779016522:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:53.818532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789098700411481:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:53.813378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789098779016523:2116] [node 6] ICP01 ready to work
2025-11-28T14:51:53.820347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789098765572808:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22969518ns) {Configure:104(22965466ns) {ProcessConfigured:135(11466059ns) {StartInitialHandshake:157(11464161ns) {DropHandshakes:470(745ns) {DropIncomingHandshake:442(156ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(2041ns) {SwitchToState:179(893ns)}}}}}
2025-11-28T14:51:53.838560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789098700411405:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16572335ns) {Configure:104(16558799ns) {ProcessConfigured:135(1532440ns) {StartInitialHandshake:157(1529835ns) {DropHandshakes:470(1586ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:477(2866ns) {SwitchToState:179(1397ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.845964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789098779016528:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25860
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.840835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789097642670562:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.846854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789097642670563:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-28T14:51:53.846604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789098645094275:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13373
2025-11-28T14:51:53.848527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789097642670562:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.846245Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789098765572871:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25687220ns) {Configure:104(25683524ns) {ProcessConfigured:135(5959336ns) {StartInitialHandshake:157(5957043ns) {DropHandshakes:470(2745ns) {DropIncomingHandshake:442(573ns) DropOutgoingHandshake:457(578ns)} GenerateSessionVirtualId:988(595ns) PrepareNewSessionHandshake:477(2213ns) {SwitchToState:179(749ns)}}}}}
2025-11-28T14:51:53.853903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789098645094276:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13373
2025-11-28T14:51:53.853987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789098645094277:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13373
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.861799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789098779016528:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.861392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789097642670482:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.863556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094267:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.863592Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789098645094267:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.864819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789098645094267:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:53.862487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789098779016536:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25860
2025-11-28T14:51:53.862883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789098779016439:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.867051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789098645094259:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.867116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.867620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789098645094259:2107] [node 4] ICP22 created new session: [1:7577789098645094279:2107]
2025-11-28T14:51:53.868794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789098645094279:2107] [node 4] ICS09 handshake done sender: [1:7577789098645094267:2113] self: [1:7577789098645094266:0] peer: [4:7577789098779016537:0] socket: 278 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.876231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789098779016538:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25860
2025-11-28T14:51:53.869786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789098645094279:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.877769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098182921732:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46097
2025-11-28T14:51:53.880157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789098779016528:2131] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.882288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789098779016439:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.883207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789097642670567:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-28T14:51:53.883409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789097642670563:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.869845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:53.877402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094275:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.883845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:53.883944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789098779016439:2048] [node 1] ICP22 created new session: [4:7577789098779016539:2048]
2025-11-28T14:51:53.878817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094263:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.878853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789098645094263:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:51:53.883836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094277:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:51:53.885975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094276:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.886234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789098779016539:2048] [node 1] ICS09 handshake done sender: [4:7577789098779016528:2131] self: [4:7577789098779016537:0] peer: [1:7577789098645094266:0] socket: 287 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.886292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789098779016539:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:53.886291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789098645094281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13373
2025-11-28T14:51:53.886603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789098645094263:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:53.886334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.886664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098182921733:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46097
2025-11-28T14:51:53.887939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789097642670569:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-28T14:51:53.888024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789097642670570:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-28T14:51:53.888554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789098645094259:2107] [node 4] ICP18 handshake (actor [1:7577789098645094277:2118]) is too old
2025-11-28T14:51:53.889483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789098645094195:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.889530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.889693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789098765572884:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57459
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.889841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789098765572885:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57459
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.889954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789098765572886:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57459
2025-11-28T14:51:53.890052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789098765572887:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57459
2025-11-28T14:51:53.890595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789098645094195:2048] [node 2] ICP22 created new session: [1:7577789098645094285:2048]
2025-11-28T14:51:53.890657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789098645094285:2048] [node 2] ICS09 handshake done sender: [1:7577789098645094263:2111] self: [1:7577789098645094262:0] peer: [2:7577789097642670568:0] socket: 276 qp: -1
2025-11-28T14:51:53.890694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789098645094285:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:53.890742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:53.891153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789098645094195:2048] [node 2] ICP18 handshake (actor [1:7577789098645094276:2117]) is too old
2025-11-28T14:51:53.891264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789098765572885:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.896596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789097642670562:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.895264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789098765572886:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.897020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098182921732:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.898285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789098779016536:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.898916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789097642670567:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.898962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789097642670482:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.898999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:53.899135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789097642670482:2048] [node 1] ICP22 created new session: [2:7577789097642670573:2048]
2025-11-28T14:51:53.899177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789097642670573:2048] [node 1] ICS09 handshake done sender: [2:7577789097642670562:2119] self: [2:7577789097642670568:0] peer: [1:7577789098645094262:0] socket: 282 qp: -1
2025-11-28T14:51:53.899221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789097642670573:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:53.899244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:53.899148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789098779016541:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25860
2025-11-28T14:51:53.899434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789098779016440:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.901245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789098779016541:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.900683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789097642670569:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.900834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789098765572887:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.901398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789098779016441:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.900965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789098765572884:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.901791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098182921735:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46097
2025-11-28T14:51:53.901256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789098765572808:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.901636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789098765572807:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.901812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789097642670572:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-28T14:51:53.902146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789098765572879:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.902344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098182921735:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.902168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789098765572879:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.902462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789097642670557:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.902503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789098182921717:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.902480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789097642670557:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.903110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789098779016530:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.903478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789097642670555:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.903134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789098779016530:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.903505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789097642670555:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.904658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094281:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.904317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789098779016530:2132] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.905082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789097642670572:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.905871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098182921737:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46097
2025-11-28T14:51:53.906339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789098765572879:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:53.905939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094265:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.906330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789098779016522:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.906407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.905990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789098645094265:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.906476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789098779016522:2108] [node 5] ICP22 created new session: [4:7577789098779016546:2108]
2025-11-28T14:51:53.906528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789098779016546:2108] [node 5] ICS09 handshake done sender: [4:7577789098779016530:2132] self: [4:7577789098779016529:0] peer: [5:7577789098182921736:0] socket: 291 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.906723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789098779016546:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:53.906943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098182921733:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.906998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098182921735:2127] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:53.907044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.907174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789098645094287:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13373
2025-11-28T14:51:53.907378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789097642670557:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:53.907456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789098182921717:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.908079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789098645094265:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:53.908170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.907399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789098765572871:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.908802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789098182921717:2108] [node 4] ICP22 created new session: [5:7577789098182921738:2108]
2025-11-28T14:51:53.907463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.908903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098182921738:2108] [node 4] ICS09 handshake done sender: [5:7577789098182921735:2127] self: [5:7577789098182921736:0] peer: [4:7577789098779016529:0] socket: 322 qp: -1
2025-11-28T14:51:53.907700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789098765572871:2107] [node 4] ICP22 created new session: [3:7577789098765572896:2107]
2025-11-28T14:51:53.908880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789098765572896:2107] [node 4] ICS09 handshake done sender: [3:7577789098765572879:2113] self: [3:7577789098765572878:0] peer: [4:7577789098779016544:0] socket: 319 qp: -1
2025-11-28T14:51:53.909643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098182921738:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:53.909679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:53.908922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789098765572896:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:53.909717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789097642670549:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.909655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.909803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.909906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789097642670549:2107] [node 4] ICP22 created new session: [2:7577789097642670581:2107]
2025-11-28T14:51:53.909972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789097642670581:2107] [node 4] ICS09 handshake done sender: [2:7577789097642670557:2116] self: [2:7577789097642670556:0] peer: [4:7577789098779016543:0] socket: 290 qp: -1
2025-11-28T14:51:53.910098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789097642670581:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.910179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:53.910089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411491:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
2025-11-28T14:51:53.910192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411492:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
2025-11-28T14:51:53.910683Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789097642670555:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.911576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789098765572898:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57459
2025-11-28T14:51:53.912279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789098779016548:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25860
2025-11-28T14:51:53.912589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789098182921644:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.912690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789098645094196:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.912751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.913252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789098779016549:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25860
2025-11-28T14:51:53.912796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789098645094196:2049] [node 3] ICP22 created new session: [1:7577789098645094291:2049]
2025-11-28T14:51:53.912910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789098645094291:2049] [node 3] ICS09 handshake done sender: [1:7577789098645094265:2112] self: [1:7577789098645094264:0] peer: [3:7577789098765572894:0] socket: 277 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.912971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789098645094291:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:53.912992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.912959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789098765572899:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57459
2025-11-28T14:51:53.913509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411493:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.916315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098182921741:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46097
2025-11-28T14:51:53.913642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789097642670483:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.914191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:53.917223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789097642670483:2049] [node 3] ICP22 created new session: [2:7577789097642670583:2049]
2025-11-28T14:51:53.917297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789097642670583:2049] [node 3] ICS09 handshake done sender: [2:7577789097642670555:2115] self: [2:7577789097642670554:0] peer: [3:7577789098765572892:0] socket: 289 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.917341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789097642670583:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:53.917422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.917527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789098765572886:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:53.917664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789098779016441:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29664) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789098779016541:2136] held: no
2025-11-28T14:51:53.917785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094287:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.917599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411494:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
2025-11-28T14:51:53.918041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789098700411491:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.919102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789098700411404:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.919072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094269:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.919106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789098645094269:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.919873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789098765572884:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.919971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789098700411494:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.920117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789098645094271:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.920146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789098645094271:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.920130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789098700411465:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.920789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789098779016440:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16898) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789098779016536:2134] held: no
2025-11-28T14:51:53.921450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789098765572808:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.921434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789098645094196:2049] [node 3] ICP18 handshake (actor [1:7577789098645094287:2120]) is too old
2025-11-28T14:51:53.920819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098182921728:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.921585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:53.921629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789098765572808:2049] [node 2] ICP22 created new session: [3:7577789098765572901:2049]
2025-11-28T14:51:53.920849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789098182921728:2124] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.922661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411497:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
2025-11-28T14:51:53.921421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789098779016541:2136] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:53.921680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789098765572901:2049] [node 2] ICS09 handshake done sender: [3:7577789098765572886:2118] self: [3:7577789098765572892:0] peer: [2:7577789097642670554:0] socket: 304 qp: -1
2025-11-28T14:51:53.922462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789098645094269:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:53.921715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789098765572901:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:53.921753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:53.923665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789098779016536:2134] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:53.923609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789098645094260:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.923663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.923699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789098645094260:2108] [node 5] ICP22 created new session: [1:7577789098645094295:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.924944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789098645094295:2108] [node 5] ICS09 handshake done sender: [1:7577789098645094269:2114] self: [1:7577789098645094268:0] peer: [5:7577789098182921740:0] socket: 314 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.924983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789098645094295:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:53.925030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.926856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789097642670559:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.926895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789097642670559:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.925098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411498:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
2025-11-28T14:51:53.925091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098182921743:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46097
2025-11-28T14:51:53.927544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789097642670559:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:53.925576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789098700411492:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.928438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098182921728:2124] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:53.925673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789098700411405:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.928522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789098765572807:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.928317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411500:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
2025-11-28T14:51:53.928585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:53.928665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789098765572807:2048] [node 1] ICP22 created new session: [3:7577789098765572903:2048]
2025-11-28T14:51:53.928680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789097642670551:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.928708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789098765572903:2048] [node 1] ICS09 handshake done sender: [3:7577789098765572884:2116] self: [3:7577789098765572894:0] peer: [1:7577789098645094264:0] socket: 301 qp: -1
2025-11-28T14:51:53.928816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789098779016441:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.928819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789098765572903:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:53.928829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.928969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789097642670551:2111] [node 6] ICP22 created new session: [2:7577789097642670587:2111]
2025-11-28T14:51:53.928868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:53.928975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:53.929094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789097642670587:2111] [node 6] ICS09 handshake done sender: [2:7577789097642670559:2117] self: [2:7577789097642670558:0] peer: [6:7577789098700411499:0] socket: 329 qp: -1
2025-11-28T14:51:53.929031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789098779016441:2050] [node 3] ICP22 created new session: [4:7577789098779016551:2050]
2025-11-28T14:51:53.929063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789098182921718:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.929239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789098645094271:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:53.929120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.929762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789097642670587:2111] [node 6] ICS10 traffic start
2025-11-28T14:51:53.929881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:53.929867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789098779016551:2050] [node 3] ICS09 handshake done sender: [4:7577789098779016541:2136] self: [4:7577789098779016544:0] peer: [3:7577789098765572878:0] socket: 292 qp: -1
2025-11-28T14:51:53.929923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789098779016551:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:53.929944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789098182921718:2110] [node 6] ICP22 created new session: [5:7577789098182921746:2110]
2025-11-28T14:51:53.930003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.930011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098182921746:2110] [node 6] ICS09 handshake done sender: [5:7577789098182921728:2124] self: [5:7577789098182921727:0] peer: [6:7577789098700411496:0] socket: 295 qp: -1
2025-11-28T14:51:53.930037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098182921746:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:53.930075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.930840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789098779016440:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.930925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:53.930923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098182921745:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46097
2025-11-28T14:51:53.930982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789098779016440:2049] [node 2] ICP22 created new session: [4:7577789098779016553:2049]
2025-11-28T14:51:53.931038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789098779016553:2049] [node 2] ICS09 handshake done sender: [4:7577789098779016536:2134] self: [4:7577789098779016543:0] peer: [2:7577789097642670556:0] socket: 297 qp: -1
2025-11-28T14:51:53.931075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789098779016553:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:53.931058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098182921733:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.931101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:53.930901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789098645094261:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.930990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.931277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789098700411492:2125] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:53.931041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789098645094261:2109] [node 6] ICP22 created new session: [1:7577789098645094297:2109]
2025-11-28T14:51:53.931571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789098700411494:2127] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:53.931080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789098645094297:2109] [node 6] ICS09 handshake done sender: [1:7577789098645094271:2115] self: [1:7577789098645094270:0] peer: [6:7577789098700411495:0] socket: 326 qp: -1
2025-11-28T14:51:53.931111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789098645094297:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:53.931143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:53.932039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789098700411405:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.932080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:53.932130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789098700411405:2049] [node 2] ICP22 created new session: [6:7577789098700411501:2049]
2025-11-28T14:51:53.932464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789098700411501:2049] [node 2] ICS09 handshake done sender: [6:7577789098700411492:2125] self: [6:7577789098700411499:0] peer: [2:7577789097642670558:0] socket: 330 qp: -1
2025-11-28T14:51:53.932488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789098700411501:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:53.932518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:53.935176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789098700411491:2124] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:53.935246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789098182921644:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.935300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:53.935768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789098182921644:2048] [node 1] ICP22 created new session: [5:7577789098182921748:2048]
2025-11-28T14:51:53.935942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098182921748:2048] [node 1] ICS09 handshake done sender: [5:7577789098182921733:2126] self: [5:7577789098182921740:0] peer: [1:7577789098645094268:0] socket: 315 qp: -1
2025-11-28T14:51:53.936005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098182921748:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:53.936044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789098700411465:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.936041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:53.936106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.936155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789098700411465:2107] [node 5] ICP22 created new session: [6:7577789098700411503:2107]
2025-11-28T14:51:53.936195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789098700411503:2107] [node 5] ICS09 handshake done sender: [6:7577789098700411494:2127] self: [6:7577789098700411496:0] peer: [5:7577789098182921727:0] socket: 339 qp: -1
2025-11-28T14:51:53.936174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789098765572808:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:53.936231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789098700411503:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:53.936267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.936280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789098182921644:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:53.936492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789098765572807:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:53.937076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098182921745:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.937502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789098182921645:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.938015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789097642670561:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.938052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789097642670561:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.938540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789098182921741:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.938639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789098182921646:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.938817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789097642670561:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.938945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789098765572881:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.939157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098182921752:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46097
2025-11-28T14:51:53.939202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789097642670550:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.938975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789098765572881:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.939259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.939310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789097642670550:2109] [node 5] ICP22 created new session: [2:7577789097642670590:2109]
2025-11-28T14:51:53.939358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789097642670590:2109] [node 5] ICS09 handshake done sender: [2:7577789097642670561:2118] self: [2:7577789097642670560:0] peer: [5:7577789098182921750:0] socket: 293 qp: -1
2025-11-28T14:51:53.939401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789097642670590:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:53.940127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.940404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098182921745:2131] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:53.940402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789098765572881:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:53.940676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789098700411404:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.940805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789098765572872:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.940722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:53.940846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789098700411404:2048] [node 1] ICP22 created new session: [6:7577789098700411505:2048]
2025-11-28T14:51:53.940858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:53.940888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789098700411505:2048] [node 1] ICS09 handshake done sender: [6:7577789098700411491:2124] self: [6:7577789098700411495:0] peer: [1:7577789098645094270:0] socket: 328 qp: -1
2025-11-28T14:51:53.940899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789098765572872:2108] [node 5] ICP22 created new session: [3:7577789098765572907:2108]
2025-11-28T14:51:53.940913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789098700411505:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:53.940911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789098182921645:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.940942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:53.940938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789098765572907:2108] [node 5] ICS09 handshake done sender: [3:7577789098765572881:2114] self: [3:7577789098765572880:0] peer: [5:7577789098182921751:0] socket: 337 qp: -1
2025-11-28T14:51:53.940954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:53.940968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789098765572907:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:53.940996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:53.940998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789098182921645:2049] [node 2] ICP22 created new session: [5:7577789098182921754:2049]
2025-11-28T14:51:53.941053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098182921754:2049] [node 2] ICS09 handshake done sender: [5:7577789098182921745:2131] self: [5:7577789098182921750:0] peer: [2:7577789097642670560:0] socket: 342 qp: -1
2025-11-28T14:51:53.941083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098182921754:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:53.941111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:53.941170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789098700411404:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:53.941678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789098700411465:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:53.942045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789098700411493:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.942599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789098182921753:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46097
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.942714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411507:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.942847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789098700411481:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.942859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789098765572909:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57459
2025-11-28T14:51:53.943122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789098700411507:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.943097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789098779016532:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.943129Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789098779016532:2133] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.944658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789098779016555:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25860
2025-11-28T14:51:53.944701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789098182921646:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29664) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789098182921741:2129] held: no
2025-11-28T14:51:53.944785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789098182921741:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:53.945041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789098779016532:2133] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:53.945273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789098182921646:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.945321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:53.945334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789098779016523:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.945360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789098182921646:2050] [node 3] ICP22 created new session: [5:7577789098182921756:2050]
2025-11-28T14:51:53.945395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789098182921756:2050] [node 3] ICS09 handshake done sender: [5:7577789098182921741:2129] self: [5:7577789098182921751:0] peer: [3:7577789098765572880:0] socket: 338 qp: -1
2025-11-28T14:51:53.945396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.945417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789098182921756:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:53.945451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.945446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789098779016523:2116] [node 6] ICP22 created new session: [4:7577789098779016557:2116]
2025-11-28T14:51:53.945500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789098779016557:2116] [node 6] ICS09 handshake done sender: [4:7577789098779016532:2133] self: [4:7577789098779016531:0] peer: [6:7577789098700411510:0] socket: 305 qp: -1
2025-11-28T14:51:53.945536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789098779016557:2116] [node 6] ICS10 traffic start
2025-11-28T14:51:53.945569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:53.946519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789098700411406:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:53.946578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789098765572909:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.946864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411511:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
2025-11-28T14:51:53.946839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789098779016555:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.946902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789098765572883:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:53.946961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789098779016523:2116] [node 6] ICP18 handshake (actor [4:7577789098779016555:2139]) is too old
2025-11-28T14:51:53.946925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789098765572883:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:53.947485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789098700411493:2126] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:53.947627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789098765572883:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:53.947895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789098765572873:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:53.947952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:53.947990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789098765572873:2109] [node 6] ICP22 created new session: [3:7577789098765572912:2109]
2025-11-28T14:51:53.948035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789098765572912:2109] [node 6] ICS09 handshake done sender: [3:7577789098765572883:2115] self: [3:7577789098765572882:0] peer: [6:7577789098700411512:0] socket: 280 qp: -1
2025-11-28T14:51:53.948062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789098765572912:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:53.948092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:53.948544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789098700411481:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.948592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:53.948649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789098700411481:2108] [node 4] ICP22 created new session: [6:7577789098700411514:2108]
2025-11-28T14:51:53.948687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789098700411514:2108] [node 4] ICS09 handshake done sender: [6:7577789098700411493:2126] self: [6:7577789098700411510:0] peer: [4:7577789098779016531:0] socket: 336 qp: -1
2025-11-28T14:51:53.948717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789098700411514:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:53.948784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:53.949000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789098700411481:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:53.950388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789098700411406:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29664) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789098700411507:2131] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:53.950589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789098700411513:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14357
2025-11-28T14:51:53.950995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789098700411507:2131] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:53.951163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789098700411406:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:53.951198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:53.951235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789098700411406:2050] [node 3] ICP22 created new session: [6:7577789098700411516:2050]
2025-11-28T14:51:53.951278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789098700411516:2050] [node 3] ICS09 handshake done sender: [6:7577789098700411507:2131] self: [6:7577789098700411512:0] peer: [3:7577789098765572882:0] socket: 283 qp: -1
2025-11-28T14:51:53.951314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789098700411516:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:53.951344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:53.971014Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:53.971256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789097642670573:2048] [node 1] ICS01 socket: 282 reason# EndOfStream
2025-11-28T14:51:53.971305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789097642670482:2048] [node 1] ICP30 unregister session Session# [2:7577789097642670573:2048] VirtualId# [2:7577789097642670568:0]
2025-11-28T14:51:53.971340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789097642670573:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.971429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.971480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.972145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:53.972389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789098765572903:2048] [node 1] ICS01 socket: 301 reason# EndOfStream
2025-11-28T14:51:53.972423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789098765572807:2048] [node 1] ICP30 unregister session Session# [3:7577789098765572903:2048] VirtualId# [3:7577789098765572894:0]
2025-11-28T14:51:53.972464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789098765572903:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.972540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.972591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.975663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:53.975723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:53.975872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789098182921748:2048] [node 1] ICS07 socket disconnect 315 reason# EndOfStream
2025-11-28T14:51:53.975907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789098182921748:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.975926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789098700411505:2048] [node 1] ICS07 socket disconnect 328 reason# EndOfStream
2025-11-28T14:51:53.975976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.975956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789098700411505:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.976014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.976069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789098700411505:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:53.976037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789098182921748:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:53.977037Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789098182921644:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15668) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.977081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789098182921748:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.977099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789098700411404:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15668) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.977100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789098182921644:2048] [node 1] ICP30 unregister session Session# [5:7577789098182921748:2048] VirtualId# [5:7577789098182921740:0]
2025-11-28T14:51:53.977149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789098700411505:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.977140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.977166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789098182921644:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.977191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789098700411404:2048] [node 1] ICP30 unregister session Session# [6:7577789098700411505:2048] VirtualId# [6:7577789098700411495:0]
2025-11-28T14:51:53.977189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.977236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.976320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:53.977292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789098700411404:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.976536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789098779016539:2048] [node 1] ICS07 socket disconnect 287 reason# EndOfStream
2025-11-28T14:51:53.977313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.976560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789098779016539:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:53.976648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:53.976695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789098779016539:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:53.977661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789098779016439:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15668) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:53.977740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789098779016539:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:53.977804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789098779016439:2048] [node 1] ICP30 unregister session Session# [4:7577789098779016539:2048] VirtualId# [4:7577789098779016537:0]
2025-11-28T14:51:53.977838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:53.977860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789098779016439:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.977881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:53.987657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789098700411404:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:53.988440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789098182921644:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:53.991185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789098779016439:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:54.000063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:54.000226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789098765572901:2049] [node 2] ICS01 socket: 304 reason# EndOfStream
2025-11-28T14:51:54.000272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789098765572808:2049] [node 2] ICP30 unregister session Session# [3:7577789098765572901:2049] VirtualId# [3:7577789098765572892:0]
2025-11-28T14:51:54.000299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789098765572901:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.000392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:54.000444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:54.002929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:54.003103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789098779016553:2049] [node 2] ICS01 socket: 297 reason# EndOfStream
2025-11-28T14:51:54.003139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789098779016440:2049] [node 2] ICP30 unregister session Session# [4:7577789098779016553:2049] VirtualId# [4:7577789098779016543:0]
2025-11-28T14:51:54.003163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789098779016553:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.003227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:54.003255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:54.003954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:54.004127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789098182921754:2049] [node 2] ICS01 socket: 342 reason# EndOfStream
2025-11-28T14:51:54.004164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789098182921645:2049] [node 2] ICP30 unregister session Session# [5:7577789098182921754:2049] VirtualId# [5:7577789098182921750:0]
2025-11-28T14:51:54.004191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789098182921754:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.004270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:54.004318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:54.005198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:54.005411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789098700411501:2049] [node 2] ICS01 socket: 330 reason# EndOfStream
2025-11-28T14:51:54.005446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789098700411405:2049] [node 2] ICP30 unregister session Session# [6:7577789098700411501:2049] VirtualId# [6:7577789098700411499:0]
2025-11-28T14:51:54.005481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789098700411501:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.005549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:54.005588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:54.030342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:54.030537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789098182921756:2050] [node 3] ICS01 socket: 338 reason# EndOfStream
2025-11-28T14:51:54.030575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789098182921646:2050] [node 3] ICP30 unregister session Session# [5:7577789098182921756:2050] VirtualId# [5:7577789098182921751:0]
2025-11-28T14:51:54.030605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789098182921756:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.030695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:54.030747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:54.031294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:54.031538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789098700411516:2050] [node 3] ICS01 socket: 283 reason# EndOfStream
2025-11-28T14:51:54.031583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789098700411406:2050] [node 3] ICP30 unregister session Session# [6:7577789098700411516:2050] VirtualId# [6:7577789098700411512:0]
2025-11-28T14:51:54.031626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789098700411516:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.031738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:54.031794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:54.032517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:54.032708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789098779016551:2050] [node 3] ICS01 socket: 292 reason# EndOfStream
2025-11-28T14:51:54.032743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789098779016441:2050] [node 3] ICP30 unregister session Session# [4:7577789098779016551:2050] VirtualId# [4:7577789098779016544:0]
2025-11-28T14:51:54.032769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789098779016551:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.032847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:54.032882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:54.064735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:54.064968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789098182921738:2108] [node 4] ICS07 socket disconnect 322 reason# EndOfStream
2025-11-28T14:51:54.065011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789098182921738:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.065116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:54.065170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789098182921738:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:54.070998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:54.071289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789098700411514:2108] [node 4] ICS07 socket disconnect 336 reason# EndOfStream
2025-11-28T14:51:54.071320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789098700411514:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.071458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:54.071511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789098700411514:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:54.072957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789098700411481:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1125) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:54.073018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789098700411514:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:54.073060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789098700411481:2108] [node 4] ICP30 unregister session Session# [6:7577789098700411514:2108] VirtualId# [6:7577789098700411510:0]
2025-11-28T14:51:54.073108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:54.073136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789098700411481:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:54.073149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:54.084420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789098700411481:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:54.099799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:54.100048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789098700411503:2107] [node 5] ICS01 socket: 339 reason# EndOfStream
2025-11-28T14:51:54.100163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789098700411465:2107] [node 5] ICP30 unregister session Session# [6:7577789098700411503:2107] VirtualId# [6:7577789098700411496:0]
2025-11-28T14:51:54.100207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789098700411503:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.100320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:54.100384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:54.170962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789104640577305:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:54.170966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789104640577304:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:54.191734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789101819520541:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:54.191813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789101819520542:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:54.217968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789104822171355:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:54.218036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789104822171356:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:54.238729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789101701358930:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:54.238795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789101701358932:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:54.239443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789101701358931:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:54.258635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789103071140228:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:54.258699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789103071140229:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:54.258741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789103071140230:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:54.288654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789102784624016:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:54.288708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789102784624017:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:54.288896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789102784624018:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:54.381084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789102784624078:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:54.435368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789101701358994:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:54.433766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789104640577305:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51026464ns) {Configure:104(51022013ns) {ProcessConfigured:135(83264ns) {StartInitialHandshake:157(79478ns) {DropHandshakes:470(4177ns) {DropIncomingHandshake:442(1303ns) DropOutgoingHandshake:457(811ns)} GenerateSessionVirtualId:988(1472ns) PrepareNewSessionHandshake:477(2452ns) {SwitchToState:179(664ns)}}}}}
2025-11-28T14:51:54.434504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789101819520622:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:54.435057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789104640577389:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:54.435762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789104822171418:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:54.441288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789101819520623:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:54.435681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789101701358993:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:54.452722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789104822171419:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:54.481484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789101701358930:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(45772949ns) {Configure:104(45766467ns) {ProcessConfigured:135(64769ns) {StartInitialHandshake:157(62334ns) {DropHandshakes:470(2549ns) {DropIncomingHandshake:442(584ns) DropOutgoingHandshake:457(423ns)} GenerateSessionVirtualId:988(872ns) PrepareNewSessionHandshake:477(1429ns) {SwitchToState:179(494ns)}}}}}
2025-11-28T14:51:54.449358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789104640577304:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(68508807ns) {Configure:104(68498043ns) {ProcessConfigured:135(12525194ns) {StartInitialHandshake:157(12522103ns) {DropHandshakes:470(1632ns) {DropIncomingHandshake:442(458ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(1086ns) PrepareNewSessionHandshake:477(4309ns) {SwitchToState:179(2371ns)}}}}}
2025-11-28T14:51:54.458880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789101819520624:2117] [node 6] ICP01 ready to work
2025-11-28T14:51:54.465119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789102784624078:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(54424976ns) {Configure:104(54415143ns) {ProcessConfigured:135(11794510ns) {StartInitialHandshake:157(11790977ns) {DropHandshakes:470(2625ns) {DropIncomingHandshake:442(626ns) DropOutgoingHandshake:457(762ns)} GenerateSessionVirtualId:988(1548ns) PrepareNewSessionHandshake:477(2776ns) {SwitchToState:179(1554ns)}}}}}
2025-11-28T14:51:54.449896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789104640577392:2116] [node 5] ICP01 ready to work
2025-11-28T14:51:54.460772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789104640577399:2129] [node 6] ICP01 ready to work
2025-11-28T14:51:54.479257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789102784624016:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31466715ns) {Configure:104(31459500ns) {ProcessConfigured:135(18021933ns) {StartInitialHandshake:157(12124956ns) {DropHandshakes:470(3702ns) {DropIncomingHandshake:442(538ns) DropOutgoingHandshake:457(398ns)} GenerateSessionVirtualId:988(2569ns) PrepareNewSessionHandshake:477(4206ns) {SwitchToState:179(1714ns)}}}}}
2025-11-28T14:51:54.485552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789104822171420:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:54.486347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789102784624084:2110] [node 4] ICP01 ready to work
2025-11-28T14:51:54.519787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789103071140228:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(106490549ns) {Configure:104(106483447ns) {ProcessConfigured:135(8110067ns) {StartInitialHandshake:157(8106084ns) {DropHandshakes:470(6290ns) {DropIncomingHandshake:442(906ns) DropOutgoingHandshake:457(817ns)} GenerateSessionVirtualId:988(1488ns) PrepareNewSessionHandshake:477(3993ns) {SwitchToState:179(1465ns)}}}}}
2025-11-28T14:51:54.521191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789103071140293:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:54.522557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789103071140294:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:54.522566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789101701358932:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(80746223ns) {Configure:104(80731529ns) {ProcessConfigured:135(699683ns) {StartInitialHandshake:157(695672ns) {DropHandshakes:470(3803ns) {DropIncomingHandshake:442(1212ns) DropOutgoingHandshake:457(705ns)} GenerateSessionVirtualId:988(1099ns) PrepareNewSessionHandshake:477(2734ns) {SwitchToState:179(827ns)}}}}}
2025-11-28T14:51:54.523975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789102784624084:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32648888ns) {Configure:104(32645578ns) {ProcessConfigured:135(693269ns) {StartInitialHandshake:157(691623ns) {DropHandshakes:470(1281ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(1174ns) {SwitchToState:179(459ns)}}}}}
2025-11-28T14:51:54.524070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789101701358993:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(37461260ns) {Configure:104(37454809ns) {ProcessConfigured:135(597795ns) {StartInitialHandshake:157(593667ns) {DropHandshakes:470(3529ns) {DropIncomingHandshake:442(985ns) DropOutgoingHandshake:457(532ns)} GenerateSessionVirtualId:988(1145ns) PrepareNewSessionHandshake:477(2204ns) {SwitchToState:179(917ns)}}}}}
2025-11-28T14:51:54.524008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789104640577399:2129] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(46771416ns) {Configure:104(46761508ns) {ProcessConfigured:135(1714319ns) {StartInitialHandshake:157(1702490ns) {DropHandshakes:470(1400ns) {DropIncomingHandshake:442(343ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(309ns) PrepareNewSessionHandshake:477(1739ns) {SwitchToState:179(973ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.527681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789104822171356:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33917997ns) {Configure:104(33909455ns) {ProcessConfigured:135(1010593ns) {StartInitialHandshake:157(1008578ns) {DropHandshakes:470(1749ns) {DropIncomingHandshake:442(259ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(493ns) PrepareNewSessionHandshake:477(2202ns) {SwitchToState:179(811ns)}}}}}
2025-11-28T14:51:54.527899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789104640577403:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47141
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.528055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789101819520622:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35471914ns) {Configure:104(35461743ns) {ProcessConfigured:135(450271ns) {StartInitialHandshake:157(448042ns) {DropHandshakes:470(269727ns) {DropIncomingHandshake:442(481ns) DropOutgoingHandshake:457(956ns)} GenerateSessionVirtualId:988(1145ns) PrepareNewSessionHandshake:477(1804ns) {SwitchToState:179(1241ns)}}}}}
2025-11-28T14:51:54.528925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789101819520623:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16208667ns) {Configure:104(16198383ns) {ProcessConfigured:135(361717ns) {StartInitialHandshake:157(358495ns) {DropHandshakes:470(1347ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(554ns) PrepareNewSessionHandshake:477(6511ns) {SwitchToState:179(4972ns)}}}}}
2025-11-28T14:51:54.529010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789101701359005:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47719
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.532437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624094:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.530546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789101701359005:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.532683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789104640577403:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.533125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789101701358930:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.534216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789104640577404:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47141
2025-11-28T14:51:54.535096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789104640577394:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.535130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789104640577394:2132] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:54.538397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789104640577394:2132] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.539372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789103071140299:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7973
2025-11-28T14:51:54.541689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789104640577389:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.541746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:54.541759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789102784624094:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.542337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789103071140299:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.542482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789102784624084:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.541791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789104640577389:2107] [node 4] ICP22 created new session: [1:7577789104640577408:2107]
2025-11-28T14:51:54.542492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789101701359010:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47719
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.541812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789104640577404:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.542283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789104640577408:2107] [node 4] ICS09 handshake done sender: [1:7577789104640577394:2132] self: [1:7577789104640577393:0] peer: [4:7577789101701359009:0] socket: 284 qp: -1
2025-11-28T14:51:54.542455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789104640577408:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:54.543007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789104822171425:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-28T14:51:54.543090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789101701358998:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.542699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:54.543130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789101701358998:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.544851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789104640577407:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47141
2025-11-28T14:51:54.544868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789101701359005:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:54.545346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789101701359011:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47719
2025-11-28T14:51:54.545166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789104640577389:2107] [node 4] ICP18 handshake (actor [1:7577789104640577404:2136]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.546758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789101819520633:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26923
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.546179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789101701358930:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.546903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789101819520634:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26923
No RDMA context found for GID: 2025-11-28T14:51:54.547098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789104822171425:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:51:54.547012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:54.548036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789101819520635:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26923
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.550752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789101819520634:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.547086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789101701358930:2048] [node 1] ICP22 created new session: [4:7577789101701359014:2048]
2025-11-28T14:51:54.549651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789104822171355:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.550611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789103071140308:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7973
2025-11-28T14:51:54.552192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624096:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.548203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789101701359014:2048] [node 1] ICS09 handshake done sender: [4:7577789101701359005:2115] self: [4:7577789101701359009:0] peer: [1:7577789104640577393:0] socket: 288 qp: -1
2025-11-28T14:51:54.552975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789104640577388:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.553032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789104640577388:2130] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.550612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789101701359014:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.553759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789101819520633:2132] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.552167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:54.554521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789103071140309:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7973
2025-11-28T14:51:54.555742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789101701359012:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47719
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.554169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789101819520639:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26923
2025-11-28T14:51:54.555565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789101819520638:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26923
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.556096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789104640577407:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.556116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624097:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.556214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624098:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.556227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789101819520635:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.556398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789101701358998:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:54.556312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789101819520639:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.563871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789103071140308:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.563893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789101819520541:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.579003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789103071140228:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.578981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789104640577388:2130] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:54.569375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789101819520638:2136] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.580919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789101701359012:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.581067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789102784624094:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:54.581034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789104640577391:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.581231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789101701359011:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.581075Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789104640577391:2131] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.583366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789104640577305:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.581473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789101701358994:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.582906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789102784624084:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.583406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:54.581711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:54.581754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789101701358994:2109] [node 6] ICP22 created new session: [4:7577789101701359018:2109]
2025-11-28T14:51:54.583600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789104640577305:2049] [node 3] ICP22 created new session: [1:7577789104640577413:2049]
2025-11-28T14:51:54.583237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789101701359018:2109] [node 6] ICS09 handshake done sender: [4:7577789101701358998:2111] self: [4:7577789101701358997:0] peer: [6:7577789102784624095:0] socket: 296 qp: -1
2025-11-28T14:51:54.583590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:54.583824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789104640577413:2049] [node 3] ICS09 handshake done sender: [1:7577789104640577388:2130] self: [1:7577789104640577387:0] peer: [3:7577789104822171435:0] socket: 282 qp: -1
2025-11-28T14:51:54.583920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789104822171436:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-28T14:51:54.583387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789101701359018:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:54.583755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789102784624084:2110] [node 4] ICP22 created new session: [6:7577789102784624099:2110]
2025-11-28T14:51:54.583435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:54.583808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789102784624099:2110] [node 4] ICS09 handshake done sender: [6:7577789102784624094:2115] self: [6:7577789102784624095:0] peer: [4:7577789101701358997:0] socket: 299 qp: -1
2025-11-28T14:51:54.583871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789104640577413:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:54.584031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789102784624099:2110] [node 4] ICS10 traffic start
2025-11-28T14:51:54.583946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789104640577396:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.583989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789104640577396:2133] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:54.584064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:54.584001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:54.587334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789104640577391:2131] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:54.587748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789104640577396:2133] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.587851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789104822171425:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.588142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789101819520648:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26923
2025-11-28T14:51:54.588616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789101701358932:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.589162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789104640577304:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.589214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:54.588947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789103071140311:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7973
2025-11-28T14:51:54.589274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789104640577304:2048] [node 2] ICP22 created new session: [1:7577789104640577418:2048]
2025-11-28T14:51:54.589294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789101701359016:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47719
2025-11-28T14:51:54.589310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789104822171355:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.589329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789104640577418:2048] [node 2] ICS09 handshake done sender: [1:7577789104640577391:2131] self: [1:7577789104640577390:0] peer: [2:7577789101819520644:0] socket: 283 qp: -1
2025-11-28T14:51:54.589359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789104640577418:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:54.589366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:54.589425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789104640577392:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.589416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789104822171355:2048] [node 1] ICP22 created new session: [3:7577789104822171438:2048]
2025-11-28T14:51:54.589362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789103071140308:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:54.589439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:54.589556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:54.589853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789104822171438:2048] [node 1] ICS09 handshake done sender: [3:7577789104822171425:2112] self: [3:7577789104822171435:0] peer: [1:7577789104640577387:0] socket: 292 qp: -1
2025-11-28T14:51:54.589752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789104640577392:2116] [node 5] ICP22 created new session: [1:7577789104640577420:2116]
2025-11-28T14:51:54.589933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789104822171438:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:54.589829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789104640577420:2116] [node 5] ICS09 handshake done sender: [1:7577789104640577396:2133] self: [1:7577789104640577395:0] peer: [5:7577789103071140310:0] socket: 311 qp: -1
2025-11-28T14:51:54.589979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:54.589811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789103071140228:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.589982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789104640577420:2116] [node 5] ICS10 traffic start
2025-11-28T14:51:54.590007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:54.589996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:54.590170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789103071140228:2048] [node 1] ICP22 created new session: [5:7577789103071140312:2048]
2025-11-28T14:51:54.590181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624102:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.590208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789103071140312:2048] [node 1] ICS09 handshake done sender: [5:7577789103071140308:2116] self: [5:7577789103071140310:0] peer: [1:7577789104640577395:0] socket: 312 qp: -1
2025-11-28T14:51:54.590230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789103071140312:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:54.590254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.592101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789103071140314:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7973
2025-11-28T14:51:54.592480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789103071140314:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.592588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789103071140293:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.593602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789104822171424:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.593643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789104822171424:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:54.592836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789101701359004:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.592861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789101701359004:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:54.594629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789101701359004:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:54.594817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789101701358993:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.594868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.594914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789101701358993:2108] [node 5] ICP22 created new session: [4:7577789101701359024:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.594937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789102784624102:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.595027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789101701359024:2108] [node 5] ICS09 handshake done sender: [4:7577789101701359004:2114] self: [4:7577789101701359003:0] peer: [5:7577789103071140316:0] socket: 339 qp: -1
2025-11-28T14:51:54.595029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789104822171437:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-28T14:51:54.595065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789101701359024:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:54.595100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:54.595046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789103071140317:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7973
2025-11-28T14:51:54.595128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789102784624016:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.595285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789103071140314:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:54.595356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789104640577402:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.595385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789104640577402:2134] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:54.596012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789104822171440:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-28T14:51:54.595973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789103071140293:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.596040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.596099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789103071140293:2108] [node 4] ICP22 created new session: [5:7577789103071140318:2108]
2025-11-28T14:51:54.596174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789103071140318:2108] [node 4] ICS09 handshake done sender: [5:7577789103071140314:2119] self: [5:7577789103071140316:0] peer: [4:7577789101701359003:0] socket: 305 qp: -1
No RDMA context found for GID: 2025-11-28T14:51:54.596181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789104822171441:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
::ffff:127.1.0.0
2025-11-28T14:51:54.596210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789103071140318:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.596192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789104640577402:2134] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:54.596236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:54.596309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789101701359026:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47719
2025-11-28T14:51:54.596320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624104:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.596467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789104640577399:2129] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.596431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789104822171437:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.596505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:54.596860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789104640577399:2129] [node 6] ICP22 created new session: [1:7577789104640577423:2129]
2025-11-28T14:51:54.597032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789104822171440:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.597017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789104640577423:2129] [node 6] ICS09 handshake done sender: [1:7577789104640577402:2134] self: [1:7577789104640577401:0] peer: [6:7577789102784624103:0] socket: 334 qp: -1
2025-11-28T14:51:54.597051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789104640577423:2129] [node 6] ICS10 traffic start
2025-11-28T14:51:54.597141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789102784624102:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:54.597092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:54.598386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789104822171424:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:54.598909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789102784624016:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.598937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789103071140293:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:54.598962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:54.599017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789102784624016:2048] [node 1] ICP22 created new session: [6:7577789102784624105:2048]
2025-11-28T14:51:54.599220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789102784624105:2048] [node 1] ICS09 handshake done sender: [6:7577789102784624102:2119] self: [6:7577789102784624103:0] peer: [1:7577789104640577401:0] socket: 338 qp: -1
2025-11-28T14:51:54.599200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789101701359012:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:54.599684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789102784624105:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:54.599946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789101701358932:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.599775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:54.599988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:54.600277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789104822171418:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.600323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:54.600200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789101701358932:2050] [node 3] ICP22 created new session: [4:7577789101701359027:2050]
2025-11-28T14:51:54.601378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789101701359027:2050] [node 3] ICS09 handshake done sender: [4:7577789101701359012:2118] self: [4:7577789101701359020:0] peer: [3:7577789104822171423:0] socket: 316 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.601603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789104822171418:2107] [node 4] ICP22 created new session: [3:7577789104822171445:2107]
2025-11-28T14:51:54.602008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789101701359027:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:54.601557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789102784624016:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:54.602040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:54.601676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789104822171445:2107] [node 4] ICS09 handshake done sender: [3:7577789104822171424:2111] self: [3:7577789104822171423:0] peer: [4:7577789101701359020:0] socket: 314 qp: -1
2025-11-28T14:51:54.601710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789104822171445:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:54.602211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789103071140320:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7973
2025-11-28T14:51:54.601776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.602981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624107:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.603316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789104822171441:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.603538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789104822171418:2107] [node 4] ICP18 handshake (actor [3:7577789104822171440:2118]) is too old
2025-11-28T14:51:54.608819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789103071140320:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.608959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789103071140230:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.609354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789104822171429:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.609390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789104822171429:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:54.609954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789102784624107:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.610161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789102784624078:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.610316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789104822171429:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.613103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789102784624096:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.612332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789103071140304:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.612326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789104822171428:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:54.612391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789103071140304:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:54.626037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789102784624018:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.626079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789104822171419:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.613191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789103071140323:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7973
2025-11-28T14:51:54.644566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789103071140320:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:54.628760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:54.644493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789104822171419:2108] [node 5] ICP22 created new session: [3:7577789104822171449:2108]
2025-11-28T14:51:54.644805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789104822171428:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.645315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789104822171449:2108] [node 5] ICS09 handshake done sender: [3:7577789104822171429:2114] self: [3:7577789104822171426:0] peer: [5:7577789103071140322:0] socket: 347 qp: -1
2025-11-28T14:51:54.646068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624110:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.645373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789104822171449:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:54.645431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:54.647631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789104822171428:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:54.648555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789103071140230:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.648665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:54.649636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789103071140230:2050] [node 3] ICP22 created new session: [5:7577789103071140325:2050]
2025-11-28T14:51:54.650198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624111:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.649995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789104822171420:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.650720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789102784624096:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:54.650081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:54.650706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789103071140325:2050] [node 3] ICS09 handshake done sender: [5:7577789103071140320:2121] self: [5:7577789103071140322:0] peer: [3:7577789104822171426:0] socket: 286 qp: -1
2025-11-28T14:51:54.651118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789102784624018:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.650137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789104822171420:2109] [node 6] ICP22 created new session: [3:7577789104822171452:2109]
2025-11-28T14:51:54.650806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789103071140325:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:54.651186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:54.651230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789102784624018:2050] [node 3] ICP22 created new session: [6:7577789102784624112:2050]
2025-11-28T14:51:54.651279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789102784624112:2050] [node 3] ICS09 handshake done sender: [6:7577789102784624096:2116] self: [6:7577789102784624109:0] peer: [3:7577789104822171427:0] socket: 319 qp: -1
2025-11-28T14:51:54.650193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789104822171452:2109] [node 6] ICS09 handshake done sender: [3:7577789104822171428:2113] self: [3:7577789104822171427:0] peer: [6:7577789102784624109:0] socket: 317 qp: -1
2025-11-28T14:51:54.651312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789102784624112:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:54.650233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789104822171452:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:54.650954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:54.650272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:54.651345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:54.653612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789103071140304:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:54.656208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789102784624107:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:54.653772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789104822171419:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24314274ns) {HandleHandshakeStatus:329(24308407ns) {DropIncomingHandshake:442(27346ns) DropOutgoingHandshake:457(518ns) SwitchToState:179(444ns) ProcessPendingSessionEvents:495(2469402ns) {ForwardSessionEventToSession:612(1953157ns) ForwardSessionEventToSession:612(447230ns) ForwardSessionEventToSession:612(4976ns) ForwardSessionEventToSession:612(4723ns) ForwardSessionEventToSession:612(5942ns) ForwardSessionEventToSession:612(3293ns) ForwardSessionEventToSession:612(7155ns) ForwardSessionEventToSession:612(6691ns) ForwardSessionEventToSession:612(4165ns) ForwardSessionEventToSession:612(15705ns)}}}
2025-11-28T14:51:54.654490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789103071140230:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:54.657822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789103071140294:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.658043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:54.658104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789103071140294:2109] [node 6] ICP22 created new session: [5:7577789103071140327:2109]
2025-11-28T14:51:54.658772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789103071140327:2109] [node 6] ICS09 handshake done sender: [5:7577789103071140304:2115] self: [5:7577789103071140303:0] peer: [6:7577789102784624108:0] socket: 290 qp: -1
2025-11-28T14:51:54.658804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789103071140327:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:54.658835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:54.658277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789102784624078:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.658320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:54.658381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789102784624078:2107] [node 5] ICP22 created new session: [6:7577789102784624114:2107]
2025-11-28T14:51:54.658433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789102784624114:2107] [node 5] ICS09 handshake done sender: [6:7577789102784624107:2121] self: [6:7577789102784624108:0] peer: [5:7577789103071140303:0] socket: 308 qp: -1
2025-11-28T14:51:54.658469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789102784624114:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:54.658513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:54.691488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789101819520628:2129] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:54.696583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789103071140309:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.696912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789103071140229:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.718322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789101819520626:2128] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:54.719073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789101819520633:2132] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:54.721784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789101701359016:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.721987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789101701358931:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.737128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789101701359030:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47719
2025-11-28T14:51:54.732795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789101819520628:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.732977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789101819520626:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.733052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789101819520541:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.733124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:54.733183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789101819520541:2048] [node 1] ICP22 created new session: [2:7577789101819520649:2048]
2025-11-28T14:51:54.733234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789101819520649:2048] [node 1] ICS09 handshake done sender: [2:7577789101819520633:2132] self: [2:7577789101819520644:0] peer: [1:7577789104640577390:0] socket: 300 qp: -1
2025-11-28T14:51:54.733271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789101819520649:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:54.733614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:54.736666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789101819520628:2129] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:54.740207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789101819520626:2128] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:54.740761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789101819520623:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.740836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:54.740887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789101819520623:2108] [node 5] ICP22 created new session: [2:7577789101819520654:2108]
2025-11-28T14:51:54.740933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789101819520654:2108] [node 5] ICS09 handshake done sender: [2:7577789101819520628:2129] self: [2:7577789101819520627:0] peer: [5:7577789103071140329:0] socket: 322 qp: -1
2025-11-28T14:51:54.740968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789101819520654:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:54.741000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:54.741403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789101819520622:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.741447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:54.741488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789101819520622:2107] [node 4] ICP22 created new session: [2:7577789101819520656:2107]
2025-11-28T14:51:54.741528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789101819520656:2107] [node 4] ICS09 handshake done sender: [2:7577789101819520626:2128] self: [2:7577789101819520625:0] peer: [4:7577789101701359029:0] socket: 294 qp: -1
2025-11-28T14:51:54.741558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789101819520656:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:54.741589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:54.742689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789103071140330:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7973
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.744196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789104822171454:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-28T14:51:54.744868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789101701359016:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:54.747404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789103071140309:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:54.745120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789101701358931:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.745177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:54.745229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789101701358931:2049] [node 2] ICP22 created new session: [4:7577789101701359031:2049]
2025-11-28T14:51:54.745275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789101701359031:2049] [node 2] ICS09 handshake done sender: [4:7577789101701359016:2119] self: [4:7577789101701359029:0] peer: [2:7577789101819520625:0] socket: 326 qp: -1
2025-11-28T14:51:54.745314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789101701359031:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:54.745345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:54.752086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789103071140229:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11113) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789103071140309:2117] held: no
2025-11-28T14:51:54.752196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789103071140229:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.752243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:54.752297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789103071140229:2049] [node 2] ICP22 created new session: [5:7577789103071140331:2049]
2025-11-28T14:51:54.752354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789103071140331:2049] [node 2] ICS09 handshake done sender: [5:7577789103071140309:2117] self: [5:7577789103071140329:0] peer: [2:7577789101819520627:0] socket: 323 qp: -1
2025-11-28T14:51:54.752479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789103071140331:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:54.752513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:54.804523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789103071140229:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(40370349ns) {HandleHandshakeStatus:329(40368189ns) {DropIncomingHandshake:442(853ns) DropOutgoingHandshake:457(124ns) SwitchToState:179(790ns) ProcessPendingSessionEvents:495(2130ns)}}
2025-11-28T14:51:54.805366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789101819520631:2130] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:54.808654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789102784624098:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.808865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789102784624017:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.810148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789101819520631:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.811280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789101819520631:2130] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.820819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789101819520624:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.820923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:54.820990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789101819520624:2117] [node 6] ICP22 created new session: [2:7577789101819520659:2117]
2025-11-28T14:51:54.821038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789101819520659:2117] [node 6] ICS09 handshake done sender: [2:7577789101819520631:2130] self: [2:7577789101819520629:0] peer: [6:7577789102784624116:0] socket: 324 qp: -1
2025-11-28T14:51:54.821087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789101819520659:2117] [node 6] ICS10 traffic start
2025-11-28T14:51:54.821138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:54.829524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789102784624117:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-28T14:51:54.858659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789102784624017:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11113) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789102784624098:2118] held: no
2025-11-28T14:51:54.860895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789102784624098:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:54.861223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789102784624017:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.861277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:54.861324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789102784624017:2049] [node 2] ICP22 created new session: [6:7577789102784624118:2049]
2025-11-28T14:51:54.861376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789102784624118:2049] [node 2] ICS09 handshake done sender: [6:7577789102784624098:2118] self: [6:7577789102784624116:0] peer: [2:7577789101819520629:0] socket: 325 qp: -1
2025-11-28T14:51:54.861409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789102784624118:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:54.861446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:54.882608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789104822171454:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.882791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789104822171356:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:54.881373Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789101819520632:2131] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:54.885769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789104822171456:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-28T14:51:54.884652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789101819520632:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:54.891601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789104822171356:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11113) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789104822171454:2120] held: no
2025-11-28T14:51:54.891788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789104822171454:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:54.888760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789101819520632:2131] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:54.889065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789101819520542:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:54.889148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:54.889202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789101819520542:2049] [node 3] ICP22 created new session: [2:7577789101819520662:2049]
2025-11-28T14:51:54.892556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789104822171356:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:54.889255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789101819520662:2049] [node 3] ICS09 handshake done sender: [2:7577789101819520632:2131] self: [2:7577789101819520630:0] peer: [3:7577789104822171455:0] socket: 306 qp: -1
2025-11-28T14:51:54.892719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:54.889291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789101819520662:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:54.889329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:54.892777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789104822171356:2049] [node 2] ICP22 created new session: [3:7577789104822171457:2049]
2025-11-28T14:51:54.893872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789104822171457:2049] [node 2] ICS09 handshake done sender: [3:7577789104822171454:2120] self: [3:7577789104822171455:0] peer: [2:7577789101819520630:0] socket: 335 qp: -1
2025-11-28T14:51:54.893925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789104822171457:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:54.893988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:54.938359Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:54.938541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789101819520649:2048] [node 1] ICS01 socket: 300 reason# EndOfStream
2025-11-28T14:51:54.938573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789101819520541:2048] [node 1] ICP30 unregister session Session# [2:7577789101819520649:2048] VirtualId# [2:7577789101819520644:0]
2025-11-28T14:51:54.938598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789101819520649:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.938677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:54.938710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:54.944512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:54.945208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789104822171438:2048] [node 1] ICS01 socket: 292 reason# EndOfStream
2025-11-28T14:51:54.945252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789104822171355:2048] [node 1] ICP30 unregister session Session# [3:7577789104822171438:2048] VirtualId# [3:7577789104822171435:0]
2025-11-28T14:51:54.945293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789104822171438:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.945423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:54.945488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:54.946274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:54.946526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789101701359014:2048] [node 1] ICS07 socket disconnect 288 reason# EndOfStream
2025-11-28T14:51:54.946554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789101701359014:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.946647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:54.946693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789101701359014:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:54.947436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789101701358930:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9656) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:54.947484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789101701359014:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:54.947518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789101701358930:2048] [node 1] ICP30 unregister session Session# [4:7577789101701359014:2048] VirtualId# [4:7577789101701359009:0]
2025-11-28T14:51:54.947560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:54.947586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789101701358930:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:54.947600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:54.948482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:54.948743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789103071140312:2048] [node 1] ICS01 socket: 312 reason# EndOfStream
2025-11-28T14:51:54.948785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789103071140228:2048] [node 1] ICP30 unregister session Session# [5:7577789103071140312:2048] VirtualId# [5:7577789103071140310:0]
2025-11-28T14:51:54.948819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789103071140312:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.948940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:54.963285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789101701358930:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:54.956504Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:54.969583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789102784624105:2048] [node 1] ICS07 socket disconnect 338 reason# EndOfStream
2025-11-28T14:51:54.969626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789102784624105:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.969782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:54.969833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789102784624105:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:54.948997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:54.971285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789102784624016:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9656) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:54.971355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789102784624105:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:54.971394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789102784624016:2048] [node 1] ICP30 unregister session Session# [6:7577789102784624105:2048] VirtualId# [6:7577789102784624103:0]
2025-11-28T14:51:54.971459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:54.971494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789102784624016:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:54.971529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:54.972481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:54.972734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789104822171457:2049] [node 2] ICS01 socket: 335 reason# EndOfStream
2025-11-28T14:51:54.972777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789104822171356:2049] [node 2] ICP30 unregister session Session# [3:7577789104822171457:2049] VirtualId# [3:7577789104822171455:0]
2025-11-28T14:51:54.972810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789104822171457:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.972898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:54.972942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:54.981972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789102784624016:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:54.984610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:54.984893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789103071140331:2049] [node 2] ICS01 socket: 323 reason# EndOfStream
2025-11-28T14:51:54.984942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789103071140229:2049] [node 2] ICP30 unregister session Session# [5:7577789103071140331:2049] VirtualId# [5:7577789103071140329:0]
2025-11-28T14:51:54.984637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:54.984977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789103071140331:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.985073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:54.984935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789102784624118:2049] [node 2] ICS01 socket: 325 reason# EndOfStream
2025-11-28T14:51:54.985126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:54.984993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789102784624017:2049] [node 2] ICP30 unregister session Session# [6:7577789102784624118:2049] VirtualId# [6:7577789102784624116:0]
2025-11-28T14:51:54.985036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789102784624118:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.985168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:54.985218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:54.985677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:54.985895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789101701359031:2049] [node 2] ICS01 socket: 326 reason# EndOfStream
2025-11-28T14:51:54.985948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789101701358931:2049] [node 2] ICP30 unregister session Session# [4:7577789101701359031:2049] VirtualId# [4:7577789101701359029:0]
2025-11-28T14:51:54.985978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789101701359031:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:54.986086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:54.986136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:55.040540Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:55.040771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789101701359027:2050] [node 3] ICS07 socket disconnect 316 reason# EndOfStream
2025-11-28T14:51:55.040801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789101701359027:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.040975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:55.041208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789101701359027:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:55.042234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789101701358932:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.042292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789101701359027:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.042329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789101701358932:2050] [node 3] ICP30 unregister session Session# [4:7577789101701359027:2050] VirtualId# [4:7577789101701359020:0]
2025-11-28T14:51:55.042389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:55.042419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789101701358932:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.042435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.045402Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:55.044587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:55.044968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789103071140325:2050] [node 3] ICS07 socket disconnect 286 reason# EndOfStream
2025-11-28T14:51:55.045001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789103071140325:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.045174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:55.045280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789103071140325:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:55.052945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789102784624112:2050] [node 3] ICS07 socket disconnect 319 reason# EndOfStream
2025-11-28T14:51:55.053039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789102784624112:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.053188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:55.053234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789102784624112:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:55.058896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789103071140230:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.058970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789103071140325:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.059001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789103071140230:2050] [node 3] ICP30 unregister session Session# [5:7577789103071140325:2050] VirtualId# [5:7577789103071140322:0]
2025-11-28T14:51:55.059050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:55.059083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789103071140230:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.059107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.068510Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789102784624018:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.068592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789102784624112:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.068656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789102784624018:2050] [node 3] ICP30 unregister session Session# [6:7577789102784624112:2050] VirtualId# [6:7577789102784624109:0]
2025-11-28T14:51:55.068713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:55.068752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789102784624018:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.068767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.068962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789103071140230:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:55.081994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789102784624018:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:55.084534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:55.084801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789103071140318:2108] [node 4] ICS07 socket disconnect 305 reason# EndOfStream
2025-11-28T14:51:55.084829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789103071140318:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.084896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:55.084924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789103071140318:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:55.085854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789103071140293:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26554) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.085910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789103071140318:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.085946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789103071140293:2108] [node 4] ICP30 unregister session Session# [5:7577789103071140318:2108] VirtualId# [5:7577789103071140316:0]
2025-11-28T14:51:55.085984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:55.086011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789103071140293:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.086034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.092555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:55.092814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789102784624099:2110] [node 4] ICS07 socket disconnect 299 reason# EndOfStream
2025-11-28T14:51:55.092848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789102784624099:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.092958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:55.092998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789102784624099:2110] [node 4] ICS15 start handshake
2025-11-28T14:51:55.096606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789103071140293:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:51:55.100847Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789102784624084:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26554) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.105455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789102784624099:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.105529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789102784624084:2110] [node 4] ICP30 unregister session Session# [6:7577789102784624099:2110] VirtualId# [6:7577789102784624095:0]
2025-11-28T14:51:55.105598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:55.105644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789102784624084:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.105660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.120493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789102784624084:2110] [node 4] ICP33 wake up from error state
2025-11-28T14:51:55.144500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:55.144765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789102784624114:2107] [node 5] ICS07 socket disconnect 308 reason# EndOfStream
2025-11-28T14:51:55.144796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789102784624114:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.144998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:55.145096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789102784624114:2107] [node 5] ICS15 start handshake
2025-11-28T14:51:55.147462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789102784624078:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9503) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.147607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789102784624114:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.147685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789102784624078:2107] [node 5] ICP30 unregister session Session# [6:7577789102784624114:2107] VirtualId# [6:7577789102784624108:0]
2025-11-28T14:51:55.147751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:55.147799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789102784624078:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9503) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.147816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9503) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:55.206675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789106256183657:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:55.207305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789106256183656:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:55.229701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789106865336742:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:55.230000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789106865336743:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:55.251766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789107904428445:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:55.253342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789107904428446:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:55.277506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789109519084442:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:55.277556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789109519084443:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:55.280820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789109519084441:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:55.304725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789106225483531:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:55.304731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789106225483530:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:55.304775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789106225483532:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:55.328263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789106229651896:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:55.328322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789106229651897:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:55.328488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789106229651895:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:55.440566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789106229651955:2105] [node 5] ICP01 ready to work
2025-11-28T14:51:55.503265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789106865336806:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:55.504598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789109519084504:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:55.504918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789106225483602:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:55.505227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789106865336807:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:55.508751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789107904428535:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:55.505276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789106256183719:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:55.513469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789106229651957:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:55.505662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789109519084505:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:55.513412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789106225483603:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:55.516594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789106256183720:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:55.517946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789107904428536:2135] [node 5] ICP01 ready to work
2025-11-28T14:51:55.508854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789106865336808:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:55.521713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789106256183721:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:55.523081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789107904428537:2124] [node 6] ICP01 ready to work
2025-11-28T14:51:55.525621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789109519084441:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16751889ns) {Configure:104(16745358ns) {ProcessConfigured:135(2286494ns) {StartInitialHandshake:157(2282146ns) {DropHandshakes:470(2877ns) {DropIncomingHandshake:442(800ns) DropOutgoingHandshake:457(525ns)} GenerateSessionVirtualId:988(738ns) PrepareNewSessionHandshake:477(387398ns) {SwitchToState:179(1682ns)}}}}}
2025-11-28T14:51:55.528176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789106229651955:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(87306382ns) {Configure:104(87302737ns) {ProcessConfigured:135(62835ns) {StartInitialHandshake:157(60074ns) {DropHandshakes:470(1488ns) {DropIncomingHandshake:442(396ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(772ns) PrepareNewSessionHandshake:477(1267ns) {SwitchToState:179(593ns)}}}}}
2025-11-28T14:51:55.526240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789106256183656:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(120414905ns) {Configure:104(120411431ns) {ProcessConfigured:135(1231898ns) {StartInitialHandshake:157(1229821ns) {DropHandshakes:470(454833ns) {DropIncomingHandshake:442(1979ns) DropOutgoingHandshake:457(318ns)} GenerateSessionVirtualId:988(1579ns) PrepareNewSessionHandshake:477(1482ns) {SwitchToState:179(696ns)}}}}}
2025-11-28T14:51:55.530245Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789106225483531:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(87855695ns) {Configure:104(87849660ns) {ProcessConfigured:135(1710776ns) {StartInitialHandshake:157(1707941ns) {DropHandshakes:470(2428ns) {DropIncomingHandshake:442(907ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:477(16717ns) {SwitchToState:179(15107ns)}}}}}
2025-11-28T14:51:55.536165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789107904428446:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(107765980ns) {Configure:104(107761784ns) {ProcessConfigured:135(929800ns) {StartInitialHandshake:157(927674ns) {DropHandshakes:470(372710ns) {DropIncomingHandshake:442(600ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(897ns) PrepareNewSessionHandshake:477(1154ns) {SwitchToState:179(182ns)}}}}}
2025-11-28T14:51:55.535967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789106865336742:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(115070612ns) {Configure:104(115061025ns) {ProcessConfigured:135(3150704ns) {StartInitialHandshake:157(3148327ns) {DropHandshakes:470(1567ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(1094ns) PrepareNewSessionHandshake:477(3119ns) {SwitchToState:179(1211ns)}}}}}
2025-11-28T14:51:55.536256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789107904428445:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(107980384ns) {Configure:104(107974054ns) {ProcessConfigured:135(1458749ns) {StartInitialHandshake:157(1456960ns) {DropHandshakes:470(2687ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:477(1825ns) {SwitchToState:179(202ns)}}}}}
2025-11-28T14:51:55.539615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789106229651897:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27565771ns) {Configure:104(27559201ns) {ProcessConfigured:135(420030ns) {StartInitialHandshake:157(418259ns) {DropHandshakes:470(3296ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(601ns) PrepareNewSessionHandshake:477(733ns) {SwitchToState:179(290ns)}}}}}
2025-11-28T14:51:55.539215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789109519084442:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27215273ns) {Configure:104(27211071ns) {ProcessConfigured:135(3432510ns) {StartInitialHandshake:157(3430302ns) {DropHandshakes:470(2459ns) {DropIncomingHandshake:442(818ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(1056ns) PrepareNewSessionHandshake:477(2205ns) {SwitchToState:179(1204ns)}}}}}
2025-11-28T14:51:55.552501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789106229651896:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20151021ns) {Configure:104(20144792ns) {ProcessConfigured:135(969117ns) {StartInitialHandshake:157(967695ns) {DropHandshakes:470(186674ns) {DropIncomingHandshake:442(493ns) DropOutgoingHandshake:457(530ns)} GenerateSessionVirtualId:988(777ns) PrepareNewSessionHandshake:477(2218ns) {SwitchToState:179(462ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.541317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789106225483532:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19973416ns) {Configure:104(19963091ns) {ProcessConfigured:135(1508324ns) {StartInitialHandshake:157(671579ns) {DropHandshakes:470(1687ns) {DropIncomingHandshake:442(352ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(1136ns) PrepareNewSessionHandshake:477(3374ns) {SwitchToState:179(649ns)}}}}}
2025-11-28T14:51:55.541324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789106256183657:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16943431ns) {Configure:104(16939546ns) {ProcessConfigured:135(857469ns) {StartInitialHandshake:157(855092ns) {DropHandshakes:470(1225ns) {DropIncomingHandshake:442(436ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(864ns) PrepareNewSessionHandshake:477(1278ns) {SwitchToState:179(538ns)}}}}}
2025-11-28T14:51:55.553089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789106865336743:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21245898ns) {Configure:104(21239235ns) {ProcessConfigured:135(2951078ns) {StartInitialHandshake:157(2948030ns) {DropHandshakes:470(665ns) {DropIncomingHandshake:442(209ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(2966ns) {SwitchToState:179(1423ns)}}}}}
2025-11-28T14:51:55.565440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789106865336819:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13875
2025-11-28T14:51:55.565501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789106865336820:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13875
2025-11-28T14:51:55.549013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789106225483530:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21580771ns) {Configure:104(19737186ns) {ProcessConfigured:135(3643104ns) {StartInitialHandshake:157(3640586ns) {DropHandshakes:470(1390ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(582ns) PrepareNewSessionHandshake:477(1650ns) {SwitchToState:179(729ns)}}}}}
2025-11-28T14:51:55.569017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789106865336806:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27999903ns) {Configure:104(27994225ns) {ProcessConfigured:135(7217324ns) {StartInitialHandshake:157(7215300ns) {DropHandshakes:470(1723ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(429ns)} GenerateSessionVirtualId:988(692ns) PrepareNewSessionHandshake:477(2617ns) {SwitchToState:179(651ns)}}}}}
2025-11-28T14:51:55.575225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789107904428535:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43051459ns) {Configure:104(43045470ns) {ProcessConfigured:135(8067837ns) {StartInitialHandshake:157(8064484ns) {DropHandshakes:470(2321ns) {DropIncomingHandshake:442(587ns) DropOutgoingHandshake:457(347ns)} GenerateSessionVirtualId:988(756ns) PrepareNewSessionHandshake:477(2318ns) {SwitchToState:179(889ns)}}}}}
2025-11-28T14:51:55.576770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789106256183720:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(44934476ns) {Configure:104(44927365ns) {ProcessConfigured:135(5144850ns) {StartInitialHandshake:157(5141503ns) {DropHandshakes:470(5326ns) {DropIncomingHandshake:442(648ns) DropOutgoingHandshake:457(467ns)} GenerateSessionVirtualId:988(741ns) PrepareNewSessionHandshake:477(2693ns) {SwitchToState:179(1061ns)}}}}}
2025-11-28T14:51:55.577843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789106229651895:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46911397ns) {Configure:104(46907555ns) {ProcessConfigured:135(4570562ns) {StartInitialHandshake:157(4569189ns) {DropHandshakes:470(725ns) {DropIncomingHandshake:442(206ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(2562ns) {SwitchToState:179(293ns)}}}}}
2025-11-28T14:51:55.578710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789109519084505:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50305922ns) {Configure:104(50291310ns) {ProcessConfigured:135(5683284ns) {StartInitialHandshake:157(5680499ns) {DropHandshakes:470(1854ns) {DropIncomingHandshake:442(493ns) DropOutgoingHandshake:457(300ns)} GenerateSessionVirtualId:988(614ns) PrepareNewSessionHandshake:477(2474ns) {SwitchToState:179(131ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.581906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789107904428536:2135] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(45064814ns) {Configure:104(44846343ns) {ProcessConfigured:135(1712677ns) {StartInitialHandshake:157(1710605ns) {DropHandshakes:470(1181ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(1509ns) {SwitchToState:179(212ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.581669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789109519084514:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38244
2025-11-28T14:51:55.581535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789106865336820:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.582031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789106229651957:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(39618419ns) {Configure:104(39611863ns) {ProcessConfigured:135(2633274ns) {StartInitialHandshake:157(1893586ns) {DropHandshakes:470(2681ns) {DropIncomingHandshake:442(570ns) DropOutgoingHandshake:457(342ns)} GenerateSessionVirtualId:988(1272ns) PrepareNewSessionHandshake:477(1989ns) {SwitchToState:179(766ns)}}}}}
2025-11-28T14:51:55.582283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789106865336824:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13875
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.582279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789106225483602:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(44950135ns) {Configure:104(44473543ns) {ProcessConfigured:135(745775ns) {StartInitialHandshake:157(742695ns) {DropHandshakes:470(1243ns) {DropIncomingHandshake:442(362ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(440ns) PrepareNewSessionHandshake:477(2345ns) {SwitchToState:179(1111ns)}}}}}
2025-11-28T14:51:55.582908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789106865336819:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.582682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651966:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.583211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789109519084514:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.583404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789106256183733:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7518
2025-11-28T14:51:55.582942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651968:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
2025-11-28T14:51:55.584398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789106865336742:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.585324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789109519084441:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.585373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789106256183736:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7518
2025-11-28T14:51:55.582957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651967:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
2025-11-28T14:51:55.583684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789106229651966:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.586824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789106225483610:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58688
2025-11-28T14:51:55.585378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789107904428550:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2616
2025-11-28T14:51:55.585392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789106229651968:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.586432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789106256183723:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.585402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789106229651967:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.587365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789107904428550:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.586973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789106229651896:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.587360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789106229651895:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.586463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789106256183723:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.586962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789106256183733:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.588054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789106256183725:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.588083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789106256183725:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.587364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789106229651897:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.592805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789107904428543:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.593776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789106225483610:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.598808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789106256183723:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:55.604863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789106225483531:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.592816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789107904428553:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2616
2025-11-28T14:51:55.609538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789106865336817:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.608604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789106256183727:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.592839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789107904428543:2139] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.608661Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789106256183727:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.609598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789106865336817:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.608715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789106225483616:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58688
2025-11-28T14:51:55.612511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789106256183737:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7518
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.592847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789107904428552:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2616
2025-11-28T14:51:55.612746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789106256183725:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:55.608777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789107904428543:2139] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:55.626948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789106865336815:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.626990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789106865336815:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.627168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789106865336827:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13875
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:51:55.613018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789106256183656:2048] [node 2] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:51:55.613061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:55.626749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789107904428537:2124] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.622861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789106256183656:2048] [node 2] ICP22 created new session: [1:7577789106256183742:2048]
2025-11-28T14:51:55.626814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:55.625461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789106256183742:2048] [node 2] ICS09 handshake done sender: [1:7577789106256183723:2110] self: [1:7577789106256183722:0] peer: [2:7577789106865336826:0] socket: 288 qp: -1
2025-11-28T14:51:55.627767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789106865336828:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13875
2025-11-28T14:51:55.627761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789109519084521:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38244
2025-11-28T14:51:55.627076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789107904428537:2124] [node 6] ICP22 created new session: [3:7577789107904428556:2124]
2025-11-28T14:51:55.627925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651979:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
2025-11-28T14:51:55.625808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789106256183742:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:55.628722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789106865336817:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:55.627234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789107904428556:2124] [node 6] ICS09 handshake done sender: [3:7577789107904428543:2139] self: [3:7577789107904428542:0] peer: [6:7577789106229651978:0] socket: 293 qp: -1
2025-11-28T14:51:55.628784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789106865336815:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:55.625845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:55.627589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789107904428556:2124] [node 6] ICS10 traffic start
2025-11-28T14:51:55.627632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:55.626838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789106256183737:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.626961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789106256183719:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.631040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789109519084514:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:55.632832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789106865336829:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13875
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:55.626987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:55.633131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789106865336807:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.627163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789106256183739:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7518
2025-11-28T14:51:55.633320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789109519084441:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.627209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789106256183719:2107] [node 4] ICP22 created new session: [1:7577789106256183744:2107]
2025-11-28T14:51:55.633167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:55.627576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789106256183744:2107] [node 4] ICS09 handshake done sender: [1:7577789106256183725:2111] self: [1:7577789106256183724:0] peer: [4:7577789109519084520:0] socket: 289 qp: -1
2025-11-28T14:51:55.633374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:55.633497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789109519084441:2048] [node 1] ICP22 created new session: [4:7577789109519084523:2048]
2025-11-28T14:51:55.633460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789107904428559:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2616
2025-11-28T14:51:55.627701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789106256183744:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:55.633743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789109519084523:2048] [node 1] ICS09 handshake done sender: [4:7577789109519084514:2114] self: [4:7577789109519084520:0] peer: [1:7577789106256183724:0] socket: 294 qp: -1
2025-11-28T14:51:55.627728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:55.634420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789109519084523:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:55.633310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789106865336807:2110] [node 6] ICP22 created new session: [2:7577789106865336832:2110]
2025-11-28T14:51:55.629473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789106256183727:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:55.634890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:55.633504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789106865336832:2110] [node 6] ICS09 handshake done sender: [2:7577789106865336817:2114] self: [2:7577789106865336816:0] peer: [6:7577789106229651976:0] socket: 302 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.633606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789106865336832:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:55.631584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789106256183721:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.633631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:55.632754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:55.635229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789106865336808:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.635889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789109519084522:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38244
2025-11-28T14:51:55.632845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789106256183719:2107] [node 4] ICP18 handshake (actor [1:7577789106256183737:2117]) is too old
2025-11-28T14:51:55.632880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789106256183721:2109] [node 6] ICP22 created new session: [1:7577789106256183749:2109]
2025-11-28T14:51:55.636497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789107904428559:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.635262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:55.633035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789106256183749:2109] [node 6] ICS09 handshake done sender: [1:7577789106256183727:2112] self: [1:7577789106256183726:0] peer: [6:7577789106229651977:0] socket: 290 qp: -1
2025-11-28T14:51:55.635343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789106865336808:2109] [node 5] ICP22 created new session: [2:7577789106865336834:2109]
2025-11-28T14:51:55.635632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789106865336834:2109] [node 5] ICS09 handshake done sender: [2:7577789106865336815:2113] self: [2:7577789106865336814:0] peer: [5:7577789106225483619:0] socket: 301 qp: -1
2025-11-28T14:51:55.636965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789107904428445:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.633161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789106256183749:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:55.633306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:55.635737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789106865336834:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:55.635828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:55.637285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789106256183732:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.637325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789106256183732:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.637288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789106865336819:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:55.638110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789106256183732:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:55.638400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789106865336742:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.638861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789106256183657:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.638847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.639224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789106865336742:2048] [node 1] ICP22 created new session: [2:7577789106865336838:2048]
2025-11-28T14:51:55.638964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:55.639224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789106256183657:2049] [node 3] ICP22 created new session: [1:7577789106256183752:2049]
2025-11-28T14:51:55.640106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789106256183752:2049] [node 3] ICS09 handshake done sender: [1:7577789106256183732:2113] self: [1:7577789106256183731:0] peer: [3:7577789107904428560:0] socket: 341 qp: -1
2025-11-28T14:51:55.640075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789106865336838:2048] [node 1] ICS09 handshake done sender: [2:7577789106865336819:2116] self: [2:7577789106865336826:0] peer: [1:7577789106256183722:0] socket: 298 qp: -1
2025-11-28T14:51:55.644845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789109519084526:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38244
2025-11-28T14:51:55.640109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789106865336838:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:55.640960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789106256183752:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.640788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:55.644616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:55.645272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789107904428562:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2616
2025-11-28T14:51:55.645519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789107904428445:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789107904428559:2145] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.646870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789107904428561:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2616
2025-11-28T14:51:55.647461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789107904428559:2145] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:55.648005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789109519084526:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.648236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789109519084442:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.648153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789107904428562:2147] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.648506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789106865336818:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.648543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789106865336818:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.648215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789107904428445:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.649362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789106865336818:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:55.648274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.648329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789107904428445:2048] [node 1] ICP22 created new session: [3:7577789107904428564:2048]
2025-11-28T14:51:55.649841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789106865336806:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.648508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789107904428564:2048] [node 1] ICS09 handshake done sender: [3:7577789107904428559:2145] self: [3:7577789107904428560:0] peer: [1:7577789106256183731:0] socket: 342 qp: -1
2025-11-28T14:51:55.650374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:55.650505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789109519084528:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38244
2025-11-28T14:51:55.648553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789107904428564:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:55.650438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789106865336806:2107] [node 4] ICP22 created new session: [2:7577789106865336841:2107]
2025-11-28T14:51:55.648586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:55.650483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789106865336841:2107] [node 4] ICS09 handshake done sender: [2:7577789106865336818:2115] self: [2:7577789106865336813:0] peer: [4:7577789109519084527:0] socket: 315 qp: -1
2025-11-28T14:51:55.650916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789107904428446:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.650834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789109519084526:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:55.650516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789106865336841:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:55.650553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:55.651806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789109519084442:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.651991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789106865336812:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.651854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:55.652025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789106865336812:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.652051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789109519084442:2049] [node 2] ICP22 created new session: [4:7577789109519084529:2049]
2025-11-28T14:51:55.652747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789109519084529:2049] [node 2] ICS09 handshake done sender: [4:7577789109519084526:2118] self: [4:7577789109519084527:0] peer: [2:7577789106865336813:0] socket: 345 qp: -1
2025-11-28T14:51:55.652789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789109519084529:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:55.652849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:55.652980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789106865336812:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:55.653272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789106865336743:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.653614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.653733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789106865336743:2049] [node 3] ICP22 created new session: [2:7577789106865336844:2049]
2025-11-28T14:51:55.653844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789106865336844:2049] [node 3] ICS09 handshake done sender: [2:7577789106865336812:2112] self: [2:7577789106865336811:0] peer: [3:7577789107904428566:0] socket: 346 qp: -1
2025-11-28T14:51:55.653971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789107904428567:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2616
2025-11-28T14:51:55.653939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789106865336844:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:55.653982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:55.655641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789107904428446:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789107904428562:2147] held: no
2025-11-28T14:51:55.655839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789107904428562:2147] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:55.656066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789107904428446:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.656107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:55.656144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789107904428446:2049] [node 2] ICP22 created new session: [3:7577789107904428568:2049]
2025-11-28T14:51:55.656173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789107904428568:2049] [node 2] ICS09 handshake done sender: [3:7577789107904428562:2147] self: [3:7577789107904428566:0] peer: [2:7577789106865336811:0] socket: 348 qp: -1
2025-11-28T14:51:55.656200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789107904428568:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:55.656230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:55.659130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789106865336810:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.672807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789109519084511:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.673841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789106865336824:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.689170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789106256183736:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.689166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789106865336806:2107] [node 4] ICP18 handshake (actor [2:7577789106865336824:2118]) is too old
2025-11-28T14:51:55.697912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789106256183656:2048] [node 2] ICP18 handshake (actor [1:7577789106256183736:2116]) is too old
2025-11-28T14:51:55.705823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789106229651961:2110] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.719637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789106229651967:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:55.720000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789106229651897:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.720065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:55.720128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789106229651897:2050] [node 3] ICP22 created new session: [6:7577789106229651983:2050]
2025-11-28T14:51:55.720191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789106229651983:2050] [node 3] ICS09 handshake done sender: [6:7577789106229651967:2114] self: [6:7577789106229651978:0] peer: [3:7577789107904428542:0] socket: 306 qp: -1
2025-11-28T14:51:55.720231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789106229651983:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:55.720273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.721067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651980:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.721514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
2025-11-28T14:51:55.721570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789106229651963:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.725458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651982:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
2025-11-28T14:51:55.725534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789107904428552:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.725682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789107904428537:2124] [node 6] ICP18 handshake (actor [3:7577789107904428552:2143]) is too old
2025-11-28T14:51:55.750458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789106229651968:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:55.750520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789106229651966:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:55.750833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789106229651896:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.750903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:55.750931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789106229651895:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.750973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789106229651896:2049] [node 2] ICP22 created new session: [6:7577789106229651985:2049]
2025-11-28T14:51:55.751002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:55.751020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789106229651985:2049] [node 2] ICS09 handshake done sender: [6:7577789106229651968:2115] self: [6:7577789106229651976:0] peer: [2:7577789106865336816:0] socket: 307 qp: -1
2025-11-28T14:51:55.751065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789106229651895:2048] [node 1] ICP22 created new session: [6:7577789106229651986:2048]
2025-11-28T14:51:55.751070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789106229651985:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:55.751115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:55.751136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789106229651986:2048] [node 1] ICS09 handshake done sender: [6:7577789106229651966:2113] self: [6:7577789106229651977:0] peer: [1:7577789106256183726:0] socket: 305 qp: -1
2025-11-28T14:51:55.751176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789106229651986:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:55.751225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.752764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789109519084532:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38244
2025-11-28T14:51:55.778182Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789106225483608:2120] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.796131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789107904428553:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.796278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789106225483616:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.796854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789106225483620:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58688
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.797320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789106225483621:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58688
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.797465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789106225483622:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58688
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.797570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789106225483623:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58688
2025-11-28T14:51:55.797767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789106225483621:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.797821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789106225483610:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:55.798041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789106225483622:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.798117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789106225483623:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.798156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789106225483531:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.798204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:55.798245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789106225483531:2049] [node 2] ICP22 created new session: [5:7577789106225483625:2049]
2025-11-28T14:51:55.798292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789106225483625:2049] [node 2] ICS09 handshake done sender: [5:7577789106225483610:2122] self: [5:7577789106225483619:0] peer: [2:7577789106865336814:0] socket: 309 qp: -1
2025-11-28T14:51:55.798330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789106225483625:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:55.798363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:55.798972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789106225483530:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.799030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789106225483532:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.799266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789106225483602:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.799268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789106256183735:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.799295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789106256183735:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.799415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789107904428551:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.799464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789109519084517:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.799459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789107904428551:2142] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.799497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789109519084517:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:55.800600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789106256183735:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:55.801219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789107904428551:2142] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:55.802346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789106256183720:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.803703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:55.804021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789107904428536:2135] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.803920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789106256183720:2108] [node 5] ICP22 created new session: [1:7577789106256183755:2108]
2025-11-28T14:51:55.803974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789106256183755:2108] [node 5] ICS09 handshake done sender: [1:7577789106256183735:2115] self: [1:7577789106256183734:0] peer: [5:7577789106225483627:0] socket: 343 qp: -1
2025-11-28T14:51:55.804475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789109519084517:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:55.804139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:55.804193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789107904428536:2135] [node 5] ICP22 created new session: [3:7577789107904428572:2135]
2025-11-28T14:51:55.804316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789107904428572:2135] [node 5] ICS09 handshake done sender: [3:7577789107904428551:2142] self: [3:7577789107904428549:0] peer: [5:7577789106225483628:0] socket: 292 qp: -1
2025-11-28T14:51:55.804012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789106256183755:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:55.804420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789107904428572:2135] [node 5] ICS10 traffic start
2025-11-28T14:51:55.804092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:55.804519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:55.805192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789109519084505:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.805324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:55.805386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789109519084505:2108] [node 5] ICP22 created new session: [4:7577789109519084534:2108]
2025-11-28T14:51:55.805434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789109519084534:2108] [node 5] ICS09 handshake done sender: [4:7577789109519084517:2115] self: [4:7577789109519084516:0] peer: [5:7577789106225483629:0] socket: 291 qp: -1
2025-11-28T14:51:55.805463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789109519084534:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:55.805506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.806165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789106225483630:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58688
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.806893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789106225483631:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58688
2025-11-28T14:51:55.807957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789106225483532:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789106225483622:2128] held: no
2025-11-28T14:51:55.808154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789106225483609:2121] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.808340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789106225483622:2128] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.830158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789106225483632:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58688
2025-11-28T14:51:55.830335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789106225483621:2127] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:55.835528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789106256183739:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.835810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789106256183720:2108] [node 5] ICP18 handshake (actor [1:7577789106256183739:2118]) is too old
2025-11-28T14:51:55.835785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789106225483532:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.835856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:55.835903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789106225483532:2050] [node 3] ICP22 created new session: [5:7577789106225483634:2050]
2025-11-28T14:51:55.835910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789106225483530:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.835943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789106225483634:2050] [node 3] ICS09 handshake done sender: [5:7577789106225483622:2128] self: [5:7577789106225483628:0] peer: [3:7577789107904428549:0] socket: 352 qp: -1
2025-11-28T14:51:55.835967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:55.835975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789106225483634:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:55.836007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:55.836025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789106225483530:2048] [node 1] ICP22 created new session: [5:7577789106225483635:2048]
2025-11-28T14:51:55.836072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789106225483635:2048] [node 1] ICS09 handshake done sender: [5:7577789106225483621:2127] self: [5:7577789106225483627:0] peer: [1:7577789106256183734:0] socket: 319 qp: -1
2025-11-28T14:51:55.836113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789106225483635:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:55.836155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:55.836788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789106225483623:2129] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.837279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789109519084536:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38244
2025-11-28T14:51:55.837411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789106225483602:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.837444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:55.837473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789106225483602:2108] [node 4] ICP22 created new session: [5:7577789106225483638:2108]
2025-11-28T14:51:55.837498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789106225483638:2108] [node 4] ICS09 handshake done sender: [5:7577789106225483623:2129] self: [5:7577789106225483629:0] peer: [4:7577789109519084516:0] socket: 353 qp: -1
2025-11-28T14:51:55.837537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789106225483638:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:55.837568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.838516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651991:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
2025-11-28T14:51:55.852300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789106225483605:2119] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.873984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789106865336828:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.874178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789106865336808:2109] [node 5] ICP18 handshake (actor [2:7577789106865336828:2120]) is too old
2025-11-28T14:51:55.877907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789106229651965:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.888409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789106865336827:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.888573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789106865336807:2110] [node 6] ICP18 handshake (actor [2:7577789106865336827:2119]) is too old
2025-11-28T14:51:55.892147Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789107904428548:2140] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.902201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789109519084522:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.902404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789109519084443:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.902674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789107904428548:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.903121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789107904428548:2140] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:55.903325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789107904428535:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.903400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:55.903446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789107904428535:2107] [node 4] ICP22 created new session: [3:7577789107904428575:2107]
No RDMA context found for GID: 2025-11-28T14:51:55.903501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789107904428575:2107] [node 4] ICS09 handshake done sender: [3:7577789107904428548:2140] self: [3:7577789107904428547:0] peer: [4:7577789109519084537:0] socket: 336 qp: -1
::ffff:127.1.0.0
2025-11-28T14:51:55.903549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789107904428575:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:55.903597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:55.903666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789109519084538:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38244
2025-11-28T14:51:55.904516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789109519084443:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789109519084522:2117] held: no
2025-11-28T14:51:55.904762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789109519084522:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:55.904892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789109519084443:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.904937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:55.905389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789109519084443:2050] [node 3] ICP22 created new session: [4:7577789109519084539:2050]
2025-11-28T14:51:55.905434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789109519084539:2050] [node 3] ICS09 handshake done sender: [4:7577789109519084522:2117] self: [4:7577789109519084537:0] peer: [3:7577789107904428547:0] socket: 337 qp: -1
2025-11-28T14:51:55.905471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789109519084539:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:55.905515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:55.906510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789109519084513:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.919886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789106229651982:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.920156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789106229651957:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.920432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789109519084513:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.920996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789109519084513:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.921167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651993:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
2025-11-28T14:51:55.921197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789109519084504:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.921252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:55.921430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789106229651982:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:55.921314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789109519084504:2109] [node 6] ICP22 created new session: [4:7577789109519084542:2109]
2025-11-28T14:51:55.921494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789109519084542:2109] [node 6] ICS09 handshake done sender: [4:7577789109519084513:2113] self: [4:7577789109519084512:0] peer: [6:7577789106229651992:0] socket: 310 qp: -1
2025-11-28T14:51:55.921722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789106229651957:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.921545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789109519084542:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:55.921773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:55.921666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:55.921833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789106229651957:2107] [node 4] ICP22 created new session: [6:7577789106229651994:2107]
2025-11-28T14:51:55.921881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789106229651994:2107] [node 4] ICS09 handshake done sender: [6:7577789106229651982:2121] self: [6:7577789106229651992:0] peer: [4:7577789109519084512:0] socket: 304 qp: -1
2025-11-28T14:51:55.921909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789106229651994:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:55.921947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:55.923802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789106229651972:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.924667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789109519084532:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.924818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789109519084504:2109] [node 6] ICP18 handshake (actor [4:7577789109519084532:2120]) is too old
2025-11-28T14:51:55.934903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789106225483618:2125] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:55.935674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789106229651991:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.935859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789106229651955:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:55.936049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789106225483618:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:55.936467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789106225483618:2125] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:55.936708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789106229651997:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63243
2025-11-28T14:51:55.936610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789106225483603:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:55.936689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:55.936773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789106225483603:2109] [node 6] ICP22 created new session: [5:7577789106225483642:2109]
2025-11-28T14:51:55.936823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789106225483642:2109] [node 6] ICS09 handshake done sender: [5:7577789106225483618:2125] self: [5:7577789106225483617:0] peer: [6:7577789106229651996:0] socket: 323 qp: -1
2025-11-28T14:51:55.936873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789106225483642:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:55.936918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:55.937274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789106229651991:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:55.937334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789106229651955:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16613) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789106229651991:2122] held: no
2025-11-28T14:51:55.937601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789106229651955:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:55.937655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:55.937701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789106229651955:2105] [node 5] ICP22 created new session: [6:7577789106229651998:2105]
2025-11-28T14:51:55.937766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789106229651998:2105] [node 5] ICS09 handshake done sender: [6:7577789106229651991:2122] self: [6:7577789106229651996:0] peer: [5:7577789106225483617:0] socket: 327 qp: -1
2025-11-28T14:51:55.937798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789106229651998:2105] [node 5] ICS10 traffic start
2025-11-28T14:51:55.937846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:55.972411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:55.972580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789106865336838:2048] [node 1] ICS07 socket disconnect 298 reason# EndOfStream
2025-11-28T14:51:55.972604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789106865336838:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.972703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:55.972740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789106865336838:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:55.973393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:55.973578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789107904428564:2048] [node 1] ICS01 socket: 342 reason# EndOfStream
2025-11-28T14:51:55.973623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789107904428445:2048] [node 1] ICP30 unregister session Session# [3:7577789107904428564:2048] VirtualId# [3:7577789107904428560:0]
2025-11-28T14:51:55.973673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789107904428564:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.973759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:55.973806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:55.974702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789106865336742:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:55.974728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789106865336838:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:55.975502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789106865336742:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.975551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789106865336838:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.975734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789106865336742:2048] [node 1] ICP30 unregister session Session# [2:7577789106865336838:2048] VirtualId# [2:7577789106865336826:0]
2025-11-28T14:51:55.975783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:55.975808Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789106865336742:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.975816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.975949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:55.976127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789109519084523:2048] [node 1] ICS07 socket disconnect 294 reason# EndOfStream
2025-11-28T14:51:55.976160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789109519084523:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.976228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:55.976258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789109519084523:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:55.976911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789109519084441:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.976981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789109519084523:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.976951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:55.977024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789109519084441:2048] [node 1] ICP30 unregister session Session# [4:7577789109519084523:2048] VirtualId# [4:7577789109519084520:0]
2025-11-28T14:51:55.977073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:55.977107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789109519084441:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.977126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.977185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789106225483635:2048] [node 1] ICS07 socket disconnect 319 reason# EndOfStream
2025-11-28T14:51:55.977223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789106225483635:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.977339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:55.977412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789106225483635:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:55.977860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:55.978045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789106225483530:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.978141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789106229651986:2048] [node 1] ICS07 socket disconnect 305 reason# EndOfStream
2025-11-28T14:51:55.978212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789106229651986:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.978228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789106225483635:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.978277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789106225483530:2048] [node 1] ICP30 unregister session Session# [5:7577789106225483635:2048] VirtualId# [5:7577789106225483627:0]
2025-11-28T14:51:55.978292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:55.978339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789106229651986:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:55.978339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:55.978387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789106225483530:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.978402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.979032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789106229651895:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:55.979083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789106229651986:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:55.979121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789106229651895:2048] [node 1] ICP30 unregister session Session# [6:7577789106229651986:2048] VirtualId# [6:7577789106229651977:0]
2025-11-28T14:51:55.979165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:55.979195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789106229651895:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.979208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:55.986627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789109519084441:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:55.988616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789106225483530:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:55.988663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789106229651895:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:55.999291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:55.999539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789107904428568:2049] [node 2] ICS01 socket: 348 reason# EndOfStream
2025-11-28T14:51:55.999588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789107904428446:2049] [node 2] ICP30 unregister session Session# [3:7577789107904428568:2049] VirtualId# [3:7577789107904428566:0]
2025-11-28T14:51:55.999626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789107904428568:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:55.999729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:55.999797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:56.002251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:56.002399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789109519084529:2049] [node 2] ICS07 socket disconnect 345 reason# EndOfStream
2025-11-28T14:51:56.002424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789109519084529:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:56.002492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:56.002525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789109519084529:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:56.003235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789109519084442:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:56.003339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789109519084529:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:56.003390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789109519084442:2049] [node 2] ICP30 unregister session Session# [4:7577789109519084529:2049] VirtualId# [4:7577789109519084527:0]
2025-11-28T14:51:56.003276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:56.003447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:56.003491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789109519084442:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.003507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.003487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789106225483625:2049] [node 2] ICS07 socket disconnect 309 reason# EndOfStream
2025-11-28T14:51:56.003514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789106225483625:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:56.003590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:56.003630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789106225483625:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:56.004345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789106225483531:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:56.004441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789106225483625:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:56.004466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789106225483531:2049] [node 2] ICP30 unregister session Session# [5:7577789106225483625:2049] VirtualId# [5:7577789106225483619:0]
2025-11-28T14:51:56.004473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:56.004513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:56.004535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789106225483531:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.004553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.004718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789106229651985:2049] [node 2] ICS07 socket disconnect 307 reason# EndOfStream
2025-11-28T14:51:56.004757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789106229651985:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:56.004843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:56.004881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789106229651985:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:56.005474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789106229651896:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:56.005516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789106229651985:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:56.005569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789106229651896:2049] [node 2] ICP30 unregister session Session# [6:7577789106229651985:2049] VirtualId# [6:7577789106229651976:0]
2025-11-28T14:51:56.005601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:56.005620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789106229651896:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.005633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.013315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789109519084442:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:56.014189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789106225483531:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:56.015248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789106229651896:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:56.048999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:56.049253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789109519084539:2050] [node 3] ICS01 socket: 337 reason# EndOfStream
2025-11-28T14:51:56.049314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789109519084443:2050] [node 3] ICP30 unregister session Session# [4:7577789109519084539:2050] VirtualId# [4:7577789109519084537:0]
2025-11-28T14:51:56.049364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789109519084539:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:56.049489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:56.049546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:56.049955Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:56.050163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789106229651983:2050] [node 3] ICS07 socket disconnect 306 reason# EndOfStream
2025-11-28T14:51:56.050191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789106229651983:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:56.050256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:56.050296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789106229651983:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:56.050840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:56.051023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789106225483634:2050] [node 3] ICS01 socket: 352 reason# EndOfStream
2025-11-28T14:51:56.051005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789106229651897:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14346) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:56.051210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789106225483532:2050] [node 3] ICP30 unregister session Session# [5:7577789106225483634:2050] VirtualId# [5:7577789106225483628:0]
2025-11-28T14:51:56.051246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789106225483634:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:56.051309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:56.051060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789106229651983:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:56.051343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:56.051257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789106229651897:2050] [node 3] ICP30 unregister session Session# [6:7577789106229651983:2050] VirtualId# [6:7577789106229651978:0]
2025-11-28T14:51:56.051399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:56.051441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789106229651897:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.051471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.061392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789106229651897:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:56.077178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:56.077388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789106225483638:2108] [node 4] ICS07 socket disconnect 353 reason# EndOfStream
2025-11-28T14:51:56.077431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789106225483638:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:56.077518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:56.077560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789106225483638:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:56.078259Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:56.078326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789106225483602:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25749) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:56.078379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789106225483638:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:56.078491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789106229651994:2107] [node 4] ICS07 socket disconnect 304 reason# EndOfStream
2025-11-28T14:51:56.078453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789106225483602:2108] [node 4] ICP30 unregister session Session# [5:7577789106225483638:2108] VirtualId# [5:7577789106225483629:0]
2025-11-28T14:51:56.078529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789106229651994:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:56.078499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:56.078526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789106225483602:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.078606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:56.078544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.078647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789106229651994:2107] [node 4] ICS15 start handshake
2025-11-28T14:51:56.079403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789106229651957:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25749) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:56.079458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789106229651994:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:56.079565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789106229651957:2107] [node 4] ICP30 unregister session Session# [6:7577789106229651994:2107] VirtualId# [6:7577789106229651992:0]
2025-11-28T14:51:56.079621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:56.079659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789106229651957:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.079685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25749) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:56.090009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789106229651957:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:51:56.090276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:56.090499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789106229651998:2105] [node 5] ICS01 socket: 327 reason# EndOfStream
2025-11-28T14:51:56.090543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789106229651955:2105] [node 5] ICP30 unregister session Session# [6:7577789106229651998:2105] VirtualId# [6:7577789106229651996:0]
2025-11-28T14:51:56.090577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789106229651998:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:56.090661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:56.090706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:56.141245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789111395266949:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:56.141245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789111395266950:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:56.161674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789112099344167:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:56.161731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789112099344168:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:56.180864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789112966343740:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:56.180896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789112966343741:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:56.200556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789112378480072:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:56.200597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789112378480073:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:56.200778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789112378480071:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:56.219949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789110347237092:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:56.220017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789110347237093:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:56.220066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789110347237091:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:56.238502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789111133367170:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:56.238546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789111133367172:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:56.238603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789111133367171:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:56.323135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789111395267039:2119] [node 5] ICP01 ready to work
2025-11-28T14:51:56.330173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789111133367232:2107] [node 5] ICP01 ready to work
2025-11-28T14:51:56.337552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789111395267040:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:56.358276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789111395267041:2130] [node 6] ICP01 ready to work
2025-11-28T14:51:56.393413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789112099344258:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:56.402501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789112099344257:2125] [node 5] ICP01 ready to work
2025-11-28T14:51:56.402556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789112099344259:2136] [node 6] ICP01 ready to work
2025-11-28T14:51:56.398614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789112966343804:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:56.398740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789112378480138:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:56.402707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789112378480139:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:56.403052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789110347237159:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:56.403459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789110347237160:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:56.403775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789112966343805:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:56.404195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789111133367243:2109] [node 4] ICP01 ready to work
2025-11-28T14:51:56.404639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789112966343806:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:56.415307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789111395266949:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(99920102ns) {Configure:104(99913638ns) {ProcessConfigured:135(1910907ns) {StartInitialHandshake:157(1907818ns) {DropHandshakes:470(2329ns) {DropIncomingHandshake:442(893ns) DropOutgoingHandshake:457(423ns)} GenerateSessionVirtualId:988(717ns) PrepareNewSessionHandshake:477(2385ns) {SwitchToState:179(830ns)}}}}}
2025-11-28T14:51:56.416087Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789112099344168:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(89017938ns) {Configure:104(89008770ns) {ProcessConfigured:135(954902ns) {StartInitialHandshake:157(953130ns) {DropHandshakes:470(934ns) {DropIncomingHandshake:442(216ns) DropOutgoingHandshake:457(91ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:477(1020ns) {SwitchToState:179(464ns)}}}}}
2025-11-28T14:51:56.415725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789111395266950:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(101202706ns) {Configure:104(100830543ns) {ProcessConfigured:135(1698191ns) {StartInitialHandshake:157(1695800ns) {DropHandshakes:470(2119ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(708ns) PrepareNewSessionHandshake:477(1669ns) {SwitchToState:179(835ns)}}}}}
2025-11-28T14:51:56.416431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789112099344167:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(90224119ns) {Configure:104(85702650ns) {ProcessConfigured:135(612010ns) {StartInitialHandshake:157(609911ns) {DropHandshakes:470(1854ns) {DropIncomingHandshake:442(1055ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(287ns) PrepareNewSessionHandshake:477(943ns) {SwitchToState:179(291ns)}}}}}
2025-11-28T14:51:56.417040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789111133367172:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42580507ns) {Configure:104(42574591ns) {ProcessConfigured:135(124487ns) {StartInitialHandshake:157(120618ns) {DropHandshakes:470(1066ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:477(1213ns) {SwitchToState:179(131ns)}}}}}
2025-11-28T14:51:56.417356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789110347237091:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(67041161ns) {Configure:104(57303845ns) {ProcessConfigured:135(855445ns) {StartInitialHandshake:157(853226ns) {DropHandshakes:470(1182ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(547ns) PrepareNewSessionHandshake:477(1899ns) {SwitchToState:179(1082ns)}}}}}
2025-11-28T14:51:56.418195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789110347237092:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46645516ns) {Configure:104(46638674ns) {ProcessConfigured:135(1073038ns) {StartInitialHandshake:157(1071635ns) {DropHandshakes:470(768ns) {DropIncomingHandshake:442(217ns) DropOutgoingHandshake:457(174ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:477(1067ns) {SwitchToState:179(61ns)}}}}}
2025-11-28T14:51:56.417942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789111133367232:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(46104295ns) {Configure:104(33264631ns) {ProcessConfigured:135(438556ns) {StartInitialHandshake:157(436984ns) {DropHandshakes:470(709ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(1144ns) {SwitchToState:179(416ns)}}}}}
2025-11-28T14:51:56.418971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789112378480073:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17853334ns) {Configure:104(17845727ns) {ProcessConfigured:135(911391ns) {StartInitialHandshake:157(910422ns) {DropHandshakes:470(571ns) {DropIncomingHandshake:442(102ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(922ns) {SwitchToState:179(80ns)}}}}}
2025-11-28T14:51:56.418346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789111133367170:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33904449ns) {Configure:104(22920745ns) {ProcessConfigured:135(330863ns) {StartInitialHandshake:157(329572ns) {DropHandshakes:470(858ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(152ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:477(1036ns) {SwitchToState:179(98ns)}}}}}
2025-11-28T14:51:56.422081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789112966343740:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16614457ns) {Configure:104(16609464ns) {ProcessConfigured:135(821735ns) {StartInitialHandshake:157(819985ns) {DropHandshakes:470(1013ns) {DropIncomingHandshake:442(183ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(2775ns) PrepareNewSessionHandshake:477(1754ns) {SwitchToState:179(718ns)}}}}}
2025-11-28T14:51:56.419630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789112378480072:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17488599ns) {Configure:104(17481841ns) {ProcessConfigured:135(514236ns) {StartInitialHandshake:157(513016ns) {DropHandshakes:470(948ns) {DropIncomingHandshake:442(307ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:477(848ns) {SwitchToState:179(106ns)}}}}}
2025-11-28T14:51:56.426816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789112966343804:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16858176ns) {Configure:104(16854678ns) {ProcessConfigured:135(2071175ns) {StartInitialHandshake:157(2067645ns) {DropHandshakes:470(3772ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(1764ns) {SwitchToState:179(615ns)}}}}}
2025-11-28T14:51:56.440620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789112966343741:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21490604ns) {Configure:104(18206786ns) {ProcessConfigured:135(1540903ns) {StartInitialHandshake:157(1539128ns) {DropHandshakes:470(1206ns) {DropIncomingHandshake:442(350ns) DropOutgoingHandshake:457(258ns)} GenerateSessionVirtualId:988(665ns) PrepareNewSessionHandshake:477(1391ns) {SwitchToState:179(523ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.445038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789112099344273:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:783
2025-11-28T14:51:56.440637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789112966343805:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16398427ns) {Configure:104(16390824ns) {ProcessConfigured:135(1443232ns) {StartInitialHandshake:157(1248519ns) {DropHandshakes:470(1304ns) {DropIncomingHandshake:442(382ns) DropOutgoingHandshake:457(310ns)} GenerateSessionVirtualId:988(1161ns) PrepareNewSessionHandshake:477(1211ns) {SwitchToState:179(206ns)}}}}}
2025-11-28T14:51:56.446229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789110347237171:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-11-28T14:51:56.449997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789111133367243:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(33554477ns) {Configure:104(33550437ns) {ProcessConfigured:135(2944899ns) {StartInitialHandshake:157(2941816ns) {DropHandshakes:470(2769ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(594ns) PrepareNewSessionHandshake:477(2954ns) {SwitchToState:179(1158ns)}}}}}
2025-11-28T14:51:56.453066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789110347237171:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.453219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789112378480139:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(35898090ns) {Configure:104(35893263ns) {ProcessConfigured:135(3709069ns) {StartInitialHandshake:157(3705948ns) {DropHandshakes:470(2903308ns) {DropIncomingHandshake:442(2075ns) DropOutgoingHandshake:457(258ns)} GenerateSessionVirtualId:988(1202ns) PrepareNewSessionHandshake:477(2266ns) {SwitchToState:179(1169ns)}}}}}
2025-11-28T14:51:56.452352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789112099344273:2142] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.453025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789112099344167:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.454005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789110347237159:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36122516ns) {Configure:104(36117820ns) {ProcessConfigured:135(1105451ns) {StartInitialHandshake:157(1103423ns) {DropHandshakes:470(1438ns) {DropIncomingHandshake:442(459ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(1647ns) {SwitchToState:179(596ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.454809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789111395267055:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8506
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.454872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789111395267048:2139] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.462438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789111133367254:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:56.455797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789112966343815:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48444
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.465926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789110347237091:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.454899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789111395267048:2139] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.458506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789112966343806:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30998929ns) {Configure:104(30992471ns) {ProcessConfigured:135(44359ns) {StartInitialHandshake:157(42650ns) {DropHandshakes:470(2194ns) {DropIncomingHandshake:442(566ns) DropOutgoingHandshake:457(258ns)} GenerateSessionVirtualId:988(381ns) PrepareNewSessionHandshake:477(2452ns) {SwitchToState:179(952ns)}}}}}
2025-11-28T14:51:56.467159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789110347237172:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.463240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789111395267048:2139] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:56.464283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789112378480148:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14142
2025-11-28T14:51:56.468849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789111395267045:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.466211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789112966343816:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48444
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.468729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789111133367254:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:56.468397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789110347237175:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.468896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789112099344275:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:783
2025-11-28T14:51:56.468980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789112966343815:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.467603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789112378480151:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14142
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.468876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789111395267045:2137] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:56.473193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789112099344278:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:783
2025-11-28T14:51:56.469322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789111133367171:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.468992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789110347237172:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.470146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789111395266949:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.472291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789112378480152:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14142
2025-11-28T14:51:56.472444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789112378480148:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.475940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789112378480151:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.476122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789112378480071:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.476184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789112378480072:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.485488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789112378480156:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14142
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.473969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789112099344277:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:783
2025-11-28T14:51:56.473218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789110347237092:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.508840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789112966343823:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48444
2025-11-28T14:51:56.472255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:56.473229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789110347237177:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-11-28T14:51:56.510631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789112378480152:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.474357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789112099344276:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:783
2025-11-28T14:51:56.505277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789110347237167:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.473238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789111395266949:2048] [node 2] ICP22 created new session: [1:7577789111395267059:2048]
2025-11-28T14:51:56.483039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789112099344269:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.483095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789112099344269:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:56.473767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789111395267056:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8506
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.483116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789112099344265:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.473819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789111395267059:2048] [node 2] ICS09 handshake done sender: [1:7577789111395267048:2139] self: [1:7577789111395267043:0] peer: [2:7577789112099344274:0] socket: 296 qp: -1
No RDMA context found for GID: 2025-11-28T14:51:56.527333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789110347237091:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789110347237171:2119] held: no
::ffff:127.1.0.0
2025-11-28T14:51:56.476639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789111395267058:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8506
2025-11-28T14:51:56.483160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789112099344265:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:56.527431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789112966343823:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.528117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789110347237181:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-11-28T14:51:56.517531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789112099344265:2139] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:56.528483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789112966343740:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.476741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789111395267059:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:56.528883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789110347237180:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-11-28T14:51:56.518599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789112099344263:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.485501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.490195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789111395267055:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.536455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789110347237172:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:56.518668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789112099344263:2137] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:56.537779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789111133367259:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:56.518665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789112099344273:2142] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:56.538873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789110347237171:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:56.539406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789112378480159:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14142
2025-11-28T14:51:56.491232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789111395267046:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.519835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789112099344269:2141] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:56.491281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789111395267046:2138] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:56.525371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789112099344257:2125] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.539878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789110347237092:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.501079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789111395267045:2137] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.539925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:56.525423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:56.518713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789111395266949:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(36532536ns) {HandleHandshakeStatus:329(36530615ns) {DropIncomingHandshake:442(421ns) DropOutgoingHandshake:457(614ns) SwitchToState:179(366ns) ProcessPendingSessionEvents:495(5445096ns) {ForwardSessionEventToSession:612(5393350ns) ForwardSessionEventToSession:612(9474ns) ForwardSessionEventToSession:612(2369ns) ForwardSessionEventToSession:612(3078ns) ForwardSessionEventToSession:612(4460ns) ForwardSessionEventToSession:612(4777ns) ForwardSessionEventToSession:612(3677ns) ForwardSessionEventToSession:612(1969ns) ForwardSessionEventToSession:612(2139ns) ForwardSessionEventToSession:612(4727ns)}}}
2025-11-28T14:51:56.526403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789112099344257:2125] [node 5] ICP22 created new session: [2:7577789112099344281:2125]
2025-11-28T14:51:56.526601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789112099344259:2136] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.527164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:56.527290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789112099344167:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.543501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789111133367260:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:56.527312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789112099344259:2136] [node 6] ICP22 created new session: [2:7577789112099344282:2136]
2025-11-28T14:51:56.527520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:56.543920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789111133367254:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:56.527986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789112099344282:2136] [node 6] ICS09 handshake done sender: [2:7577789112099344269:2141] self: [2:7577789112099344268:0] peer: [6:7577789111133367258:0] socket: 300 qp: -1
2025-11-28T14:51:56.518914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789111395267046:2138] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:51:56.528075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789112099344281:2125] [node 5] ICS09 handshake done sender: [2:7577789112099344265:2139] self: [2:7577789112099344262:0] peer: [5:7577789110347237179:0] socket: 299 qp: -1
::ffff:127.1.0.0
2025-11-28T14:51:56.520758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789111395267039:2119] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.528118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789112099344281:2125] [node 5] ICS10 traffic start
2025-11-28T14:51:56.520848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:56.528247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:56.540054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789110347237091:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.528289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789112099344167:2048] [node 1] ICP22 created new session: [2:7577789112099344283:2048]
2025-11-28T14:51:56.540193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:56.520899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789111395267039:2119] [node 5] ICP22 created new session: [1:7577789111395267065:2119]
2025-11-28T14:51:56.528319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789112099344282:2136] [node 6] ICS10 traffic start
2025-11-28T14:51:56.540386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789110347237091:2048] [node 1] ICP22 created new session: [5:7577789110347237183:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.528357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:56.544516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789112966343825:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48444
2025-11-28T14:51:56.526851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789111395267065:2119] [node 5] ICS09 handshake done sender: [1:7577789111395267045:2137] self: [1:7577789111395267042:0] peer: [5:7577789110347237178:0] socket: 294 qp: -1
2025-11-28T14:51:56.540387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789110347237092:2049] [node 2] ICP22 created new session: [5:7577789110347237182:2049]
2025-11-28T14:51:56.528487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789112099344283:2048] [node 1] ICS09 handshake done sender: [2:7577789112099344273:2142] self: [2:7577789112099344274:0] peer: [1:7577789111395267043:0] socket: 302 qp: -1
2025-11-28T14:51:56.540595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789110347237182:2049] [node 2] ICS09 handshake done sender: [5:7577789110347237172:2120] self: [5:7577789110347237179:0] peer: [2:7577789112099344262:0] socket: 305 qp: -1
2025-11-28T14:51:56.527174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789111395267065:2119] [node 5] ICS10 traffic start
2025-11-28T14:51:56.528793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789112099344283:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:56.527223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:56.540799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789110347237182:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:56.528916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:56.539330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789112099344263:2137] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:56.527292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789111395267040:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.540294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789112099344258:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.541028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:56.527332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:56.540597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:56.541198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789110347237183:2048] [node 1] ICS09 handshake done sender: [5:7577789110347237171:2119] self: [5:7577789110347237178:0] peer: [1:7577789111395267042:0] socket: 301 qp: -1
2025-11-28T14:51:56.527388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789111395267040:2107] [node 4] ICP22 created new session: [1:7577789111395267068:2107]
2025-11-28T14:51:56.540660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789112099344258:2107] [node 4] ICP22 created new session: [2:7577789112099344290:2107]
2025-11-28T14:51:56.528347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789111395267068:2107] [node 4] ICS09 handshake done sender: [1:7577789111395267046:2138] self: [1:7577789111395267044:0] peer: [4:7577789112378480157:0] socket: 295 qp: -1
2025-11-28T14:51:56.541580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789112099344290:2107] [node 4] ICS09 handshake done sender: [2:7577789112099344263:2137] self: [2:7577789112099344261:0] peer: [4:7577789112378480158:0] socket: 303 qp: -1
2025-11-28T14:51:56.528773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789111395267068:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:56.541480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789110347237183:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:56.541853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789112099344290:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:56.528978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:56.543161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:56.530376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789111395267049:2140] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.541617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:56.530401Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789111395267049:2140] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:56.539666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789111395267049:2140] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:56.540821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789111395266950:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.540855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:56.541469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789111395266950:2049] [node 3] ICP22 created new session: [1:7577789111395267071:2049]
2025-11-28T14:51:56.543413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789111395267071:2049] [node 3] ICS09 handshake done sender: [1:7577789111395267049:2140] self: [1:7577789111395267047:0] peer: [3:7577789112966343824:0] socket: 340 qp: -1
2025-11-28T14:51:56.543661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789111395267071:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:56.543706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:56.544452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789112378480148:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:56.545654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789112966343826:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48444
2025-11-28T14:51:56.546099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789111133367260:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.546236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789111133367171:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.546263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789112378480071:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.546284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:56.546357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789111133367171:2049] [node 2] ICP22 created new session: [6:7577789111133367261:2049]
2025-11-28T14:51:56.546357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:56.546401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789111133367261:2049] [node 2] ICS09 handshake done sender: [6:7577789111133367254:2123] self: [6:7577789111133367258:0] peer: [2:7577789112099344268:0] socket: 310 qp: -1
2025-11-28T14:51:56.546507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789112378480071:2048] [node 1] ICP22 created new session: [4:7577789112378480162:2048]
2025-11-28T14:51:56.546515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789111133367261:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:56.546550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:56.546545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789112378480162:2048] [node 1] ICS09 handshake done sender: [4:7577789112378480148:2118] self: [4:7577789112378480157:0] peer: [1:7577789111395267044:0] socket: 297 qp: -1
2025-11-28T14:51:56.546572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789112378480162:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:56.546601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:56.546929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789112966343823:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:56.547507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789112966343740:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:56.547577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:56.547647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789112378480161:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14142
2025-11-28T14:51:56.547913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789111133367246:2119] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.547963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789111133367170:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.547637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789112966343740:2048] [node 1] ICP22 created new session: [3:7577789112966343827:2048]
2025-11-28T14:51:56.548268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789111133367263:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:56.547698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789112966343827:2048] [node 1] ICS09 handshake done sender: [3:7577789112966343823:2118] self: [3:7577789112966343824:0] peer: [1:7577789111395267047:0] socket: 342 qp: -1
2025-11-28T14:51:56.568574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789111395267054:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.547748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789112966343827:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:56.568643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789111395267054:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:56.547791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:56.573336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789112378480151:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:56.573393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789112966343740:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24801385ns) {HandleHandshakeStatus:329(24798277ns) {DropIncomingHandshake:442(1257ns) DropOutgoingHandshake:457(4804ns) SwitchToState:179(1252ns) ProcessPendingSessionEvents:495(24333430ns) {ForwardSessionEventToSession:612(10353649ns) ForwardSessionEventToSession:612(98019ns) ForwardSessionEventToSession:612(3584ns) ForwardSessionEventToSession:612(3532ns) ForwardSessionEventToSession:612(7849494ns) ForwardSessionEventToSession:612(3895ns) ForwardSessionEventToSession:612(2266129ns) ForwardSessionEventToSession:612(8397ns) ForwardSessionEventToSession:612(15855ns) ForwardSessionEventToSession:612(3706314ns)}}}
2025-11-28T14:51:56.575966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789112378480072:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.576048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:56.576103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789112378480072:2049] [node 2] ICP22 created new session: [4:7577789112378480164:2049]
2025-11-28T14:51:56.576146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789112378480164:2049] [node 2] ICS09 handshake done sender: [4:7577789112378480151:2119] self: [4:7577789112378480158:0] peer: [2:7577789112099344261:0] socket: 311 qp: -1
2025-11-28T14:51:56.576183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789112378480164:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:56.576220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:56.576419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789111395267054:2141] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:56.576518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789112966343816:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.576874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789111395267041:2130] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.576962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:56.576884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789111133367265:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:56.577005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789111395267041:2130] [node 6] ICP22 created new session: [1:7577789111395267074:2130]
2025-11-28T14:51:56.577142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789111395267074:2130] [node 6] ICS09 handshake done sender: [1:7577789111395267054:2141] self: [1:7577789111395267053:0] peer: [6:7577789111133367264:0] socket: 309 qp: -1
2025-11-28T14:51:56.577192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789111395267074:2130] [node 6] ICS10 traffic start
2025-11-28T14:51:56.577242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:56.577299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789111133367260:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:56.578568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789111133367170:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.578636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:56.578688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789111133367170:2048] [node 1] ICP22 created new session: [6:7577789111133367266:2048]
2025-11-28T14:51:56.578748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789111133367266:2048] [node 1] ICS09 handshake done sender: [6:7577789111133367260:2126] self: [6:7577789111133367264:0] peer: [1:7577789111395267053:0] socket: 348 qp: -1
2025-11-28T14:51:56.578805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789111133367266:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:56.578858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.579888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789112378480166:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14142
2025-11-28T14:51:56.585217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789112378480143:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.608338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789110347237175:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.608728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789110347237160:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.609283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789112378480143:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.609821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789112378480143:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:56.610259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789112378480138:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.610330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.610424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789112378480138:2108] [node 5] ICP22 created new session: [4:7577789112378480168:2108]
2025-11-28T14:51:56.610497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789112378480168:2108] [node 5] ICS09 handshake done sender: [4:7577789112378480143:2115] self: [4:7577789112378480141:0] peer: [5:7577789110347237188:0] socket: 314 qp: -1
2025-11-28T14:51:56.610549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789112378480168:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:56.610589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:56.610565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789110347237189:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-11-28T14:51:56.611210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789110347237175:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:56.614532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789110347237160:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.614614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:56.614671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789110347237160:2108] [node 4] ICP22 created new session: [5:7577789110347237190:2108]
2025-11-28T14:51:56.614724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789110347237190:2108] [node 4] ICS09 handshake done sender: [5:7577789110347237175:2121] self: [5:7577789110347237188:0] peer: [4:7577789112378480141:0] socket: 315 qp: -1
2025-11-28T14:51:56.614762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789110347237190:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:56.614806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:56.615125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789110347237160:2108] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.618966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789112966343830:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48444
2025-11-28T14:51:56.619474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789111133367249:2121] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.648762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789112099344275:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.648935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789112099344259:2136] [node 6] ICP18 handshake (actor [2:7577789112099344275:2143]) is too old
2025-11-28T14:51:56.672239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789111133367248:2120] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.673182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789110347237177:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.720047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789112378480142:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.721652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789111395267056:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.721911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789111395267040:2107] [node 4] ICP18 handshake (actor [1:7577789111395267056:2143]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.752075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789111133367269:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:56.765859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789112966343813:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.793616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789112378480156:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.793790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789112378480073:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.794089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789112966343813:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.794696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789112966343813:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:56.794823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789112378480173:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14142
2025-11-28T14:51:56.794922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789112966343804:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.794986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:56.795038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789112966343804:2107] [node 4] ICP22 created new session: [3:7577789112966343832:2107]
2025-11-28T14:51:56.795120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789112966343832:2107] [node 4] ICS09 handshake done sender: [3:7577789112966343813:2113] self: [3:7577789112966343810:0] peer: [4:7577789112378480172:0] socket: 325 qp: -1
2025-11-28T14:51:56.795225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789112966343832:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:56.795334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:56.795392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789112378480156:2122] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.796003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789110347237193:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-11-28T14:51:56.795773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789112378480073:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.795825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:56.795873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789112378480073:2050] [node 3] ICP22 created new session: [4:7577789112378480174:2050]
2025-11-28T14:51:56.795920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789112378480174:2050] [node 3] ICS09 handshake done sender: [4:7577789112378480156:2122] self: [4:7577789112378480172:0] peer: [3:7577789112966343810:0] socket: 330 qp: -1
2025-11-28T14:51:56.795949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789112378480174:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:56.795989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:56.800096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789112966343808:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.821982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789111395267058:2144] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.822174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789111395266950:2049] [node 3] ICP18 handshake (actor [1:7577789111395267058:2144]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.824064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789111133367270:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:56.833571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789112378480146:2116] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.862216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789112966343812:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.869777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789112099344277:2145] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.886109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789112099344276:2144] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.886400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789112099344258:2107] [node 4] ICP18 handshake (actor [2:7577789112099344276:2144]) is too old
2025-11-28T14:51:56.893353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789112099344264:2138] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.896769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789112966343825:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.896977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789112966343741:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.897450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789112099344264:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.903234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789112099344264:2138] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:56.902960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789112966343837:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48444
2025-11-28T14:51:56.905114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789112966343825:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:56.909668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789110347237176:2122] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.907661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789112966343741:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.907602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789112099344168:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.907732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:56.920894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789112966343741:2049] [node 2] ICP22 created new session: [3:7577789112966343838:2049]
2025-11-28T14:51:56.928648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789112966343838:2049] [node 2] ICS09 handshake done sender: [3:7577789112966343825:2119] self: [3:7577789112966343836:0] peer: [2:7577789112099344260:0] socket: 350 qp: -1
2025-11-28T14:51:56.936812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789112966343838:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:56.936913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:56.909293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:56.918156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789112099344168:2049] [node 3] ICP22 created new session: [2:7577789112099344294:2049]
2025-11-28T14:51:56.936430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789112099344294:2049] [node 3] ICS09 handshake done sender: [2:7577789112099344264:2138] self: [2:7577789112099344260:0] peer: [3:7577789112966343836:0] socket: 349 qp: -1
2025-11-28T14:51:56.940462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789112099344294:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:56.945884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789110347237176:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.945455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:56.946251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789112099344168:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(38536112ns) {HandleHandshakeStatus:329(38432131ns) {DropIncomingHandshake:442(1623673ns) DropOutgoingHandshake:457(1197ns) SwitchToState:179(1060ns) ProcessPendingSessionEvents:495(391106ns) {ForwardSessionEventToSession:612(329341ns) ForwardSessionEventToSession:612(9917ns) ForwardSessionEventToSession:612(4125ns) ForwardSessionEventToSession:612(2433ns) ForwardSessionEventToSession:612(5776ns) ForwardSessionEventToSession:612(2737ns) ForwardSessionEventToSession:612(8819ns) ForwardSessionEventToSession:612(4050ns) ForwardSessionEventToSession:612(2834ns) ForwardSessionEventToSession:612(7224ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:56.945396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789111133367263:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.946569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789110347237176:2122] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:56.945594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789111133367232:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:56.946947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789111133367272:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:56.947538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789112966343741:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(39288069ns) {HandleHandshakeStatus:329(39039427ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(6547ns) SwitchToState:179(857ns) ProcessPendingSessionEvents:495(901432ns) {ForwardSessionEventToSession:612(58969ns) ForwardSessionEventToSession:612(7682ns) ForwardSessionEventToSession:612(3511ns) ForwardSessionEventToSession:612(2800ns) ForwardSessionEventToSession:612(8215ns) ForwardSessionEventToSession:612(3191ns) ForwardSessionEventToSession:612(4354ns) ForwardSessionEventToSession:612(528004ns) ForwardSessionEventToSession:612(8919ns) ForwardSessionEventToSession:612(262322ns)}}}
2025-11-28T14:51:56.947915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789110347237159:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:56.948001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:56.948406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789111133367263:2127] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:56.949296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789110347237159:2109] [node 6] ICP22 created new session: [5:7577789110347237195:2109]
2025-11-28T14:51:56.950236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789110347237195:2109] [node 6] ICS09 handshake done sender: [5:7577789110347237176:2122] self: [5:7577789110347237174:0] peer: [6:7577789111133367271:0] socket: 354 qp: -1
2025-11-28T14:51:56.950353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789110347237195:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:56.954080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:56.949110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789111133367232:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789111133367263:2127] held: no
2025-11-28T14:51:56.949800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789111133367232:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:56.953179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789111133367257:2124] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:56.954203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:56.954548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789111133367232:2107] [node 5] ICP22 created new session: [6:7577789111133367273:2107]
2025-11-28T14:51:56.955377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789111133367273:2107] [node 5] ICS09 handshake done sender: [6:7577789111133367263:2127] self: [6:7577789111133367271:0] peer: [5:7577789110347237174:0] socket: 355 qp: -1
2025-11-28T14:51:56.955418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789111133367273:2107] [node 5] ICS10 traffic start
2025-11-28T14:51:56.955476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:56.956587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789112378480166:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.973294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789112378480147:2117] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:57.002365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789112966343830:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:56.994036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789112378480154:2121] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:57.003361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789112966343804:2107] [node 4] ICP18 handshake (actor [3:7577789112966343830:2121]) is too old
2025-11-28T14:51:57.008505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789112966343814:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:57.009719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789112378480154:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:57.010285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789112378480154:2121] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:57.005666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789111133367269:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:57.006137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789111133367243:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:57.017790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789110347237193:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:57.018026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789110347237093:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:57.029135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789112966343814:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.029983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789114642204494:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.029296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789111133367243:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15927) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789111133367269:2129] held: no
2025-11-28T14:51:57.036764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789115428334572:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:57.012734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789112378480139:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:57.012826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:57.028564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789112378480139:2109] [node 6] ICP22 created new session: [4:7577789116673447474:2109]
2025-11-28T14:51:57.028618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789116673447474:2109] [node 6] ICS09 handshake done sender: [4:7577789112378480154:2121] self: [4:7577789112378480153:0] peer: [6:7577789115428334571:0] socket: 318 qp: -1
2025-11-28T14:51:57.028684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789116673447474:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:57.028770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:57.036768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789112378480139:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(25898982ns) {HandleHandshakeStatus:329(25895020ns) {DropIncomingHandshake:442(23247ns) DropOutgoingHandshake:457(433ns) SwitchToState:179(366ns) ProcessPendingSessionEvents:495(7601552ns) {ForwardSessionEventToSession:612(67150ns) ForwardSessionEventToSession:612(2460ns) ForwardSessionEventToSession:612(5807ns) ForwardSessionEventToSession:612(3291ns) ForwardSessionEventToSession:612(5463ns) ForwardSessionEventToSession:612(3368ns) ForwardSessionEventToSession:612(3427ns) ForwardSessionEventToSession:612(4996ns) ForwardSessionEventToSession:612(7455142ns) ForwardSessionEventToSession:612(30671ns)}}}
2025-11-28T14:51:57.037798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789110347237193:2127] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:57.048645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789111133367269:2129] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:57.037363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789112966343814:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:57.063319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789110347237093:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:57.057591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789112966343805:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:57.060918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:57.061069Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789112966343821:2117] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:57.064578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789112966343805:2108] [node 5] ICP22 created new session: [3:7577789117261311137:2108]
2025-11-28T14:51:57.065588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789111133367243:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:57.067012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789117261311137:2108] [node 5] ICS09 handshake done sender: [3:7577789112966343814:2114] self: [3:7577789112966343811:0] peer: [5:7577789114642204493:0] socket: 362 qp: -1
2025-11-28T14:51:57.063452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:57.071579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789117261311137:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:57.071852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:57.065588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789110347237093:2050] [node 3] ICP22 created new session: [5:7577789114642204495:2050]
2025-11-28T14:51:57.067077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789114642204495:2050] [node 3] ICS09 handshake done sender: [5:7577789110347237193:2127] self: [5:7577789114642204493:0] peer: [3:7577789112966343811:0] socket: 363 qp: -1
2025-11-28T14:51:57.067117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789114642204495:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:57.067170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:57.065695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:57.065771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789111133367243:2109] [node 4] ICP22 created new session: [6:7577789115428334573:2109]
2025-11-28T14:51:57.071748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789115428334573:2109] [node 4] ICS09 handshake done sender: [6:7577789111133367269:2129] self: [6:7577789115428334571:0] peer: [4:7577789112378480153:0] socket: 322 qp: -1
2025-11-28T14:51:57.072127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789115428334573:2109] [node 4] ICS10 traffic start
2025-11-28T14:51:57.072196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:57.073432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789112966343805:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24620592ns) {HandleHandshakeStatus:329(24618258ns) {DropIncomingHandshake:442(25110ns) DropOutgoingHandshake:457(539ns) SwitchToState:179(5502ns) ProcessPendingSessionEvents:495(118082ns) {ForwardSessionEventToSession:612(49947ns) ForwardSessionEventToSession:612(20245ns) ForwardSessionEventToSession:612(5673ns) ForwardSessionEventToSession:612(2640ns) ForwardSessionEventToSession:612(2253ns) ForwardSessionEventToSession:612(8562ns) ForwardSessionEventToSession:612(2756ns) ForwardSessionEventToSession:612(2594ns) ForwardSessionEventToSession:612(3750ns) ForwardSessionEventToSession:612(5309ns)}}}
2025-11-28T14:51:57.073561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789112966343821:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:57.073038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789110347237093:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(24156468ns) {HandleHandshakeStatus:329(24154099ns) {DropIncomingHandshake:442(3625ns) DropOutgoingHandshake:457(19166ns) SwitchToState:179(1127ns) ProcessPendingSessionEvents:495(967721ns) {ForwardSessionEventToSession:612(910552ns) ForwardSessionEventToSession:612(5283ns) ForwardSessionEventToSession:612(7334ns) ForwardSessionEventToSession:612(2990ns) ForwardSessionEventToSession:612(12621ns) ForwardSessionEventToSession:612(3702ns) ForwardSessionEventToSession:612(3109ns) ForwardSessionEventToSession:612(6228ns) ForwardSessionEventToSession:612(2966ns) ForwardSessionEventToSession:612(2975ns)}}}
2025-11-28T14:51:57.073068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789111133367270:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:57.073270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789111133367172:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:57.074186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789112966343821:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.074467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789112966343806:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:57.074474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789115428334576:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15110
2025-11-28T14:51:57.074527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:57.074588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789112966343806:2109] [node 6] ICP22 created new session: [3:7577789117261311140:2109]
2025-11-28T14:51:57.074655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789117261311140:2109] [node 6] ICS09 handshake done sender: [3:7577789112966343821:2117] self: [3:7577789112966343820:0] peer: [6:7577789115428334575:0] socket: 364 qp: -1
2025-11-28T14:51:57.074689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789117261311140:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:57.075284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789111133367270:2130] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:57.074719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:57.075601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789111133367172:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15549) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789111133367270:2130] held: no
2025-11-28T14:51:57.076123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789111133367172:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:57.076192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:57.076239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789111133367172:2050] [node 3] ICP22 created new session: [6:7577789115428334577:2050]
2025-11-28T14:51:57.076450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789115428334577:2050] [node 3] ICS09 handshake done sender: [6:7577789111133367270:2130] self: [6:7577789115428334575:0] peer: [3:7577789112966343820:0] socket: 327 qp: -1
2025-11-28T14:51:57.076491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789115428334577:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:57.076535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:57.152542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:57.152824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789112099344283:2048] [node 1] ICS07 socket disconnect 302 reason# EndOfStream
2025-11-28T14:51:57.152857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789112099344283:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.152990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:57.153198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789112099344283:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:57.152570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:57.152834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789112966343827:2048] [node 1] ICS07 socket disconnect 342 reason# EndOfStream
2025-11-28T14:51:57.152869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789112966343827:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.152960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:57.153004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789112966343827:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:57.158294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789112966343740:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:57.158345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789112966343827:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:57.159694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789112099344167:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:57.159744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789112099344283:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:57.162101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:57.162519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:57.162408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789110347237183:2048] [node 1] ICS01 socket: 301 reason# EndOfStream
2025-11-28T14:51:57.168521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789110347237091:2048] [node 1] ICP30 unregister session Session# [5:7577789110347237183:2048] VirtualId# [5:7577789110347237178:0]
2025-11-28T14:51:57.168597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789110347237183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.168766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:57.168818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:57.169827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:57.168901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789112378480162:2048] [node 1] ICS07 socket disconnect 297 reason# EndOfStream
2025-11-28T14:51:57.168959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789112378480162:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.169073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:57.169136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789112378480162:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:57.187585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789111133367266:2048] [node 1] ICS07 socket disconnect 348 reason# EndOfStream
2025-11-28T14:51:57.171036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789112966343740:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:57.189536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789111133367266:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.200525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789112378480071:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:57.191616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789112966343827:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:57.192471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789112966343740:2048] [node 1] ICP30 unregister session Session# [3:7577789112966343827:2048] VirtualId# [3:7577789112966343824:0]
2025-11-28T14:51:57.189686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:57.192543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:57.196128Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789112966343740:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.199356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.199700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [3:7577789112966343740:2048] [node 1] ICP35 event processing took too much time StateWork:294(29389389ns) {HandleHandshakeStatus:395(29384661ns) {LogHandshakeFail:472(976824ns) {UpdateErrorStateLog:545(2933ns)} DropOutgoingHandshake:457(971ns) ProcessPendingSessionEvents:495(593ns) UnregisterSession:538(856570ns) {ProcessPendingSessionEvents:495(797ns) SwitchToInitialState:202(2243ns) {SwitchToState:179(1069ns)}} TransitToErrorState:752(6830891ns) {SwitchToState:179(8640ns) ProcessPendingSessionEvents:495(515ns) DropHandshakes:470(9212ns) {DropIncomingHandshake:442(1133ns) DropOutgoingHandshake:457(623ns)}}}}
2025-11-28T14:51:57.205264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789112378480162:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:57.205411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789112378480071:2048] [node 1] ICP30 unregister session Session# [4:7577789112378480162:2048] VirtualId# [4:7577789112378480157:0]
2025-11-28T14:51:57.205471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:57.209383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789112378480071:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.209412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.189742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789111133367266:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:57.208930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789111133367170:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:57.208996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789111133367266:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:57.209034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789111133367170:2048] [node 1] ICP30 unregister session Session# [6:7577789111133367266:2048] VirtualId# [6:7577789111133367264:0]
2025-11-28T14:51:57.209090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:57.209133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789111133367170:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.209170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.210225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789112966343740:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:57.218799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789111133367170:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:57.220254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789112378480071:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:57.223199Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:57.223433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789112966343838:2049] [node 2] ICS07 socket disconnect 350 reason# EndOfStream
2025-11-28T14:51:57.223475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789112966343838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.223569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:57.223609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789112966343838:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:57.227765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789112966343741:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:57.227819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789112966343838:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:57.232908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789112966343741:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:57.233013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789112966343838:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:57.233059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789112966343741:2049] [node 2] ICP30 unregister session Session# [3:7577789112966343838:2049] VirtualId# [3:7577789112966343836:0]
2025-11-28T14:51:57.233135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:57.233182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789112966343741:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.233200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.232539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:57.232799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789112378480164:2049] [node 2] ICS07 socket disconnect 311 reason# EndOfStream
2025-11-28T14:51:57.232836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789112378480164:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.232924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:57.232971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789112378480164:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:57.233562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:57.233752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789110347237182:2049] [node 2] ICS07 socket disconnect 305 reason# EndOfStream
2025-11-28T14:51:57.233786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789110347237182:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.233876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:57.233920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789110347237182:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:57.235273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789110347237092:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:57.235350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789110347237182:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:57.235391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789110347237092:2049] [node 2] ICP30 unregister session Session# [5:7577789110347237182:2049] VirtualId# [5:7577789110347237179:0]
2025-11-28T14:51:57.235439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:57.235473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789110347237092:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.235488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.236435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789112378480072:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:57.236488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789112378480164:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:57.236525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789112378480072:2049] [node 2] ICP30 unregister session Session# [4:7577789112378480164:2049] VirtualId# [4:7577789112378480158:0]
2025-11-28T14:51:57.236563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:57.236588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789112378480072:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.236619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.240535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:57.240768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789111133367261:2049] [node 2] ICS07 socket disconnect 310 reason# EndOfStream
2025-11-28T14:51:57.240796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789111133367261:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.240888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:57.240932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789111133367261:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:57.242061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789111133367171:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:57.242121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789111133367261:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:57.242150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789111133367171:2049] [node 2] ICP30 unregister session Session# [6:7577789111133367261:2049] VirtualId# [6:7577789111133367258:0]
2025-11-28T14:51:57.242193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:57.242223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789111133367171:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.242234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.252651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789111133367171:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:57.248567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789110347237092:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:57.252951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789112378480072:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:57.267676Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:57.267910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789114642204495:2050] [node 3] ICS01 socket: 363 reason# EndOfStream
2025-11-28T14:51:57.267956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789110347237093:2050] [node 3] ICP30 unregister session Session# [5:7577789114642204495:2050] VirtualId# [5:7577789114642204493:0]
2025-11-28T14:51:57.267991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789114642204495:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.268084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:57.268134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:57.269044Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:57.269261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789115428334577:2050] [node 3] ICS01 socket: 327 reason# EndOfStream
2025-11-28T14:51:57.269303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789111133367172:2050] [node 3] ICP30 unregister session Session# [6:7577789115428334577:2050] VirtualId# [6:7577789115428334575:0]
2025-11-28T14:51:57.267861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:57.269337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789115428334577:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.269422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:57.269479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:57.268118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789112378480174:2050] [node 3] ICS07 socket disconnect 330 reason# EndOfStream
2025-11-28T14:51:57.268152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789112378480174:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.268287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:57.268337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789112378480174:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:57.269657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789112378480073:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15549) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:57.269714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789112378480174:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:57.269766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789112378480073:2050] [node 3] ICP30 unregister session Session# [4:7577789112378480174:2050] VirtualId# [4:7577789112378480172:0]
2025-11-28T14:51:57.269815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:57.269870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789112378480073:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15549) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.269889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15549) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:57.310122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:57.310377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789110347237190:2108] [node 4] ICS01 socket: 315 reason# EndOfStream
2025-11-28T14:51:57.310427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789110347237160:2108] [node 4] ICP30 unregister session Session# [5:7577789110347237190:2108] VirtualId# [5:7577789110347237188:0]
2025-11-28T14:51:57.310480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789110347237190:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.310603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:57.310659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:57.312554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:57.312834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789115428334573:2109] [node 4] ICS01 socket: 322 reason# EndOfStream
2025-11-28T14:51:57.312884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789111133367243:2109] [node 4] ICP30 unregister session Session# [6:7577789115428334573:2109] VirtualId# [6:7577789115428334571:0]
2025-11-28T14:51:57.312926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789115428334573:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.313054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:57.313109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:57.375176Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:57.375429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789111133367273:2107] [node 5] ICS01 socket: 355 reason# EndOfStream
2025-11-28T14:51:57.375481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789111133367232:2107] [node 5] ICP30 unregister session Session# [6:7577789111133367273:2107] VirtualId# [6:7577789111133367271:0]
2025-11-28T14:51:57.375515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789111133367273:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:57.375611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:57.375669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:57.439751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789115487185915:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:57.439835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789115487185916:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:57.469321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789115051921861:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:57.469365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789115051921862:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:57.499424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789117571128989:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:57.499469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789117571128990:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:57.532845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789115044340069:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:57.532975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789115044340071:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:57.533013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789115044340070:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:57.561070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789117368798186:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:57.561104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789117368798187:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:57.561120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789117368798188:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:57.588163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789117631372469:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:57.588208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789117631372470:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:57.588241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789117631372471:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:57.684947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789117631372531:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:57.703508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789115051921923:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:57.737296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789115487185980:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.774362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789115487185981:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:57.818854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789115044340153:2111] [node 6] ICP01 ready to work
2025-11-28T14:51:57.826265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789115044340150:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45649
2025-11-28T14:51:57.821905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789115051921926:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:57.828651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789117571129053:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:57.824554Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789117631372531:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18931219ns) {Configure:104(18926565ns) {ProcessConfigured:135(15603109ns) {StartInitialHandshake:157(15599688ns) {DropHandshakes:470(3056ns) {DropIncomingHandshake:442(649ns) DropOutgoingHandshake:457(809ns)} GenerateSessionVirtualId:988(744ns) PrepareNewSessionHandshake:477(3146ns) {SwitchToState:179(1481ns)}}}}}
2025-11-28T14:51:57.805786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789115487185915:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29112013ns) {Configure:104(29108451ns) {ProcessConfigured:135(10106972ns) {StartInitialHandshake:157(10104160ns) {DropHandshakes:470(2510ns) {DropIncomingHandshake:442(684ns) DropOutgoingHandshake:457(472ns)} GenerateSessionVirtualId:988(698ns) PrepareNewSessionHandshake:477(5087ns) {SwitchToState:179(3301ns)}}}}}
2025-11-28T14:51:57.822319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789115051921925:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:57.857869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789115044340070:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(120916592ns) {Configure:104(120910294ns) {ProcessConfigured:135(105293ns) {StartInitialHandshake:157(101914ns) {DropHandshakes:470(2652ns) {DropIncomingHandshake:442(709ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(904ns) PrepareNewSessionHandshake:477(1653ns) {SwitchToState:179(431ns)}}}}}
2025-11-28T14:51:57.832454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789117631372535:2110] [node 4] ICP01 ready to work
2025-11-28T14:51:57.829349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789117571129052:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:57.782013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789115487185982:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:57.852774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789117368798266:2114] [node 6] ICP01 ready to work
2025-11-28T14:51:57.854640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789117368798267:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:57.858268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789115044340157:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:57.847451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789117571129054:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.914147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789117571129054:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33407455ns) {Configure:104(33399502ns) {ProcessConfigured:135(79591ns) {StartInitialHandshake:157(75846ns) {DropHandshakes:470(5996ns) {DropIncomingHandshake:442(923ns) DropOutgoingHandshake:457(732ns)} GenerateSessionVirtualId:988(1216ns) PrepareNewSessionHandshake:477(2209ns) {SwitchToState:179(786ns)}}}}}
2025-11-28T14:51:57.908467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789117631372469:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(56386801ns) {Configure:104(56383035ns) {ProcessConfigured:135(82250ns) {StartInitialHandshake:157(79811ns) {DropHandshakes:470(3001ns) {DropIncomingHandshake:442(656ns) DropOutgoingHandshake:457(614ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:477(1708ns) {SwitchToState:179(621ns)}}}}}
2025-11-28T14:51:57.924416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789117631372538:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.923699Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789117571128990:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(143797887ns) {Configure:104(143785231ns) {ProcessConfigured:135(90374ns) {StartInitialHandshake:157(87200ns) {DropHandshakes:470(3164ns) {DropIncomingHandshake:442(829ns) DropOutgoingHandshake:457(515ns)} GenerateSessionVirtualId:988(1166ns) PrepareNewSessionHandshake:477(2546ns) {SwitchToState:179(1323ns)}}}}}
2025-11-28T14:51:57.930482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789115044340150:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.942413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789117368798186:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(159823096ns) {Configure:104(159815462ns) {ProcessConfigured:135(77321ns) {StartInitialHandshake:157(74698ns) {DropHandshakes:470(2506ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(480ns)} GenerateSessionVirtualId:988(654ns) PrepareNewSessionHandshake:477(2095ns) {SwitchToState:179(845ns)}}}}}
2025-11-28T14:51:57.942867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789117368798270:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36647
2025-11-28T14:51:57.938317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789117571128989:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(163519752ns) {Configure:104(163511744ns) {ProcessConfigured:135(98336ns) {StartInitialHandshake:157(95443ns) {DropHandshakes:470(2935ns) {DropIncomingHandshake:442(965ns) DropOutgoingHandshake:457(402ns)} GenerateSessionVirtualId:988(1336ns) PrepareNewSessionHandshake:477(2114ns) {SwitchToState:179(1129ns)}}}}}
2025-11-28T14:51:57.938863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789117631372539:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
2025-11-28T14:51:57.943903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789115044340069:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(201423921ns) {Configure:104(201418431ns) {ProcessConfigured:135(19092081ns) {StartInitialHandshake:157(19087842ns) {DropHandshakes:470(4174ns) {DropIncomingHandshake:442(1013ns) DropOutgoingHandshake:457(812ns)} GenerateSessionVirtualId:988(1560ns) PrepareNewSessionHandshake:477(3666ns) {SwitchToState:179(1850ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.963558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789115487185993:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30072
2025-11-28T14:51:57.957006Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789115051921923:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(247289222ns) {Configure:104(247279712ns) {ProcessConfigured:135(139349ns) {StartInitialHandshake:157(136372ns) {DropHandshakes:470(2441ns) {DropIncomingHandshake:442(605ns) DropOutgoingHandshake:457(444ns)} GenerateSessionVirtualId:988(957ns) PrepareNewSessionHandshake:477(1770ns) {SwitchToState:179(759ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.976765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789115487185981:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(163093820ns) {Configure:104(163084313ns) {ProcessConfigured:135(15645407ns) {StartInitialHandshake:157(15640893ns) {DropHandshakes:470(3381ns) {DropIncomingHandshake:442(890ns) DropOutgoingHandshake:457(519ns)} GenerateSessionVirtualId:988(783ns) PrepareNewSessionHandshake:477(6055ns) {SwitchToState:179(3132ns)}}}}}
2025-11-28T14:51:57.948256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789117571129059:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-11-28T14:51:57.957388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789115044340069:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:57.941907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789117631372538:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:57.984407Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789117368798266:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(102334583ns) {Configure:104(102329051ns) {ProcessConfigured:135(65054ns) {StartInitialHandshake:157(62407ns) {DropHandshakes:470(1501ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(835ns) PrepareNewSessionHandshake:477(1487ns) {SwitchToState:179(432ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.985243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789117368798270:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:57.979123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789117571129059:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.980708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789115487185916:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(248141593ns) {Configure:104(248133774ns) {ProcessConfigured:135(12056400ns) {StartInitialHandshake:157(12050305ns) {DropHandshakes:470(3110ns) {DropIncomingHandshake:442(735ns) DropOutgoingHandshake:457(526ns)} GenerateSessionVirtualId:988(1694ns) PrepareNewSessionHandshake:477(4629ns) {SwitchToState:179(2166ns)}}}}}
2025-11-28T14:51:58.000732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789117571129052:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(141486669ns) {Configure:104(141475257ns) {ProcessConfigured:135(17968413ns) {StartInitialHandshake:157(17966155ns) {DropHandshakes:470(2545ns) {DropIncomingHandshake:442(650ns) DropOutgoingHandshake:457(531ns)} GenerateSessionVirtualId:988(570ns) PrepareNewSessionHandshake:477(3122ns) {SwitchToState:179(1352ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.959113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789117631372539:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.012940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789117571129053:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(168467114ns) {Configure:104(168455664ns) {ProcessConfigured:135(10628959ns) {StartInitialHandshake:157(10624886ns) {DropHandshakes:470(1594ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(425ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:477(4828ns) {SwitchToState:179(3125ns)}}}}}
2025-11-28T14:51:58.001358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789117368798187:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(218729396ns) {Configure:104(218719226ns) {ProcessConfigured:135(18553565ns) {StartInitialHandshake:157(18551685ns) {DropHandshakes:470(535ns) {DropIncomingHandshake:442(156ns) DropOutgoingHandshake:457(39ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(4203ns) {SwitchToState:179(99ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:57.998191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789115051921925:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(111657203ns) {Configure:104(111651684ns) {ProcessConfigured:135(83254ns) {StartInitialHandshake:157(80090ns) {DropHandshakes:470(3499ns) {DropIncomingHandshake:442(239ns) DropOutgoingHandshake:457(102ns)} GenerateSessionVirtualId:988(542ns) PrepareNewSessionHandshake:477(1357ns) {SwitchToState:179(554ns)}}}}}
2025-11-28T14:51:57.992215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789115487185993:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.045995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789119339307463:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45649
2025-11-28T14:51:57.977256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789117631372542:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
2025-11-28T14:51:58.037044Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789117368798267:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(154220151ns) {Configure:104(154212240ns) {ProcessConfigured:135(11345851ns) {StartInitialHandshake:157(11343139ns) {DropHandshakes:470(2287ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(738ns)} GenerateSessionVirtualId:988(690ns) PrepareNewSessionHandshake:477(3004ns) {SwitchToState:179(1391ns)}}}}}
2025-11-28T14:51:58.003831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789115051921927:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58738
2025-11-28T14:51:58.002453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789115487185984:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.059852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789115044340157:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(136958396ns) {Configure:104(136948005ns) {ProcessConfigured:135(28870115ns) {StartInitialHandshake:157(28867631ns) {DropHandshakes:470(1597ns) {DropIncomingHandshake:442(499ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(879ns) PrepareNewSessionHandshake:477(2650ns) {SwitchToState:179(1576ns)}}}}}
2025-11-28T14:51:58.052564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789117368798188:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(186251521ns) {Configure:104(175297098ns) {ProcessConfigured:135(27715252ns) {StartInitialHandshake:157(10718811ns) {DropHandshakes:470(3106ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(792ns)} GenerateSessionVirtualId:988(1212ns) PrepareNewSessionHandshake:477(3694ns) {SwitchToState:179(201ns)}}}}}
2025-11-28T14:51:58.002492Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789115487185984:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:57.980428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789117631372469:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.004029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789115051921935:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58738
2025-11-28T14:51:57.989168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789117631372470:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(166931994ns) {Configure:104(166925950ns) {ProcessConfigured:135(8292113ns) {StartInitialHandshake:157(8289343ns) {DropHandshakes:470(2856ns) {DropIncomingHandshake:442(599ns) DropOutgoingHandshake:457(580ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(3248ns) {SwitchToState:179(1524ns)}}}}}
2025-11-28T14:51:58.004144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789115051921936:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58738
2025-11-28T14:51:58.022849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789115487185991:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30072
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.087775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789121866096366:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-11-28T14:51:58.004267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789115051921937:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58738
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.000411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789117631372545:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
2025-11-28T14:51:58.033947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789115487185988:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.078745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789115044340150:2128] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:58.008747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789115051921861:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(258485174ns) {Configure:104(258478225ns) {ProcessConfigured:135(10695991ns) {StartInitialHandshake:157(10692400ns) {DropHandshakes:470(3258ns) {DropIncomingHandshake:442(700ns) DropOutgoingHandshake:457(620ns)} GenerateSessionVirtualId:988(914ns) PrepareNewSessionHandshake:477(3385ns) {SwitchToState:179(1395ns)}}}}}
2025-11-28T14:51:58.084669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789115044340155:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.044008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789117631372471:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(178959894ns) {Configure:104(178954076ns) {ProcessConfigured:135(8405482ns) {StartInitialHandshake:157(8402708ns) {DropHandshakes:470(2782ns) {DropIncomingHandshake:442(737ns) DropOutgoingHandshake:457(699ns)} GenerateSessionVirtualId:988(728ns) PrepareNewSessionHandshake:477(3030ns) {SwitchToState:179(1352ns)}}}}}
2025-11-28T14:51:58.013494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789115051921926:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(165696956ns) {Configure:104(165690557ns) {ProcessConfigured:135(10508456ns) {StartInitialHandshake:157(10505985ns) {DropHandshakes:470(1030ns) {DropIncomingHandshake:442(325ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(2259ns) {SwitchToState:179(1085ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.043718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789115051921862:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(242031735ns) {Configure:104(242021910ns) {ProcessConfigured:135(32952112ns) {StartInitialHandshake:157(32949036ns) {DropHandshakes:470(3273ns) {DropIncomingHandshake:442(1096ns) DropOutgoingHandshake:457(598ns)} GenerateSessionVirtualId:988(938ns) PrepareNewSessionHandshake:477(1489ns) {SwitchToState:179(167ns)}}}}}
2025-11-28T14:51:58.117498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789121866096366:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.121226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789117571128989:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.101444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789121663765579:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36647
2025-11-28T14:51:58.033995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789115487185988:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.052815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789117631372535:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(180527266ns) {Configure:104(180519102ns) {ProcessConfigured:135(24899109ns) {StartInitialHandshake:157(24896005ns) {DropHandshakes:470(14233103ns) {DropIncomingHandshake:442(2689ns) DropOutgoingHandshake:457(516ns)} GenerateSessionVirtualId:988(4326ns) PrepareNewSessionHandshake:477(5298ns) {SwitchToState:179(3358ns)}}}}}
2025-11-28T14:51:58.084707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789115044340155:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:58.044134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789119346889236:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58738
2025-11-28T14:51:58.052563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789115487185984:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:58.087981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789115044340069:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.044222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789115051921935:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.059977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789117631372535:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.119492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789121663765581:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36647
2025-11-28T14:51:58.088057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:58.088114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789115044340069:2048] [node 1] ICP22 created new session: [4:7577789119339307465:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.141326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789121866096368:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-11-28T14:51:58.088194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789119339307465:2048] [node 1] ICS09 handshake done sender: [4:7577789115044340150:2128] self: [4:7577789115044340164:0] peer: [1:7577789115487185983:0] socket: 302 qp: -1
2025-11-28T14:51:58.049524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789115051921936:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.073319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789115487185996:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30072
2025-11-28T14:51:58.088251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789119339307465:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:58.088294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:58.114321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789115044340155:2130] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:58.120089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789115044340069:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31834918ns) {HandleHandshakeStatus:329(31831174ns) {DropIncomingHandshake:442(1183ns) DropOutgoingHandshake:457(9254ns) SwitchToState:179(734ns) ProcessPendingSessionEvents:495(5842714ns) {ForwardSessionEventToSession:612(5776317ns) ForwardSessionEventToSession:612(7534ns) ForwardSessionEventToSession:612(14795ns) ForwardSessionEventToSession:612(3458ns) ForwardSessionEventToSession:612(3274ns) ForwardSessionEventToSession:612(7052ns) ForwardSessionEventToSession:612(3095ns) ForwardSessionEventToSession:612(2470ns) ForwardSessionEventToSession:612(6748ns) ForwardSessionEventToSession:612(6169ns)}}}
2025-11-28T14:51:58.124732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789119339307464:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45649
2025-11-28T14:51:58.130940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789115044340153:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.140850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:58.084870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789117631372546:2116] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.124937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789121663765580:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36647
2025-11-28T14:51:58.141238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789121663765580:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.114923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789121926339849:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
2025-11-28T14:51:58.147326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789117368798267:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.124090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789117631372539:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.144532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789115044340153:2111] [node 6] ICP22 created new session: [4:7577789119339307469:2111]
2025-11-28T14:51:58.125629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789117631372469:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789117631372539:2113] held: no
2025-11-28T14:51:58.144611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789119339307469:2111] [node 6] ICS09 handshake done sender: [4:7577789115044340155:2130] self: [4:7577789115044340154:0] peer: [6:7577789121926339848:0] socket: 304 qp: -1
2025-11-28T14:51:58.144675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789119339307469:2111] [node 6] ICS10 traffic start
2025-11-28T14:51:58.144740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:58.146548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789115044340153:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(20741919ns) {HandleHandshakeStatus:329(20739744ns) {DropIncomingHandshake:442(1867ns) DropOutgoingHandshake:457(965ns) SwitchToState:179(830ns) ProcessPendingSessionEvents:495(129128ns) {ForwardSessionEventToSession:612(74054ns) ForwardSessionEventToSession:612(6249ns) ForwardSessionEventToSession:612(3327ns) ForwardSessionEventToSession:612(3023ns) ForwardSessionEventToSession:612(5208ns) ForwardSessionEventToSession:612(2732ns) ForwardSessionEventToSession:612(5419ns) ForwardSessionEventToSession:612(3077ns) ForwardSessionEventToSession:612(7053ns) ForwardSessionEventToSession:612(5352ns)}}}
2025-11-28T14:51:58.147447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789117631372469:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18210793ns) {HandleHandshakeStatus:395(18207728ns) {LogHandshakeFail:472(1442951ns) {UpdateErrorStateLog:545(8281ns)} DropOutgoingHandshake:457(3044ns) ProcessPendingSessionEvents:495(13585357ns) {DropSessionEvent:512(13271ns) DropSessionEvent:512(27291ns) DropSessionEvent:512(5685ns) DropSessionEvent:512(5409ns) DropSessionEvent:512(4290ns) DropSessionEvent:512(5075ns) DropSessionEvent:512(6077ns) DropSessionEvent:512(4510ns) DropSessionEvent:512(5527ns) DropSessionEvent:512(344791ns)}}}
2025-11-28T14:51:58.148162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789117631372469:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.063062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789115051921927:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.148202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:58.148237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789117631372469:2048] [node 1] ICP22 created new session: [6:7577789121926339852:2048]
2025-11-28T14:51:58.072090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789115051921861:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.089534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789119346889244:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58738
2025-11-28T14:51:58.139799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789115051921937:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.143715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789115051921927:2110] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:58.146170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789115051921861:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.146234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:58.146283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789115051921861:2048] [node 1] ICP22 created new session: [2:7577789119346889245:2048]
2025-11-28T14:51:58.146331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789119346889245:2048] [node 1] ICS09 handshake done sender: [2:7577789115051921927:2110] self: [2:7577789119346889243:0] peer: [1:7577789115487185978:0] socket: 303 qp: -1
2025-11-28T14:51:58.146363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789119346889245:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:58.146401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.150975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789121926339852:2048] [node 1] ICS09 handshake done sender: [6:7577789117631372539:2113] self: [6:7577789117631372543:0] peer: [1:7577789115487185987:0] socket: 310 qp: -1
2025-11-28T14:51:58.150984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789121926339850:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
2025-11-28T14:51:58.151028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789121926339852:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:58.151066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:58.151646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789121866096369:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-11-28T14:51:58.074609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789115487185988:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:58.151793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789119339307462:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.080561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789115487185996:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.152214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789117631372538:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:58.151840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789119339307462:2133] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:58.163694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789121663765583:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36647
2025-11-28T14:51:58.171286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789117368798274:2129] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.080694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789115487185979:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.152283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789121866096368:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.157058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789117631372535:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.157110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:58.157145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789117631372535:2110] [node 4] ICP22 created new session: [6:7577789121926339854:2110]
2025-11-28T14:51:58.183482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789121926339854:2110] [node 4] ICS09 handshake done sender: [6:7577789117631372538:2112] self: [6:7577789121926339848:0] peer: [4:7577789115044340154:0] socket: 309 qp: -1
2025-11-28T14:51:58.157004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789119339307462:2133] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:58.184468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789121926339854:2110] [node 4] ICS10 traffic start
2025-11-28T14:51:58.186767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:58.186909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789117631372545:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.178354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789121866096366:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:58.080733Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789115487185979:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:58.183445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789115044340157:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.186749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789121663765580:2134] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:58.186051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789117571128989:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.188733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789117631372531:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.085504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789115487185980:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.188253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789117368798267:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.183495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.184456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789115044340157:2108] [node 5] ICP22 created new session: [4:7577789119339307472:2108]
2025-11-28T14:51:58.187179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789119339307472:2108] [node 5] ICS09 handshake done sender: [4:7577789119339307462:2133] self: [4:7577789119339307461:0] peer: [5:7577789121663765582:0] socket: 351 qp: -1
2025-11-28T14:51:58.187829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789119339307472:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:58.187894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:58.197363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789119339307474:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45649
2025-11-28T14:51:58.188297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:58.085577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:58.186109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:58.186195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789117571128989:2048] [node 1] ICP22 created new session: [3:7577789121866096371:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.189091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789117368798267:2108] [node 4] ICP22 created new session: [5:7577789121663765584:2108]
2025-11-28T14:51:58.085649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789115487185980:2107] [node 4] ICP22 created new session: [1:7577789119782153297:2107]
2025-11-28T14:51:58.204164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789121926339857:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
2025-11-28T14:51:58.186253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789121866096371:2048] [node 1] ICS09 handshake done sender: [3:7577789121866096366:2116] self: [3:7577789121866096367:0] peer: [1:7577789115487185990:0] socket: 347 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:58.186733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789121866096371:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:58.189196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789117368798274:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.187063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:58.085708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789119782153297:2107] [node 4] ICS09 handshake done sender: [1:7577789115487185984:2111] self: [1:7577789115487185983:0] peer: [4:7577789115044340164:0] socket: 301 qp: -1
2025-11-28T14:51:58.191294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789121663765584:2108] [node 4] ICS09 handshake done sender: [5:7577789121663765580:2134] self: [5:7577789121663765582:0] peer: [4:7577789119339307461:0] socket: 352 qp: -1
2025-11-28T14:51:58.190981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789117571128989:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:58.200731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789121663765584:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:58.206183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789117631372545:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:58.085750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789119782153297:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:58.200773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:58.206282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789121866096374:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-11-28T14:51:58.206535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789117631372531:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.201393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789117368798274:2129] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:58.085794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:58.101518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789119782153295:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30072
2025-11-28T14:51:58.207904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789119339307474:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.207904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789117631372535:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(49448024ns) {HandleHandshakeStatus:329(49445432ns) {DropIncomingHandshake:442(994ns) DropOutgoingHandshake:457(12400ns) SwitchToState:179(646ns) ProcessPendingSessionEvents:495(16312890ns) {ForwardSessionEventToSession:612(9088873ns) ForwardSessionEventToSession:612(2328313ns) ForwardSessionEventToSession:612(745293ns) ForwardSessionEventToSession:612(7841ns) ForwardSessionEventToSession:612(1434859ns) ForwardSessionEventToSession:612(3743ns) ForwardSessionEventToSession:612(3430ns) ForwardSessionEventToSession:612(2288850ns) ForwardSessionEventToSession:612(6456ns) ForwardSessionEventToSession:612(385285ns)}}}
2025-11-28T14:51:58.205870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789117368798266:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.210453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789115044340071:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.208028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:58.101562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789115487185979:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:58.206084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:58.211043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789117571129066:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.210155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789117631372531:2106] [node 5] ICP22 created new session: [6:7577789121926339858:2106]
2025-11-28T14:51:58.206265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789117368798266:2114] [node 6] ICP22 created new session: [5:7577789121663765588:2114]
2025-11-28T14:51:58.211088Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789117571129066:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:58.207033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789121663765588:2114] [node 6] ICS09 handshake done sender: [5:7577789117368798274:2129] self: [5:7577789117368798273:0] peer: [6:7577789121926339856:0] socket: 327 qp: -1
2025-11-28T14:51:58.210641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789121926339858:2106] [node 5] ICS09 handshake done sender: [6:7577789117631372545:2115] self: [6:7577789121926339856:0] peer: [5:7577789117368798273:0] socket: 328 qp: -1
2025-11-28T14:51:58.207191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789121663765588:2114] [node 6] ICS10 traffic start
2025-11-28T14:51:58.114212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789115487185982:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.207229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:58.114271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:58.207506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789117368798267:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19061709ns) {HandleHandshakeStatus:329(19059652ns) {DropIncomingHandshake:442(452ns) DropOutgoingHandshake:457(4632ns) SwitchToState:179(197ns) ProcessPendingSessionEvents:495(1859673ns) {ForwardSessionEventToSession:612(870527ns) ForwardSessionEventToSession:612(356387ns) ForwardSessionEventToSession:612(10507ns) ForwardSessionEventToSession:612(3905ns) ForwardSessionEventToSession:612(235464ns) ForwardSessionEventToSession:612(3419ns) ForwardSessionEventToSession:612(204255ns) ForwardSessionEventToSession:612(3420ns) ForwardSessionEventToSession:612(2855ns) ForwardSessionEventToSession:612(155277ns)}}}
2025-11-28T14:51:58.116833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789115487185915:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.210803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789121926339858:2106] [node 5] ICS10 traffic start
2025-11-28T14:51:58.210989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:58.116882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:58.116933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789115487185915:2048] [node 2] ICP22 created new session: [1:7577789119782153303:2048]
2025-11-28T14:51:58.118227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789115487185982:2109] [node 6] ICP22 created new session: [1:7577789119782153302:2109]
2025-11-28T14:51:58.121349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789119782153303:2048] [node 2] ICS09 handshake done sender: [1:7577789115487185979:2110] self: [1:7577789115487185978:0] peer: [2:7577789119346889243:0] socket: 300 qp: -1
2025-11-28T14:51:58.124798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789119782153303:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:58.124853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:58.125213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789119782153302:2109] [node 6] ICS09 handshake done sender: [1:7577789115487185988:2112] self: [1:7577789115487185987:0] peer: [6:7577789117631372543:0] socket: 308 qp: -1
2025-11-28T14:51:58.130667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789119782153302:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:58.130740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:58.139471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789115487185992:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.139503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789115487185992:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:58.142177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789115487185992:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:58.146749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789115487185915:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(22817350ns) {HandleHandshakeStatus:329(22814684ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(732ns) SwitchToState:179(642ns) ProcessPendingSessionEvents:495(14658659ns) {ForwardSessionEventToSession:612(58708ns) ForwardSessionEventToSession:612(5761ns) ForwardSessionEventToSession:612(2753ns) ForwardSessionEventToSession:612(2647ns) ForwardSessionEventToSession:612(5616657ns) ForwardSessionEventToSession:612(2892ns) ForwardSessionEventToSession:612(8152479ns) ForwardSessionEventToSession:612(495871ns) ForwardSessionEventToSession:612(6088ns) ForwardSessionEventToSession:612(291928ns)}}}
2025-11-28T14:51:58.147777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789115487185916:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.147850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:58.147949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789115487185916:2049] [node 3] ICP22 created new session: [1:7577789119782153308:2049]
2025-11-28T14:51:58.148025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789119782153308:2049] [node 3] ICS09 handshake done sender: [1:7577789115487185992:2114] self: [1:7577789115487185990:0] peer: [3:7577789121866096367:0] socket: 346 qp: -1
2025-11-28T14:51:58.148062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789119782153308:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:58.148599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:58.151870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789115487185982:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(35898890ns) {HandleHandshakeStatus:329(35896523ns) {DropIncomingHandshake:442(14805ns) DropOutgoingHandshake:457(490ns) SwitchToState:179(573ns) ProcessPendingSessionEvents:495(4536230ns) {ForwardSessionEventToSession:612(4290193ns) ForwardSessionEventToSession:612(10338ns) ForwardSessionEventToSession:612(6828ns) ForwardSessionEventToSession:612(6033ns) ForwardSessionEventToSession:612(2685ns) ForwardSessionEventToSession:612(3066ns) ForwardSessionEventToSession:612(19064ns) ForwardSessionEventToSession:612(2565ns) ForwardSessionEventToSession:612(2559ns) ForwardSessionEventToSession:612(180905ns)}}}
2025-11-28T14:51:58.212815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789117571129066:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.213041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789117571129052:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.213154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789119339307476:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45649
2025-11-28T14:51:58.213124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:58.213223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789117571129052:2107] [node 4] ICP22 created new session: [3:7577789121866096376:2107]
2025-11-28T14:51:58.213372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789121866096376:2107] [node 4] ICS09 handshake done sender: [3:7577789117571129066:2114] self: [3:7577789117571129065:0] peer: [4:7577789119339307475:0] socket: 359 qp: -1
2025-11-28T14:51:58.213634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789119339307474:2136] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:58.213410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789121866096376:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:58.213915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789115044340071:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789119339307474:2136] held: no
2025-11-28T14:51:58.216589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789115044340071:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.216643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:58.216910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789115044340071:2050] [node 3] ICP22 created new session: [4:7577789119339307477:2050]
2025-11-28T14:51:58.213629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:58.217035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789119339307477:2050] [node 3] ICS09 handshake done sender: [4:7577789119339307474:2136] self: [4:7577789119339307475:0] peer: [3:7577789117571129065:0] socket: 360 qp: -1
2025-11-28T14:51:58.217077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789119339307477:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:58.217126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.217280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789121663765590:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36647
2025-11-28T14:51:58.219188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789121663765590:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.219495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789117368798188:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.219782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789117571129058:2111] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.220302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789121866096365:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.220331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789121866096365:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:58.230911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789117631372542:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.231222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789117631372471:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.231585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789117571129058:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.235147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789121866096365:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:58.237339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789117571129058:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:58.237728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789117571129053:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.237785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:58.241416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789121926339861:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
2025-11-28T14:51:58.237915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789117571129054:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.239244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789117571129053:2108] [node 5] ICP22 created new session: [3:7577789121866096381:2108]
2025-11-28T14:51:58.240272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789121866096381:2108] [node 5] ICS09 handshake done sender: [3:7577789121866096365:2115] self: [3:7577789121866096364:0] peer: [5:7577789121663765591:0] socket: 364 qp: -1
2025-11-28T14:51:58.240282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:58.241741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789117571129054:2109] [node 6] ICP22 created new session: [3:7577789121866096382:2109]
2025-11-28T14:51:58.241713Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789117631372471:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789117631372542:2114] held: no
2025-11-28T14:51:58.241808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789121866096382:2109] [node 6] ICS09 handshake done sender: [3:7577789117571129058:2111] self: [3:7577789117571129057:0] peer: [6:7577789121926339860:0] socket: 317 qp: -1
2025-11-28T14:51:58.241861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789121866096382:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:58.241933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.242329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789117631372542:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:58.242040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789121866096381:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:58.242094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:58.242740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789117631372471:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.242793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:58.242842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789117631372471:2050] [node 3] ICP22 created new session: [6:7577789121926339862:2050]
2025-11-28T14:51:58.242902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789121926339862:2050] [node 3] ICS09 handshake done sender: [6:7577789117631372542:2114] self: [6:7577789121926339860:0] peer: [3:7577789117571129057:0] socket: 322 qp: -1
2025-11-28T14:51:58.242938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789121926339862:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:58.242979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:58.240812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789117368798269:2127] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.244275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789121663765592:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36647
2025-11-28T14:51:58.244736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789115487185991:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.244711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789121663765590:2137] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:58.249679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789117368798188:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.249763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:58.249817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789117368798188:2050] [node 3] ICP22 created new session: [5:7577789121663765593:2050]
2025-11-28T14:51:58.249868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789121663765593:2050] [node 3] ICS09 handshake done sender: [5:7577789121663765590:2137] self: [5:7577789121663765591:0] peer: [3:7577789121866096364:0] socket: 312 qp: -1
2025-11-28T14:51:58.249908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789121663765593:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:58.249956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:58.304199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789121663765573:2130] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.334413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789119346889236:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.347246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789115051921932:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.348533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789119782153295:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.348714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789115487185915:2048] [node 2] ICP18 handshake (actor [1:7577789119782153295:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.373313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789121866096385:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-11-28T14:51:58.391715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789115051921929:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.392575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789121663765579:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.392799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789117368798187:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.396044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789115051921929:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.406332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789115051921929:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:58.417723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789115051921923:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.420515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:58.420584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789115051921923:2108] [node 5] ICP22 created new session: [2:7577789119346889251:2108]
2025-11-28T14:51:58.429437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789119346889251:2108] [node 5] ICS09 handshake done sender: [2:7577789115051921929:2111] self: [2:7577789115051921928:0] peer: [5:7577789121663765595:0] socket: 335 qp: -1
2025-11-28T14:51:58.429487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789119346889251:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:58.429527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.436703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789121926339864:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
2025-11-28T14:51:58.444286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789115051921923:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17542621ns) {HandleHandshakeStatus:329(17538198ns) {DropIncomingHandshake:442(22491ns) DropOutgoingHandshake:457(1040ns) SwitchToState:179(4684ns) ProcessPendingSessionEvents:495(3488596ns) {ForwardSessionEventToSession:612(3388952ns) ForwardSessionEventToSession:612(6090ns) ForwardSessionEventToSession:612(17406ns) ForwardSessionEventToSession:612(3455ns) ForwardSessionEventToSession:612(2626ns) ForwardSessionEventToSession:612(22466ns) ForwardSessionEventToSession:612(3391ns) ForwardSessionEventToSession:612(3096ns) ForwardSessionEventToSession:612(28203ns) ForwardSessionEventToSession:612(2380ns)}}}
2025-11-28T14:51:58.434973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789117368798187:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789121663765579:2133] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.450938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789115487185995:2116] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.451046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789121663765596:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36647
2025-11-28T14:51:58.458559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789121663765579:2133] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:58.467185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789121663765581:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.467377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789117368798186:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.467440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789117368798187:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.467489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:58.467546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789117368798187:2049] [node 2] ICP22 created new session: [5:7577789121663765598:2049]
2025-11-28T14:51:58.467621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789121663765598:2049] [node 2] ICS09 handshake done sender: [5:7577789121663765579:2133] self: [5:7577789121663765595:0] peer: [2:7577789115051921928:0] socket: 336 qp: -1
2025-11-28T14:51:58.467657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789121663765598:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:58.467695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:58.467763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789115487185995:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.468259Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789115487185995:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.469349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789115487185981:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.469486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:58.469553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789115487185981:2108] [node 5] ICP22 created new session: [1:7577789119782153312:2108]
2025-11-28T14:51:58.469394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789121663765600:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36647
2025-11-28T14:51:58.469610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789119782153312:2108] [node 5] ICS09 handshake done sender: [1:7577789115487185995:2116] self: [1:7577789115487185994:0] peer: [5:7577789121663765597:0] socket: 326 qp: -1
2025-11-28T14:51:58.469654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789119782153312:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:58.469704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:58.469832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789121663765581:2135] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:58.470248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789117368798186:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.470294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:58.470336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789117368798186:2048] [node 1] ICP22 created new session: [5:7577789121663765601:2048]
2025-11-28T14:51:58.471008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789121663765601:2048] [node 1] ICS09 handshake done sender: [5:7577789121663765581:2135] self: [5:7577789121663765597:0] peer: [1:7577789115487185994:0] socket: 354 qp: -1
2025-11-28T14:51:58.471065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789121663765601:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:58.471111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:58.471469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789117368798186:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:58.473538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789119346889237:2119] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.497404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789121926339864:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.497644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789117631372470:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.497833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789119346889237:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.498235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789119346889237:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.498440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789115051921926:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.498506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789121926339866:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41526
2025-11-28T14:51:58.498532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:58.498653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789115051921926:2109] [node 6] ICP22 created new session: [2:7577789119346889255:2109]
2025-11-28T14:51:58.498728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789119346889255:2109] [node 6] ICS09 handshake done sender: [2:7577789119346889237:2119] self: [2:7577789115051921939:0] peer: [6:7577789121926339865:0] socket: 368 qp: -1
2025-11-28T14:51:58.498721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789121926339864:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:58.498781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789119346889255:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:58.498846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:58.498916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789117631372470:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.499831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:58.500049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789117631372470:2049] [node 2] ICP22 created new session: [6:7577789121926339867:2049]
2025-11-28T14:51:58.500119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789121926339867:2049] [node 2] ICS09 handshake done sender: [6:7577789121926339864:2123] self: [6:7577789121926339865:0] peer: [2:7577789115051921939:0] socket: 369 qp: -1
2025-11-28T14:51:58.500154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789121926339867:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:58.500196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:58.500538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789117631372470:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:58.504885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789121663765577:2131] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.514636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789121866096374:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.514833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789117571129053:2108] [node 5] ICP18 handshake (actor [3:7577789121866096374:2119]) is too old
2025-11-28T14:51:58.518943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789115051921938:2117] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.527764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789121866096385:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.528021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789117571128990:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.528237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789115051921938:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.528761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789115051921938:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.528983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789115051921862:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.528997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789121866096387:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-11-28T14:51:58.529242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789121866096385:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:58.529345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789117571128990:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.529080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:58.529386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:58.529149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789115051921862:2049] [node 3] ICP22 created new session: [2:7577789119346889258:2049]
2025-11-28T14:51:58.529198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789119346889258:2049] [node 3] ICS09 handshake done sender: [2:7577789115051921938:2117] self: [2:7577789115051921930:0] peer: [3:7577789121866096386:0] socket: 324 qp: -1
2025-11-28T14:51:58.529885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789117571128990:2049] [node 2] ICP22 created new session: [3:7577789121866096388:2049]
2025-11-28T14:51:58.529231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789119346889258:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:58.530504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789121866096388:2049] [node 2] ICS09 handshake done sender: [3:7577789121866096385:2120] self: [3:7577789121866096386:0] peer: [2:7577789115051921930:0] socket: 340 qp: -1
2025-11-28T14:51:58.530776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789121866096388:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:58.529279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:58.530820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:58.531068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789117571128990:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:58.531838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789115051921934:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:51:58.532868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789119339307464:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.533045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789115044340070:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:58.533230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789115051921934:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:58.533648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789115051921934:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:58.533876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789115051921925:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:58.533881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789119339307480:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45649
2025-11-28T14:51:58.533951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:58.534009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789115051921925:2107] [node 4] ICP22 created new session: [2:7577789119346889261:2107]
2025-11-28T14:51:58.534055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789119346889261:2107] [node 4] ICS09 handshake done sender: [2:7577789115051921934:2113] self: [2:7577789115051921933:0] peer: [4:7577789119339307479:0] socket: 342 qp: -1
2025-11-28T14:51:58.534099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789119346889261:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:58.534117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789119339307464:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:58.534142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:58.534290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789115044340070:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:58.534352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:58.534401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789115044340070:2049] [node 2] ICP22 created new session: [4:7577789119339307481:2049]
2025-11-28T14:51:58.534564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789119339307481:2049] [node 2] ICS09 handshake done sender: [4:7577789119339307464:2135] self: [4:7577789119339307479:0] peer: [2:7577789115051921933:0] socket: 345 qp: -1
2025-11-28T14:51:58.534594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789119339307481:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:58.534627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:58.534876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789115044340070:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:58.572017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:58.572265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789119346889245:2048] [node 1] ICS01 socket: 303 reason# EndOfStream
2025-11-28T14:51:58.572318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789115051921861:2048] [node 1] ICP30 unregister session Session# [2:7577789119346889245:2048] VirtualId# [2:7577789119346889243:0]
2025-11-28T14:51:58.572357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789119346889245:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.572467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:58.572538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:58.573150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:58.573355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789121866096371:2048] [node 1] ICS07 socket disconnect 347 reason# EndOfStream
2025-11-28T14:51:58.573390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789121866096371:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.573485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:58.573549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789121866096371:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:58.576174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789117571128989:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Socket error# Connection reset by peer state# SendExRequestHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
2025-11-28T14:51:58.576227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789121866096371:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:58.575952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:58.576237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789119339307465:2048] [node 1] ICS07 socket disconnect 302 reason# EndOfStream
2025-11-28T14:51:58.576263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789119339307465:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.584331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789117571128989:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.584418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789121866096371:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.576342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:58.580983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789119339307465:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:58.588521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789117571128989:2048] [node 1] ICP30 unregister session Session# [3:7577789121866096371:2048] VirtualId# [3:7577789121866096367:0]
2025-11-28T14:51:58.588608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:58.588840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789117571128989:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.588869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.588515Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:58.588777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789121663765601:2048] [node 1] ICS07 socket disconnect 354 reason# EndOfStream
2025-11-28T14:51:58.588815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789121663765601:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.588941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:58.588989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789121663765601:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:58.589498Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:58.589625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789115044340069:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.589769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789121926339852:2048] [node 1] ICS01 socket: 310 reason# EndOfStream
2025-11-28T14:51:58.589830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789119339307465:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.589889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789115044340069:2048] [node 1] ICP30 unregister session Session# [4:7577789119339307465:2048] VirtualId# [4:7577789115044340164:0]
2025-11-28T14:51:58.589822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789117631372469:2048] [node 1] ICP30 unregister session Session# [6:7577789121926339852:2048] VirtualId# [6:7577789117631372543:0]
2025-11-28T14:51:58.589860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789121926339852:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.589999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:58.589974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:58.590033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:58.590076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789115044340069:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.590097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.590113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789117368798186:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.590215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789121663765601:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.590268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789117368798186:2048] [node 1] ICP30 unregister session Session# [5:7577789121663765601:2048] VirtualId# [5:7577789121663765597:0]
2025-11-28T14:51:58.590315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:58.590348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789117368798186:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.590368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.597571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:58.597858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789121866096388:2049] [node 2] ICS07 socket disconnect 340 reason# EndOfStream
2025-11-28T14:51:58.597889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789121866096388:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.597974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:58.598015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789121866096388:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:58.598948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789117571128990:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.599054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789121866096388:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.599095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789117571128990:2049] [node 2] ICP30 unregister session Session# [3:7577789121866096388:2049] VirtualId# [3:7577789121866096386:0]
2025-11-28T14:51:58.599145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:58.599182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789117571128990:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.599195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.600062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789115044340069:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:58.599810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789117368798186:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:58.600311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:58.600445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789117571128989:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:58.600525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789119339307481:2049] [node 2] ICS07 socket disconnect 345 reason# EndOfStream
2025-11-28T14:51:58.600566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789119339307481:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.600692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:58.600746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789119339307481:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:58.601348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:58.601497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789115044340070:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.601555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789121663765598:2049] [node 2] ICS01 socket: 336 reason# EndOfStream
2025-11-28T14:51:58.601598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789119339307481:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.601613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789117368798187:2049] [node 2] ICP30 unregister session Session# [5:7577789121663765598:2049] VirtualId# [5:7577789121663765595:0]
2025-11-28T14:51:58.601652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789115044340070:2049] [node 2] ICP30 unregister session Session# [4:7577789119339307481:2049] VirtualId# [4:7577789119339307479:0]
2025-11-28T14:51:58.601648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789121663765598:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.601700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:58.601737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789115044340070:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.601732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:58.601754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.601793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:58.602325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:58.602522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789121926339867:2049] [node 2] ICS07 socket disconnect 369 reason# EndOfStream
2025-11-28T14:51:58.602552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789121926339867:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.602615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:58.602659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789121926339867:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:58.603337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789117631372470:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.603404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789121926339867:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.603442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789117631372470:2049] [node 2] ICP30 unregister session Session# [6:7577789121926339867:2049] VirtualId# [6:7577789121926339865:0]
2025-11-28T14:51:58.603505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:58.603555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789117631372470:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.603612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.607150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789117571128990:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:58.611072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789115044340070:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:58.613359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789117631372470:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:51:58.631073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:58.631302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789119339307477:2050] [node 3] ICS01 socket: 360 reason# EndOfStream
2025-11-28T14:51:58.631358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789115044340071:2050] [node 3] ICP30 unregister session Session# [4:7577789119339307477:2050] VirtualId# [4:7577789119339307475:0]
2025-11-28T14:51:58.631390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789119339307477:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.631479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:58.631529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:58.632149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:58.632353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789121663765593:2050] [node 3] ICS07 socket disconnect 312 reason# EndOfStream
2025-11-28T14:51:58.632400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789121663765593:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.632479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:58.632521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789121663765593:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:58.633193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:58.633384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789121926339862:2050] [node 3] ICS01 socket: 322 reason# EndOfStream
2025-11-28T14:51:58.633420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789117631372471:2050] [node 3] ICP30 unregister session Session# [6:7577789121926339862:2050] VirtualId# [6:7577789121926339860:0]
2025-11-28T14:51:58.633447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789121926339862:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.633535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:58.633590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:58.633862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789117368798188:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14811) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.633982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789121663765593:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.634026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789117368798188:2050] [node 3] ICP30 unregister session Session# [5:7577789121663765593:2050] VirtualId# [5:7577789121663765591:0]
2025-11-28T14:51:58.634084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:58.634117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789117368798188:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.634132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.643871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789117368798188:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:58.677363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:58.677633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789121663765584:2108] [node 4] ICS07 socket disconnect 352 reason# EndOfStream
2025-11-28T14:51:58.677667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789121663765584:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.677754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:58.677796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789121663765584:2108] [node 4] ICS15 start handshake
2025-11-28T14:51:58.679059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789117368798267:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20914) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.678389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:58.679125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789121663765584:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.678597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789121926339854:2110] [node 4] ICS07 socket disconnect 309 reason# EndOfStream
2025-11-28T14:51:58.679510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789117368798267:2108] [node 4] ICP30 unregister session Session# [5:7577789121663765584:2108] VirtualId# [5:7577789121663765582:0]
2025-11-28T14:51:58.678631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789121926339854:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.679571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:58.678703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:58.678742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789121926339854:2110] [node 4] ICS15 start handshake
2025-11-28T14:51:58.679652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789117368798267:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20914) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.679688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20914) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.680163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789117631372535:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20914) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.680279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789121926339854:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.680315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789117631372535:2110] [node 4] ICP30 unregister session Session# [6:7577789121926339854:2110] VirtualId# [6:7577789121926339848:0]
2025-11-28T14:51:58.680358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:58.680404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789117631372535:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20914) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.680421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20914) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.690985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789117631372535:2110] [node 4] ICP33 wake up from error state
2025-11-28T14:51:58.691462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:58.691689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789121926339858:2106] [node 5] ICS07 socket disconnect 328 reason# EndOfStream
2025-11-28T14:51:58.691721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789121926339858:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:58.691824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:58.691880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789121926339858:2106] [node 5] ICS15 start handshake
2025-11-28T14:51:58.692638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789117631372531:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10127) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:58.692721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789121926339858:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:58.692760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789117631372531:2106] [node 5] ICP30 unregister session Session# [6:7577789121926339858:2106] VirtualId# [6:7577789121926339856:0]
2025-11-28T14:51:58.692810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:51:58.692848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789117631372531:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10127) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:58.692883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10127) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:58.760218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789121797544087:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:58.760266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789121797544088:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:58.790597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789120709985619:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:58.790635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789120709985620:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:58.818794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789120823280929:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:58.818845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789120823280930:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:58.847919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789121844391709:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:58.847984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789121844391710:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:58.848035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789121844391711:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:58.873446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789118957546061:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:58.873500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789118957546062:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:58.873534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789118957546063:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:58.894589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789121026680670:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:58.894656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789121026680671:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:58.894694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789121026680672:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:59.024635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789125321648015:2093] [node 4] ICP01 ready to work
2025-11-28T14:51:59.025894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789125004952979:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:59.027073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789125118248306:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:59.024646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789126092511447:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:59.028856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789125004952980:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:59.027043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789126092511448:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:59.030455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789126139359077:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:59.030993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789125321648022:2098] [node 5] ICP01 ready to work
2025-11-28T14:51:59.031919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789126092511449:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:59.031934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789125004952981:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:59.034314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789123252513439:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:59.034599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789123252513440:2115] [node 6] ICP01 ready to work
2025-11-28T14:51:59.035485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789125118248309:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:59.035965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789125118248310:2114] [node 6] ICP01 ready to work
2025-11-28T14:51:59.036605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789126139359080:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.041807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789126092511463:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49941
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.044726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789125004952992:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16755
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.045424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789125004952993:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16755
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.045690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789125118248319:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15395
2025-11-28T14:51:59.045789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789125118248320:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15395
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 2025-11-28T14:51:59.046846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789125004952997:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16755
::ffff:127.1.0.0
2025-11-28T14:51:59.047122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789125004952998:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16755
2025-11-28T14:51:59.047111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789126139359087:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56885
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.048627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789125118248324:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15395
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.049320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789126139359090:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56885
2025-11-28T14:51:59.049469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789126139359091:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56885
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.049631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789125004952992:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.049883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789126139359092:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56885
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.051203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511463:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.052057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789126092511464:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49941
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.052242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789126092511465:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49941
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.052341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789126139359087:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.052407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789126092511466:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49941
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.052529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789126092511467:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49941
2025-11-28T14:51:59.054106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789125118248319:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.054249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789125118248320:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.054379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511465:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.054610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511464:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.054828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789125118248325:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15395
2025-11-28T14:51:59.055272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789120823280929:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.055581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789125004952993:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.055545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511453:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.055584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789126092511453:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.056256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511466:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.056480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789125118248324:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.056505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789125004952997:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.056632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789125004952999:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16755
2025-11-28T14:51:59.056843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789125004952998:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.056850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789126092511453:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:59.056906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789120823280930:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.056919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789126139359090:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.056925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789120709985619:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.057193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789125118248328:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15395
2025-11-28T14:51:59.057312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511467:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.057346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789126139359091:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.057368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789121797544088:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.057423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:59.057476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789121797544088:2049] [node 3] ICP22 created new session: [1:7577789126092511473:2049]
2025-11-28T14:51:59.057541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789126092511473:2049] [node 3] ICS09 handshake done sender: [1:7577789126092511453:2112] self: [1:7577789126092511452:0] peer: [3:7577789125118248327:0] socket: 309 qp: -1
2025-11-28T14:51:59.057617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789125004952985:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.057451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789121844391709:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.057590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789126092511473:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:59.057647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789125004952985:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.057600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789126139359092:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.057628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:59.058097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789121844391711:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.058137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789125118248325:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.058164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789125004952999:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.058211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789125118248320:2134] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:59.058277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789121844391710:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.058484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511451:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.058518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789126092511451:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.058535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789125118248314:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.058554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789125118248314:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.059261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789125004952987:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.059292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789125004952987:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.060274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789120823280929:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.060327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:59.060360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789120823280929:2048] [node 1] ICP22 created new session: [3:7577789125118248330:2048]
2025-11-28T14:51:59.060419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789125118248330:2048] [node 1] ICS09 handshake done sender: [3:7577789125118248320:2134] self: [3:7577789125118248327:0] peer: [1:7577789126092511452:0] socket: 319 qp: -1
2025-11-28T14:51:59.060447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789125118248330:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:59.060484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:59.064499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511455:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.064581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789125004952985:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:59.064550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789126092511455:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.065702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789125118248333:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15395
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.065870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789126139359100:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56885
2025-11-28T14:51:59.066028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789120709985620:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.066106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:59.066229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789125118248314:2130] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:59.066175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789120709985620:2049] [node 3] ICP22 created new session: [2:7577789125004953007:2049]
2025-11-28T14:51:59.066237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789125004953007:2049] [node 3] ICS09 handshake done sender: [2:7577789125004952985:2112] self: [2:7577789125004952984:0] peer: [3:7577789125118248329:0] socket: 327 qp: -1
2025-11-28T14:51:59.066271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789125004953007:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:59.066302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:59.066661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789125118248306:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.066728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:59.066770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789125118248306:2107] [node 4] ICP22 created new session: [3:7577789125118248336:2107]
2025-11-28T14:51:59.066818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789125118248336:2107] [node 4] ICS09 handshake done sender: [3:7577789125118248314:2130] self: [3:7577789125118248313:0] peer: [4:7577789126139359098:0] socket: 325 qp: -1
2025-11-28T14:51:59.066863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789125118248336:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:59.066777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789126092511451:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:59.066903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.067365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789123252513452:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50728
2025-11-28T14:51:59.067153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789121797544087:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.067498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:59.067545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789121797544087:2048] [node 2] ICP22 created new session: [1:7577789126092511479:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.068034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789126092511479:2048] [node 2] ICS09 handshake done sender: [1:7577789126092511451:2111] self: [1:7577789126092511450:0] peer: [2:7577789125004953000:0] socket: 307 qp: -1
2025-11-28T14:51:59.068084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789126092511479:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:59.068141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:59.068605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789125118248324:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:59.073449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789126139359091:2126] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.073848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789126092511455:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.070512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789123252513453:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50728
2025-11-28T14:51:59.073946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789126139359101:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56885
2025-11-28T14:51:59.074075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789126139359102:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56885
2025-11-28T14:51:59.074850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789125004952987:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:59.077611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789120823280930:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.077664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:59.077718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789120823280930:2049] [node 2] ICP22 created new session: [3:7577789125118248338:2049]
2025-11-28T14:51:59.077785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789125118248338:2049] [node 2] ICS09 handshake done sender: [3:7577789125118248324:2135] self: [3:7577789125118248329:0] peer: [2:7577789125004952984:0] socket: 334 qp: -1
2025-11-28T14:51:59.077814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789125118248338:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:59.077850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:59.075975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789121844391711:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.076033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:59.079329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789120823280930:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:59.076083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789121844391711:2050] [node 3] ICP22 created new session: [4:7577789126139359103:2050]
2025-11-28T14:51:59.076566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789126139359103:2050] [node 3] ICS09 handshake done sender: [4:7577789126139359091:2126] self: [4:7577789126139359098:0] peer: [3:7577789125118248313:0] socket: 330 qp: -1
2025-11-28T14:51:59.076607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789126139359103:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:59.076659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:59.080334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789125004952979:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.080601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:59.080674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789125004952979:2107] [node 4] ICP22 created new session: [2:7577789125004953011:2107]
2025-11-28T14:51:59.080726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789125004953011:2107] [node 4] ICS09 handshake done sender: [2:7577789125004952987:2113] self: [2:7577789125004952986:0] peer: [4:7577789126139359099:0] socket: 329 qp: -1
2025-11-28T14:51:59.080758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789125004953011:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:59.080788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:59.081365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789126092511447:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.081425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:59.081464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789126092511447:2107] [node 4] ICP22 created new session: [1:7577789126092511481:2107]
2025-11-28T14:51:59.081508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789126092511481:2107] [node 4] ICS09 handshake done sender: [1:7577789126092511455:2113] self: [1:7577789126092511454:0] peer: [4:7577789126139359096:0] socket: 311 qp: -1
2025-11-28T14:51:59.081683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789126092511481:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:59.082621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.085156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789123252513454:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50728
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.088549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789121844391711:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:59.087459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648044:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.090471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789125004953009:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16755
2025-11-28T14:51:59.090052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789123252513453:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.091116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789126139359087:2123] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:59.090455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789118957546063:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.091877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789120709985619:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789125004952992:2116] held: no
2025-11-28T14:51:59.092293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789125004952992:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:59.092405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789121844391710:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29505) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789126139359092:2127] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.093459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648045:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.093746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789120709985619:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.096542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789125118248316:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.093519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648046:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
2025-11-28T14:51:59.099838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789125321648046:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.098601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789123252513454:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.093795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:59.093839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789120709985619:2048] [node 1] ICP22 created new session: [2:7577789125004953014:2048]
2025-11-28T14:51:59.098856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789123252513456:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50728
2025-11-28T14:51:59.093898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789125004953014:2048] [node 1] ICS09 handshake done sender: [2:7577789125004952992:2116] self: [2:7577789125004953000:0] peer: [1:7577789126092511450:0] socket: 316 qp: -1
2025-11-28T14:51:59.098936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789118957546062:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.093936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789125004953014:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:59.096585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789125118248316:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.093971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:59.100886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789125004952989:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.100920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789125004952989:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.100304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789125321648044:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.100475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789121026680671:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.100927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789123252513452:2132] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.100529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789121026680672:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.105109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789118957546061:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.101114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789125118248316:2131] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:59.113641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789125004952991:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.116722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789121844391709:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.116796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:59.113685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789125004952991:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.115149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789125004952989:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:59.115403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789125321648045:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.137768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789125004952991:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:59.119515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789121844391709:2048] [node 1] ICP22 created new session: [4:7577789126139359105:2048]
2025-11-28T14:51:59.136507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789126139359105:2048] [node 1] ICS09 handshake done sender: [4:7577789126139359087:2123] self: [4:7577789126139359096:0] peer: [1:7577789126092511454:0] socket: 315 qp: -1
2025-11-28T14:51:59.136591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789126139359105:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:59.138860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789123252513453:2133] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:59.136697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.147611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789118957546063:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.147667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:59.147714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789118957546063:2050] [node 3] ICP22 created new session: [5:7577789123252513460:2050]
2025-11-28T14:51:59.147757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789123252513460:2050] [node 3] ICS09 handshake done sender: [5:7577789123252513453:2133] self: [5:7577789123252513455:0] peer: [3:7577789125118248315:0] socket: 359 qp: -1
2025-11-28T14:51:59.147787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789123252513460:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:59.147817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:59.148282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789123252513459:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50728
2025-11-28T14:51:59.137289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789121844391709:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20294748ns) {HandleHandshakeStatus:329(20290932ns) {DropIncomingHandshake:442(1116ns) DropOutgoingHandshake:457(8416ns) SwitchToState:179(904ns) ProcessPendingSessionEvents:495(111941ns) {ForwardSessionEventToSession:612(64523ns) ForwardSessionEventToSession:612(3410ns) ForwardSessionEventToSession:612(5436ns) ForwardSessionEventToSession:612(2650ns) ForwardSessionEventToSession:612(7386ns) ForwardSessionEventToSession:612(3835ns) ForwardSessionEventToSession:612(5486ns) ForwardSessionEventToSession:612(4443ns) ForwardSessionEventToSession:612(2515ns) ForwardSessionEventToSession:612(2315ns)}}}
2025-11-28T14:51:59.139173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789125118248318:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.148522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789125004952980:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.137385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789126139359092:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:59.148582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:59.139206Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789125118248318:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.157370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511457:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.148637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789125004952980:2109] [node 5] ICP22 created new session: [2:7577789125004953018:2109]
2025-11-28T14:51:59.157417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789126092511457:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.148689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789125004953018:2109] [node 5] ICS09 handshake done sender: [2:7577789125004952989:2114] self: [2:7577789125004952988:0] peer: [5:7577789123252513457:0] socket: 322 qp: -1
2025-11-28T14:51:59.148721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789125004953018:2109] [node 5] ICS10 traffic start
2025-11-28T14:51:59.148755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:59.149045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789125004952981:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.149080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:59.149118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789125004952981:2110] [node 6] ICP22 created new session: [2:7577789125004953020:2110]
2025-11-28T14:51:59.149162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789125004953020:2110] [node 6] ICS09 handshake done sender: [2:7577789125004952991:2115] self: [2:7577789125004952990:0] peer: [6:7577789125321648047:0] socket: 339 qp: -1
2025-11-28T14:51:59.149186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789125004953020:2110] [node 6] ICS10 traffic start
2025-11-28T14:51:59.138581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789121844391709:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:59.160215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789126092511457:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:59.157432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789121026680670:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.155898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789125118248309:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.149229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:59.146067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789121844391710:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.155955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:59.156016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789125118248309:2108] [node 5] ICP22 created new session: [3:7577789125118248342:2108]
2025-11-28T14:51:59.156066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789125118248342:2108] [node 5] ICS09 handshake done sender: [3:7577789125118248316:2131] self: [3:7577789125118248315:0] peer: [5:7577789123252513455:0] socket: 331 qp: -1
2025-11-28T14:51:59.156101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789125118248342:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:59.156134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:59.146112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:59.146165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789121844391710:2049] [node 2] ICP22 created new session: [4:7577789126139359109:2049]
2025-11-28T14:51:59.146214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789126139359109:2049] [node 2] ICS09 handshake done sender: [4:7577789126139359092:2127] self: [4:7577789126139359099:0] peer: [2:7577789125004952986:0] socket: 332 qp: -1
2025-11-28T14:51:59.146258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789126139359109:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:59.146291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.165196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789125118248318:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:59.162621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789126092511448:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.162699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:59.168590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789126092511448:2108] [node 5] ICP22 created new session: [1:7577789126092511485:2108]
2025-11-28T14:51:59.160502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648049:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
2025-11-28T14:51:59.164918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789121026680671:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29505) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789125321648046:2117] held: no
2025-11-28T14:51:59.165094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648050:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
2025-11-28T14:51:59.168542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789125321648046:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:59.170797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789126092511485:2108] [node 5] ICS09 handshake done sender: [1:7577789126092511457:2114] self: [1:7577789126092511456:0] peer: [5:7577789123252513458:0] socket: 356 qp: -1
2025-11-28T14:51:59.170855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789126092511485:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:59.170902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:59.172228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789125118248310:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.172347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789125321648050:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.172561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789121026680671:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.172610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:59.172674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789121026680671:2049] [node 2] ICP22 created new session: [6:7577789125321648053:2049]
2025-11-28T14:51:59.172729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789125321648053:2049] [node 2] ICS09 handshake done sender: [6:7577789125321648046:2117] self: [6:7577789125321648047:0] peer: [2:7577789125004952990:0] socket: 345 qp: -1
2025-11-28T14:51:59.172758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789125321648053:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:59.173008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:59.172287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:59.173938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789123252513454:2134] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:59.172332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789125118248310:2114] [node 6] ICP22 created new session: [3:7577789125118248345:2114]
2025-11-28T14:51:59.172386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789125118248345:2114] [node 6] ICS09 handshake done sender: [3:7577789125118248318:2132] self: [3:7577789125118248317:0] peer: [6:7577789125321648048:0] socket: 347 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.172422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789125118248345:2114] [node 6] ICS10 traffic start
2025-11-28T14:51:59.175799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789118957546062:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29505) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789123252513454:2134] held: no
2025-11-28T14:51:59.175906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789118957546062:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.175946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:59.175405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789126092511459:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.172452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:59.176092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789118957546062:2049] [node 2] ICP22 created new session: [5:7577789123252513464:2049]
2025-11-28T14:51:59.175439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789126092511459:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.176518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648052:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
2025-11-28T14:51:59.176158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789123252513464:2049] [node 2] ICS09 handshake done sender: [5:7577789123252513454:2134] self: [5:7577789123252513457:0] peer: [2:7577789125004952988:0] socket: 344 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.181339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789125321648015:2093] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.181451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789125321648044:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.181893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789125118248347:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15395
2025-11-28T14:51:59.182363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789121026680672:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.179966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789126092511459:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:59.180314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789126092511449:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.180422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:59.180475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789126092511449:2109] [node 6] ICP22 created new session: [1:7577789126092511488:2109]
2025-11-28T14:51:59.180537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789126092511488:2109] [node 6] ICS09 handshake done sender: [1:7577789126092511459:2115] self: [1:7577789126092511458:0] peer: [6:7577789125321648051:0] socket: 324 qp: -1
2025-11-28T14:51:59.180573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789126092511488:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:59.180634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:59.176194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789123252513464:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:59.176223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:59.180749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789123252513462:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50728
2025-11-28T14:51:59.182421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:59.182465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789121026680672:2050] [node 3] ICP22 created new session: [6:7577789125321648058:2050]
2025-11-28T14:51:59.182511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789125321648058:2050] [node 3] ICS09 handshake done sender: [6:7577789125321648044:2115] self: [6:7577789125321648048:0] peer: [3:7577789125118248317:0] socket: 342 qp: -1
2025-11-28T14:51:59.182540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789125321648058:2050] [node 3] ICS10 traffic start
2025-11-28T14:51:59.183524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789123252513462:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.183380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789126139359089:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.183432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789126139359089:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.185104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789126139359089:2124] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:59.185383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789126139359080:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.185459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:59.185507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789126139359080:2109] [node 6] ICP22 created new session: [4:7577789126139359112:2109]
2025-11-28T14:51:59.185552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789126139359112:2109] [node 6] ICS09 handshake done sender: [4:7577789126139359089:2124] self: [4:7577789126139359088:0] peer: [6:7577789125321648057:0] socket: 367 qp: -1
2025-11-28T14:51:59.185585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789126139359112:2109] [node 6] ICS10 traffic start
2025-11-28T14:51:59.185621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:59.193465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789118957546061:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789123252513452:2132] held: no
2025-11-28T14:51:59.182570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.194853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789123252513463:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50728
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.198506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789126139359114:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56885
2025-11-28T14:51:59.198054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789123252513439:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.198165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789123252513452:2132] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:59.199396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789118957546061:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.199447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:59.199494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789118957546061:2048] [node 1] ICP22 created new session: [5:7577789123252513469:2048]
2025-11-28T14:51:59.199734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789123252513469:2048] [node 1] ICS09 handshake done sender: [5:7577789123252513452:2132] self: [5:7577789123252513458:0] peer: [1:7577789126092511456:0] socket: 358 qp: -1
2025-11-28T14:51:59.199771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789123252513469:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:59.199829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:59.205001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789126139359086:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.205052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789126139359086:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.206684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789126139359086:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:59.207354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789126139359077:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.207408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.207462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789126139359077:2108] [node 5] ICP22 created new session: [4:7577789126139359116:2108]
2025-11-28T14:51:59.207516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789126139359116:2108] [node 5] ICS09 handshake done sender: [4:7577789126139359086:2122] self: [4:7577789126139359085:0] peer: [5:7577789123252513468:0] socket: 366 qp: -1
2025-11-28T14:51:59.207554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789126139359116:2108] [node 5] ICS10 traffic start
2025-11-28T14:51:59.207596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:59.207522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789123252513471:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50728
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.207960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648056:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
2025-11-28T14:51:59.208472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789126139359114:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.208734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789121026680670:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789125321648045:2116] held: no
2025-11-28T14:51:59.209369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789125321648015:2093] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13790) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789125321648050:2119] held: no
2025-11-28T14:51:59.209271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789123252513462:2137] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:59.208975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789126139359077:2108] [node 5] ICP18 handshake (actor [4:7577789126139359114:2131]) is too old
2025-11-28T14:51:59.211548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789125321648045:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:59.211501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789123252513439:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.211561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:59.211607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789123252513439:2108] [node 4] ICP22 created new session: [5:7577789123252513472:2108]
2025-11-28T14:51:59.211822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789123252513472:2108] [node 4] ICS09 handshake done sender: [5:7577789123252513462:2137] self: [5:7577789123252513468:0] peer: [4:7577789126139359085:0] socket: 368 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.211879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789123252513472:2108] [node 4] ICS10 traffic start
2025-11-28T14:51:59.211915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:59.212157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789123252513439:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:51:59.212062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789121026680670:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.212104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:59.212148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789121026680670:2048] [node 1] ICP22 created new session: [6:7577789125321648063:2048]
2025-11-28T14:51:59.212207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789125321648063:2048] [node 1] ICS09 handshake done sender: [6:7577789125321648045:2116] self: [6:7577789125321648051:0] peer: [1:7577789126092511458:0] socket: 340 qp: -1
2025-11-28T14:51:59.212243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789125321648063:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:59.212290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:59.212819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648060:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.213864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648061:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
2025-11-28T14:51:59.214331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789125321648050:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.215173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789123252513474:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50728
2025-11-28T14:51:59.216814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789125321648061:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.216869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789125321648015:2093] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.216910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:59.216952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789125321648015:2093] [node 4] ICP22 created new session: [6:7577789125321648065:2093]
2025-11-28T14:51:59.217015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789125321648065:2093] [node 4] ICS09 handshake done sender: [6:7577789125321648050:2119] self: [6:7577789125321648057:0] peer: [4:7577789126139359088:0] socket: 369 qp: -1
2025-11-28T14:51:59.217050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789125321648065:2093] [node 4] ICS10 traffic start
2025-11-28T14:51:59.217088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:59.219263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789125321648022:2098] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.220689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789123252513448:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.220729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789123252513448:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.221623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789123252513474:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.221835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789125321648068:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2898
2025-11-28T14:51:59.222072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789123252513448:2131] [node 6] ICH04 handshake succeeded
2025-11-28T14:51:59.222529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789123252513440:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.222938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789125321648061:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:59.222590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:51:59.222701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789123252513440:2115] [node 6] ICP22 created new session: [5:7577789123252513477:2115]
2025-11-28T14:51:59.222881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789123252513477:2115] [node 6] ICS09 handshake done sender: [5:7577789123252513448:2131] self: [5:7577789123252513447:0] peer: [6:7577789125321648067:0] socket: 328 qp: -1
2025-11-28T14:51:59.222923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789123252513477:2115] [node 6] ICS10 traffic start
2025-11-28T14:51:59.222968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:51:59.223452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789125321648022:2098] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.223502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:59.223545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789125321648022:2098] [node 5] ICP22 created new session: [6:7577789125321648069:2098]
2025-11-28T14:51:59.223589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789125321648069:2098] [node 5] ICS09 handshake done sender: [6:7577789125321648061:2123] self: [6:7577789125321648067:0] peer: [5:7577789123252513447:0] socket: 370 qp: -1
2025-11-28T14:51:59.223626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789125321648069:2098] [node 5] ICS10 traffic start
2025-11-28T14:51:59.223662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:51:59.244194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:51:59.244470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789125004953014:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-11-28T14:51:59.244513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789120709985619:2048] [node 1] ICP30 unregister session Session# [2:7577789125004953014:2048] VirtualId# [2:7577789125004953000:0]
2025-11-28T14:51:59.244557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789125004953014:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.244669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:59.244718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:59.244987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:51:59.245162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789125118248330:2048] [node 1] ICS01 socket: 319 reason# EndOfStream
2025-11-28T14:51:59.245192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789120823280929:2048] [node 1] ICP30 unregister session Session# [3:7577789125118248330:2048] VirtualId# [3:7577789125118248327:0]
2025-11-28T14:51:59.245218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789125118248330:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.245276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:59.245309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:59.252588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:51:59.252824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789126139359105:2048] [node 1] ICS07 socket disconnect 315 reason# EndOfStream
2025-11-28T14:51:59.252853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789126139359105:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.252929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:59.252976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789126139359105:2048] [node 1] ICS15 start handshake
2025-11-28T14:51:59.253466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:51:59.253633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789123252513469:2048] [node 1] ICS01 socket: 358 reason# EndOfStream
2025-11-28T14:51:59.253671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789118957546061:2048] [node 1] ICP30 unregister session Session# [5:7577789123252513469:2048] VirtualId# [5:7577789123252513458:0]
2025-11-28T14:51:59.253707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789123252513469:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.253767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:59.253806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:59.255373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789121844391709:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5571) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:59.255424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789126139359105:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:59.255457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789121844391709:2048] [node 1] ICP30 unregister session Session# [4:7577789126139359105:2048] VirtualId# [4:7577789126139359096:0]
2025-11-28T14:51:59.255499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:59.255537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789121844391709:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5571) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:59.255558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5571) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:59.253906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:51:59.254076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789125321648063:2048] [node 1] ICS01 socket: 340 reason# EndOfStream
2025-11-28T14:51:59.254109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789121026680670:2048] [node 1] ICP30 unregister session Session# [6:7577789125321648063:2048] VirtualId# [6:7577789125321648051:0]
2025-11-28T14:51:59.254135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789125321648063:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.254183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:51:59.254221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:51:59.267714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789121844391709:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:51:59.298447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:51:59.298597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789125118248338:2049] [node 2] ICS07 socket disconnect 334 reason# EndOfStream
2025-11-28T14:51:59.298618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789125118248338:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.298681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:59.298711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789125118248338:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:59.301759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789120823280930:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29505) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:51:59.301800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789125118248338:2049] [node 2] ICS15 start handshake
2025-11-28T14:51:59.302605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789120823280930:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29505) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:59.302653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789125118248338:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:59.302688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789120823280930:2049] [node 2] ICP30 unregister session Session# [3:7577789125118248338:2049] VirtualId# [3:7577789125118248329:0]
2025-11-28T14:51:59.302732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:59.302786Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789120823280930:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29505) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:59.302801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29505) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:59.305340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:51:59.305540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789126139359109:2049] [node 2] ICS01 socket: 332 reason# EndOfStream
2025-11-28T14:51:59.305583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789121844391710:2049] [node 2] ICP30 unregister session Session# [4:7577789126139359109:2049] VirtualId# [4:7577789126139359099:0]
2025-11-28T14:51:59.305617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789126139359109:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.305692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:59.305740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:59.306619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:51:59.306809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789123252513464:2049] [node 2] ICS01 socket: 344 reason# EndOfStream
2025-11-28T14:51:59.306840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789118957546062:2049] [node 2] ICP30 unregister session Session# [5:7577789123252513464:2049] VirtualId# [5:7577789123252513457:0]
2025-11-28T14:51:59.306866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789123252513464:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.306918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:59.306956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:59.307707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:51:59.307894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789125321648053:2049] [node 2] ICS01 socket: 345 reason# EndOfStream
2025-11-28T14:51:59.307933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789121026680671:2049] [node 2] ICP30 unregister session Session# [6:7577789125321648053:2049] VirtualId# [6:7577789125321648047:0]
2025-11-28T14:51:59.307959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789125321648053:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.308006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:51:59.308043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:51:59.344534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:51:59.344752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789126139359103:2050] [node 3] ICS01 socket: 330 reason# EndOfStream
2025-11-28T14:51:59.344801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789121844391711:2050] [node 3] ICP30 unregister session Session# [4:7577789126139359103:2050] VirtualId# [4:7577789126139359098:0]
2025-11-28T14:51:59.344837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789126139359103:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.344931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:59.344984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:59.344570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:51:59.344812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789123252513460:2050] [node 3] ICS01 socket: 359 reason# EndOfStream
2025-11-28T14:51:59.344855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789118957546063:2050] [node 3] ICP30 unregister session Session# [5:7577789123252513460:2050] VirtualId# [5:7577789123252513455:0]
2025-11-28T14:51:59.344883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789123252513460:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.344966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:59.345007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:59.344683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:51:59.344867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789125321648058:2050] [node 3] ICS07 socket disconnect 342 reason# EndOfStream
2025-11-28T14:51:59.344897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789125321648058:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.344967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:51:59.345010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789125321648058:2050] [node 3] ICS15 start handshake
2025-11-28T14:51:59.349039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789121026680672:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9020) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:51:59.349104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789125321648058:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:51:59.349155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789121026680672:2050] [node 3] ICP30 unregister session Session# [6:7577789125321648058:2050] VirtualId# [6:7577789125321648048:0]
2025-11-28T14:51:59.349211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:51:59.349250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789121026680672:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9020) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:59.349263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9020) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:51:59.358620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789121026680672:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:51:59.370784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:51:59.371011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789123252513472:2108] [node 4] ICS01 socket: 368 reason# EndOfStream
2025-11-28T14:51:59.371056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789123252513439:2108] [node 4] ICP30 unregister session Session# [5:7577789123252513472:2108] VirtualId# [5:7577789123252513468:0]
2025-11-28T14:51:59.371162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789123252513472:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.371243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:59.371286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:59.372590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:51:59.372835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789125321648065:2093] [node 4] ICS01 socket: 369 reason# EndOfStream
2025-11-28T14:51:59.372882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789125321648015:2093] [node 4] ICP30 unregister session Session# [6:7577789125321648065:2093] VirtualId# [6:7577789125321648057:0]
2025-11-28T14:51:59.372926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789125321648065:2093] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.373025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:51:59.373201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:51:59.404377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:51:59.404642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789125321648069:2098] [node 5] ICS07 socket disconnect 370 reason# EndOfStream
2025-11-28T14:51:59.404678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789125321648069:2098] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:51:59.404782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:51:59.404851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789125321648069:2098] [node 5] ICS15 start handshake
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:51:59.537277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789124293903364:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:59.537680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789124293903363:2048] [node 2] ICP01 ready to work
2025-11-28T14:51:59.553673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789123646252649:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:59.553723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789123646252650:2049] [node 3] ICP01 ready to work
2025-11-28T14:51:59.573123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789123895349487:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:59.573175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789123895349488:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:59.594926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789124615908848:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:59.594975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789124615908849:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:59.595011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789124615908850:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:59.616440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789124698243727:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:59.616484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789124698243728:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:59.616507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789124698243729:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:59.650704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789126922769209:2049] [node 2] ICP01 ready to work
2025-11-28T14:51:59.650707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789126922769208:2048] [node 1] ICP01 ready to work
2025-11-28T14:51:59.650748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789126922769210:2050] [node 3] ICP01 ready to work
2025-11-28T14:51:59.751443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789126922769269:2106] [node 5] ICP01 ready to work
2025-11-28T14:51:59.757789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789124293903426:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:59.793536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789124293903429:2109] [node 5] ICP01 ready to work
2025-11-28T14:51:59.795959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789123895349560:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:59.798413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789123646252712:2107] [node 4] ICP01 ready to work
2025-11-28T14:51:59.803682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789123646252713:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:59.826631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789123895349561:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:59.826919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789126922769269:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(44397084ns) {Configure:104(44393867ns) {ProcessConfigured:135(8634432ns) {StartInitialHandshake:157(8630571ns) {DropHandshakes:470(1900236ns) {DropIncomingHandshake:442(762ns) DropOutgoingHandshake:457(2214ns)} GenerateSessionVirtualId:988(1631ns) PrepareNewSessionHandshake:477(3658ns) {SwitchToState:179(1776ns)}}}}}
2025-11-28T14:51:59.826631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789123646252714:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:59.827261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789126922769274:2110] [node 4] ICP01 ready to work
2025-11-28T14:51:59.826702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789124293903430:2110] [node 6] ICP01 ready to work
2025-11-28T14:51:59.832464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789124698243790:2108] [node 4] ICP01 ready to work
2025-11-28T14:51:59.833366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789124698243791:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:59.831402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789123895349562:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:59.838218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789124293903364:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(78629198ns) {Configure:104(78626057ns) {ProcessConfigured:135(2149914ns) {StartInitialHandshake:157(2145996ns) {DropHandshakes:470(2339ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(892ns) PrepareNewSessionHandshake:477(6163ns) {SwitchToState:179(1348ns)}}}}}
2025-11-28T14:51:59.841268Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789124615908848:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53187897ns) {Configure:104(53181360ns) {ProcessConfigured:135(646035ns) {StartInitialHandshake:157(644242ns) {DropHandshakes:470(1280ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(605ns) PrepareNewSessionHandshake:477(1526ns) {SwitchToState:179(792ns)}}}}}
2025-11-28T14:51:59.841612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789124615908849:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(53872207ns) {Configure:104(53854469ns) {ProcessConfigured:135(164957ns) {StartInitialHandshake:157(162378ns) {DropHandshakes:470(1565ns) {DropIncomingHandshake:442(435ns) DropOutgoingHandshake:457(540ns)} GenerateSessionVirtualId:988(457ns) PrepareNewSessionHandshake:477(1035ns) {SwitchToState:179(449ns)}}}}}
2025-11-28T14:51:59.843596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789124615908925:2108] [node 5] ICP01 ready to work
2025-11-28T14:51:59.843761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789124615908926:2109] [node 6] ICP01 ready to work
2025-11-28T14:51:59.866336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789124698243790:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17110712ns) {Configure:104(17103731ns) {ProcessConfigured:135(105866ns) {StartInitialHandshake:157(101982ns) {DropHandshakes:470(4822ns) {DropIncomingHandshake:442(1519ns) DropOutgoingHandshake:457(629ns)} GenerateSessionVirtualId:988(1319ns) PrepareNewSessionHandshake:477(2162ns) {SwitchToState:179(1006ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.878345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789124615908927:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55925
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.878570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789124615908928:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55925
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.882749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789123895349577:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15203
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.880840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789124698243728:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29959394ns) {Configure:104(29955492ns) {ProcessConfigured:135(70319ns) {StartInitialHandshake:157(68029ns) {DropHandshakes:470(3376ns) {DropIncomingHandshake:442(747ns) DropOutgoingHandshake:457(472ns)} GenerateSessionVirtualId:988(636ns) PrepareNewSessionHandshake:477(2941ns) {SwitchToState:179(1205ns)}}}}}
2025-11-28T14:51:59.881139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789124293903439:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32810
2025-11-28T14:51:59.881229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789124293903440:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32810
2025-11-28T14:51:59.882834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789123895349578:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15203
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.895014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789123646252713:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(46847013ns) {Configure:104(46841170ns) {ProcessConfigured:135(6246311ns) {StartInitialHandshake:157(6242586ns) {DropHandshakes:470(3308ns) {DropIncomingHandshake:442(746ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:477(844689ns) {SwitchToState:179(2614ns)}}}}}
2025-11-28T14:51:59.882888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789123895349579:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15203
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.881754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789124293903444:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32810
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.891157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789123895349576:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15203
2025-11-28T14:51:59.897793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789123646252727:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9343
2025-11-28T14:51:59.896331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789124615908927:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.890549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903439:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.899235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789124293903445:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32810
2025-11-28T14:51:59.899521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903432:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.896351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789123895349579:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.899545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789124293903432:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.900900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789124293903432:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:59.898481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789123895349487:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.898706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789123895349577:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.901387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789123646252728:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9343
2025-11-28T14:51:59.903165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789123646252727:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.903811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789123646252649:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.904790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789124698243791:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36375872ns) {Configure:104(36370448ns) {ProcessConfigured:135(1330517ns) {StartInitialHandshake:157(838847ns) {DropHandshakes:470(2757ns) {DropIncomingHandshake:442(989ns) DropOutgoingHandshake:457(457ns)} GenerateSessionVirtualId:988(1328ns) PrepareNewSessionHandshake:477(1874ns) {SwitchToState:179(922ns)}}}}}
2025-11-28T14:51:59.904703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789124615908925:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29774918ns) {Configure:104(29770786ns) {ProcessConfigured:135(561485ns) {StartInitialHandshake:157(558905ns) {DropHandshakes:470(1083ns) {DropIncomingHandshake:442(352ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:477(520175ns) {SwitchToState:179(1797ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.904709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789123895349576:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.914820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789123646252720:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.911532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903440:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.908413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789124698243800:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.912521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789124615908928:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.914857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789123646252720:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.909586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789123895349582:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15203
2025-11-28T14:51:59.915322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789123646252716:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.915949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903425:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.913812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789124698243800:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.915348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789123646252716:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.912715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789124615908936:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55925
2025-11-28T14:51:59.913926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789124698243801:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46188
2025-11-28T14:51:59.915987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789124293903425:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.909705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789123895349488:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.916050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789123646252728:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.912869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789124615908850:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.913306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789124615908849:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.913380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789124615908939:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55925
2025-11-28T14:51:59.923849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789124615908936:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.927532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789124615908848:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.914095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789124698243729:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:51:59.915744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789123895349584:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15203
2025-11-28T14:51:59.917280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903444:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.924031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789123646252720:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:59.925985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789123646252712:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.926232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789123646252716:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:51:59.927886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:51:59.931763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789124615908942:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55925
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:51:59.928441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789123646252712:2107] [node 4] ICP22 created new session: [2:7577789123646252734:2107]
2025-11-28T14:51:59.927822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789123895349568:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.917334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789124293903364:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.915761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789124698243801:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.932146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789123646252734:2107] [node 4] ICS09 handshake done sender: [2:7577789123646252720:2112] self: [2:7577789123646252719:0] peer: [4:7577789124615908941:0] socket: 318 qp: -1
2025-11-28T14:51:59.927861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789123895349568:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.932217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789123646252650:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.917404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:59.929770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789123895349564:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.939777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789123646252734:2107] [node 4] ICS10 traffic start
2025-11-28T14:51:59.962462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789124615908849:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32548) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789124615908928:2123] held: no
2025-11-28T14:51:59.929797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789123895349564:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.940628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:51:59.917458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789124293903364:2049] [node 3] ICP22 created new session: [1:7577789124293903449:2049]
2025-11-28T14:51:59.962839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789124615908928:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:59.961039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:51:59.940705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789123895349578:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.917525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789124293903449:2049] [node 3] ICS09 handshake done sender: [1:7577789124293903432:2111] self: [1:7577789124293903431:0] peer: [3:7577789123895349580:0] socket: 335 qp: -1
2025-11-28T14:51:59.961849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789123646252650:2049] [node 3] ICP22 created new session: [2:7577789123646252736:2049]
2025-11-28T14:51:59.967361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789124615908849:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.961890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789123895349579:2128] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:59.917559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789124293903449:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:59.967406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:59.965999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789123895349487:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10880) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789123895349579:2128] held: no
2025-11-28T14:51:59.962945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789123646252736:2049] [node 3] ICS09 handshake done sender: [2:7577789123646252716:2110] self: [2:7577789123646252715:0] peer: [3:7577789123895349583:0] socket: 316 qp: -1
2025-11-28T14:51:59.917596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:59.939514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789124293903448:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32810
2025-11-28T14:51:59.966296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789123895349487:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:51:59.967463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789124615908849:2049] [node 2] ICP22 created new session: [4:7577789124615908944:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.964532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789123646252736:2049] [node 3] ICS10 traffic start
2025-11-28T14:51:59.966336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:51:59.967596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789124615908944:2049] [node 2] ICS09 handshake done sender: [4:7577789124615908928:2123] self: [4:7577789124615908941:0] peer: [2:7577789123646252719:0] socket: 325 qp: -1
2025-11-28T14:51:59.965626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:51:59.939561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903445:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.967633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789124615908944:2049] [node 2] ICS10 traffic start
2025-11-28T14:51:59.966383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789123895349487:2048] [node 1] ICP22 created new session: [3:7577789123895349587:2048]
2025-11-28T14:51:59.969709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789123646252712:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(41574163ns) {HandleHandshakeStatus:329(41572799ns) {DropIncomingHandshake:442(10576ns) DropOutgoingHandshake:457(609ns) SwitchToState:179(132ns) ProcessPendingSessionEvents:495(2964570ns) {ForwardSessionEventToSession:612(1033119ns) ForwardSessionEventToSession:612(324203ns) ForwardSessionEventToSession:612(4089ns) ForwardSessionEventToSession:612(315255ns) ForwardSessionEventToSession:612(2928ns) ForwardSessionEventToSession:612(8379ns) ForwardSessionEventToSession:612(3356ns) ForwardSessionEventToSession:612(2231ns) ForwardSessionEventToSession:612(1002119ns) ForwardSessionEventToSession:612(256678ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.967669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:59.966456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789123895349587:2048] [node 1] ICS09 handshake done sender: [3:7577789123895349579:2128] self: [3:7577789123895349580:0] peer: [1:7577789124293903431:0] socket: 340 qp: -1
2025-11-28T14:51:59.961844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903434:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.961888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789124293903434:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:51:59.972858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789126922769286:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
2025-11-28T14:51:59.966489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789123895349587:2048] [node 1] ICS10 traffic start
2025-11-28T14:51:59.973815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789124615908947:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55925
2025-11-28T14:51:59.962968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789124293903425:2108] [node 2] ICH04 handshake succeeded
2025-11-28T14:51:59.971357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789124293903363:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.971598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789124293903434:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:51:59.971411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:51:59.972047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789124293903363:2048] [node 2] ICP22 created new session: [1:7577789124293903457:2048]
2025-11-28T14:51:59.972111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789124293903457:2048] [node 2] ICS09 handshake done sender: [1:7577789124293903425:2108] self: [1:7577789124293903424:0] peer: [2:7577789123646252730:0] socket: 324 qp: -1
2025-11-28T14:51:59.972261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789124293903457:2048] [node 2] ICS10 traffic start
2025-11-28T14:51:59.966527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:51:59.972831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:51:59.970785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789123895349585:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15203
2025-11-28T14:51:59.974425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789123895349568:2122] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.983458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789124615908948:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55925
2025-11-28T14:51:59.983902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789123646252650:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(39910278ns) {HandleHandshakeStatus:329(39908409ns) {DropIncomingHandshake:442(389ns) DropOutgoingHandshake:457(524ns) SwitchToState:179(543ns) ProcessPendingSessionEvents:495(2640542ns) {ForwardSessionEventToSession:612(1867853ns) ForwardSessionEventToSession:612(5416ns) ForwardSessionEventToSession:612(2499ns) ForwardSessionEventToSession:612(339944ns) ForwardSessionEventToSession:612(3251ns) ForwardSessionEventToSession:612(231230ns) ForwardSessionEventToSession:612(2798ns) ForwardSessionEventToSession:612(3170ns) ForwardSessionEventToSession:612(168317ns) ForwardSessionEventToSession:612(2554ns)}}}
2025-11-28T14:51:59.975802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789126922769286:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:51:59.974519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789123895349564:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:51:59.995640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789123895349576:2125] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:51:59.975812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789124293903426:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.983976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:00.001037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789124615908936:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:51:59.995230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789126922769274:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:00.000741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789123895349561:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:51:59.984048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789124293903426:2107] [node 4] ICP22 created new session: [1:7577789124293903459:2107]
2025-11-28T14:52:00.000809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:51:59.996078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789126922769288:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
2025-11-28T14:51:59.992166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789124293903459:2107] [node 4] ICS09 handshake done sender: [1:7577789124293903434:2112] self: [1:7577789124293903433:0] peer: [4:7577789124615908943:0] socket: 336 qp: -1
2025-11-28T14:51:59.996810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789124293903459:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:00.001237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:00.001860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789124615908932:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.001889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789124615908932:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:00.001368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789123895349561:2108] [node 5] ICP22 created new session: [3:7577789128190316888:2108]
2025-11-28T14:52:00.005476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789124293903363:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(25459616ns) {HandleHandshakeStatus:329(25287055ns) {DropIncomingHandshake:442(12716ns) DropOutgoingHandshake:457(755ns) SwitchToState:179(616ns) ProcessPendingSessionEvents:495(4786085ns) {ForwardSessionEventToSession:612(90448ns) ForwardSessionEventToSession:612(14343ns) ForwardSessionEventToSession:612(3197ns) ForwardSessionEventToSession:612(3325ns) ForwardSessionEventToSession:612(7521ns) ForwardSessionEventToSession:612(2429ns) ForwardSessionEventToSession:612(3098ns) ForwardSessionEventToSession:612(74580ns) ForwardSessionEventToSession:612(4560064ns) ForwardSessionEventToSession:612(12568ns)}}}
2025-11-28T14:52:00.002629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789124615908927:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:00.004744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789124615908848:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.006910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789124293903426:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(29670288ns) {HandleHandshakeStatus:329(29590536ns) {DropIncomingHandshake:442(13764ns) DropOutgoingHandshake:457(439ns) SwitchToState:179(457ns) ProcessPendingSessionEvents:495(941827ns) {ForwardSessionEventToSession:612(887675ns) ForwardSessionEventToSession:612(8328ns) ForwardSessionEventToSession:612(2140ns) ForwardSessionEventToSession:612(3101ns) ForwardSessionEventToSession:612(3841ns) ForwardSessionEventToSession:612(3599ns) ForwardSessionEventToSession:612(10870ns) ForwardSessionEventToSession:612(1880ns) ForwardSessionEventToSession:612(5521ns) ForwardSessionEventToSession:612(4986ns)}}}
2025-11-28T14:52:00.004790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:00.001431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789128190316888:2108] [node 5] ICS09 handshake done sender: [3:7577789123895349564:2120] self: [3:7577789123895349563:0] peer: [5:7577789124698243807:0] socket: 312 qp: -1
2025-11-28T14:52:00.001470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789128190316888:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:00.001516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:00.004846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789124615908848:2048] [node 1] ICP22 created new session: [4:7577789128910876245:2048]
2025-11-28T14:52:00.002194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789123895349560:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:00.002841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:00.003882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789123895349560:2107] [node 4] ICP22 created new session: [3:7577789128190316890:2107]
2025-11-28T14:52:00.004593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789128190316890:2107] [node 4] ICS09 handshake done sender: [3:7577789123895349568:2122] self: [3:7577789123895349567:0] peer: [4:7577789124615908940:0] socket: 314 qp: -1
2025-11-28T14:52:00.004598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789123895349488:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.004653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:00.005595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789128190316890:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:00.005759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:00.006672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789123895349488:2049] [node 2] ICP22 created new session: [3:7577789128190316891:2049]
2025-11-28T14:52:00.006729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789128190316891:2049] [node 2] ICS09 handshake done sender: [3:7577789123895349576:2125] self: [3:7577789123895349583:0] peer: [2:7577789123646252715:0] socket: 321 qp: -1
2025-11-28T14:52:00.006772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789128190316891:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:00.006811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:00.004898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789128910876245:2048] [node 1] ICS09 handshake done sender: [4:7577789124615908936:2126] self: [4:7577789124615908943:0] peer: [1:7577789124293903433:0] socket: 346 qp: -1
2025-11-28T14:52:00.004934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789128910876245:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:00.010328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789123646252649:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10880) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789123646252727:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.004965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:00.005197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789124615908850:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.013552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789123646252739:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9343
2025-11-28T14:52:00.005236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:00.005272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789124615908850:2050] [node 3] ICP22 created new session: [4:7577789128910876248:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.005371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789128910876248:2050] [node 3] ICS09 handshake done sender: [4:7577789124615908927:2122] self: [4:7577789124615908940:0] peer: [3:7577789123895349567:0] socket: 319 qp: -1
2025-11-28T14:52:00.005412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789128910876248:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:00.016600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789123646252727:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.017031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789131217736585:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
2025-11-28T14:52:00.005445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:00.017942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789124615908932:2125] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:00.017657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789131217736587:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.018207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789131217736585:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.018716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789127941220037:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9343
2025-11-28T14:52:00.018819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789123646252649:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.018234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789131217736586:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
2025-11-28T14:52:00.019061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:00.019096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903438:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.019168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789123646252649:2048] [node 1] ICP22 created new session: [2:7577789127941220038:2048]
2025-11-28T14:52:00.019132Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789124293903438:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:00.018670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789126922769208:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:00.019575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789124615908926:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:00.019306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789127941220038:2048] [node 1] ICS09 handshake done sender: [2:7577789123646252727:2114] self: [2:7577789123646252730:0] peer: [1:7577789124293903424:0] socket: 341 qp: -1
2025-11-28T14:52:00.019615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789131217736586:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.019438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789127941220038:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:00.020575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:00.020159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789126922769286:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:00.019494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:00.021257Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789124293903438:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:00.020828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789124615908926:2109] [node 6] ICP22 created new session: [4:7577789128910876251:2109]
2025-11-28T14:52:00.020914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789126922769274:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.021028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789128910876251:2109] [node 6] ICS09 handshake done sender: [4:7577789124615908932:2125] self: [4:7577789124615908931:0] peer: [6:7577789126922769287:0] socket: 361 qp: -1
2025-11-28T14:52:00.020957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:00.021411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789128910876251:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:00.021465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:00.021170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789126922769274:2110] [node 4] ICP22 created new session: [6:7577789131217736589:2110]
2025-11-28T14:52:00.021663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789126922769210:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:00.021730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789131217736589:2110] [node 4] ICS09 handshake done sender: [6:7577789126922769286:2115] self: [6:7577789126922769287:0] peer: [4:7577789124615908931:0] socket: 362 qp: -1
2025-11-28T14:52:00.021854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789131217736589:2110] [node 4] ICS10 traffic start
2025-11-28T14:52:00.023002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789124293903430:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:00.021898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:00.023072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:00.023575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789124293903430:2110] [node 6] ICP22 created new session: [1:7577789128588870759:2110]
2025-11-28T14:52:00.024013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789128588870759:2110] [node 6] ICS09 handshake done sender: [1:7577789124293903438:2114] self: [1:7577789124293903437:0] peer: [6:7577789131217736588:0] socket: 317 qp: -1
2025-11-28T14:52:00.024097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789128588870759:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:00.024158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:00.025844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789126922769288:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.025832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789123895349575:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.025874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789123895349575:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:00.027434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789126922769208:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10880) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789131217736585:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.027701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789126922769209:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:00.027701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789131217736592:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
2025-11-28T14:52:00.027958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789123895349575:2124] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:00.028211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789123895349562:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:00.028266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.028533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789123646252722:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.028520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789123895349562:2109] [node 6] ICP22 created new session: [3:7577789128190316896:2109]
2025-11-28T14:52:00.028575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789123646252722:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:00.028677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789127941220040:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9343
2025-11-28T14:52:00.028791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789128190316896:2109] [node 6] ICS09 handshake done sender: [3:7577789123895349575:2124] self: [3:7577789123895349569:0] peer: [6:7577789131217736590:0] socket: 333 qp: -1
2025-11-28T14:52:00.028818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789128190316896:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:00.028947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:00.028641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789131217736585:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:00.028993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789126922769208:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.029039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:00.029184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789126922769208:2048] [node 1] ICP22 created new session: [6:7577789131217736596:2048]
2025-11-28T14:52:00.029258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789131217736596:2048] [node 1] ICS09 handshake done sender: [6:7577789131217736585:2117] self: [6:7577789131217736588:0] peer: [1:7577789124293903437:0] socket: 367 qp: -1
2025-11-28T14:52:00.029304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789131217736596:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:00.029337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:00.030210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789123646252722:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:00.030563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789123646252714:2109] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:00.030687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:00.030704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789131217736595:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
2025-11-28T14:52:00.030774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789123646252714:2109] [node 6] ICP22 created new session: [2:7577789127941220042:2109]
2025-11-28T14:52:00.030820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789127941220042:2109] [node 6] ICS09 handshake done sender: [2:7577789123646252722:2113] self: [2:7577789123646252721:0] peer: [6:7577789131217736594:0] socket: 323 qp: -1
2025-11-28T14:52:00.030868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789127941220042:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:00.030901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:00.031502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789126922769210:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25403) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789131217736586:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.031881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789131217736586:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:00.031854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789127941220040:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.031923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789131217736598:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
2025-11-28T14:52:00.031984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789123646252714:2109] [node 6] ICP18 handshake (actor [2:7577789127941220040:2119]) is too old
2025-11-28T14:52:00.032105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789126922769210:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.032154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:00.032233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789126922769210:2050] [node 3] ICP22 created new session: [6:7577789131217736599:2050]
2025-11-28T14:52:00.032285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789131217736599:2050] [node 3] ICS09 handshake done sender: [6:7577789131217736586:2118] self: [6:7577789131217736590:0] peer: [3:7577789123895349569:0] socket: 368 qp: -1
2025-11-28T14:52:00.032311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789126922769288:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:00.032331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789131217736599:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:00.032380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:00.033079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789126922769209:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.033115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:00.033149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789126922769209:2049] [node 2] ICP22 created new session: [6:7577789131217736601:2049]
2025-11-28T14:52:00.033191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789131217736601:2049] [node 2] ICS09 handshake done sender: [6:7577789126922769288:2116] self: [6:7577789131217736594:0] peer: [2:7577789123646252721:0] socket: 366 qp: -1
2025-11-28T14:52:00.033213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789131217736601:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:00.033240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:00.033496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789126922769209:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:00.037695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789124698243797:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:00.060851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789124615908939:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.073526Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789124698243795:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:00.096095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789123895349584:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.096307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789123895349561:2108] [node 5] ICP18 handshake (actor [3:7577789123895349584:2130]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.096915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789128993211105:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.097090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789128993211106:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.097605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789128993211107:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.098275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789128993211108:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46188
2025-11-28T14:52:00.098483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789128993211105:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.098581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789124698243729:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25403) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789124698243800:2114] held: no
2025-11-28T14:52:00.098756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789128993211107:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.098904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789124698243800:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:00.099196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789128993211108:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.099440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789124698243729:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.099490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:00.099543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789124698243729:2050] [node 3] ICP22 created new session: [5:7577789128993211109:2050]
2025-11-28T14:52:00.099596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789128993211109:2050] [node 3] ICS09 handshake done sender: [5:7577789124698243800:2114] self: [5:7577789124698243807:0] peer: [3:7577789123895349563:0] socket: 334 qp: -1
2025-11-28T14:52:00.099626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789128993211109:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:00.099667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:00.100180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789124698243727:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:00.100261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789124698243728:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.100887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903436:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.100932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789124698243790:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:00.100926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789123646252729:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.101507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789124615908938:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.100948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789127941220044:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9343
2025-11-28T14:52:00.100935Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789124293903436:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:00.101540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789124615908938:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:00.100961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789123646252729:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:00.102808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789124615908938:2127] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:00.103567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789124615908925:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:00.103653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:00.103735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789124615908925:2108] [node 5] ICP22 created new session: [4:7577789128910876255:2108]
2025-11-28T14:52:00.103782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789128910876255:2108] [node 5] ICS09 handshake done sender: [4:7577789124615908938:2127] self: [4:7577789124615908937:0] peer: [5:7577789128993211114:0] socket: 330 qp: -1
2025-11-28T14:52:00.103817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789128910876255:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:00.103851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:00.103734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789123646252729:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:00.104074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789124293903436:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:00.104149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789123646252713:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:00.104187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:00.104254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789123646252713:2108] [node 5] ICP22 created new session: [2:7577789127941220046:2108]
2025-11-28T14:52:00.104295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789127941220046:2108] [node 5] ICS09 handshake done sender: [2:7577789123646252729:2116] self: [2:7577789123646252726:0] peer: [5:7577789128993211113:0] socket: 371 qp: -1
2025-11-28T14:52:00.104323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789127941220046:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:00.104391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:00.104494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789124293903429:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:00.104540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:00.104588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789124293903429:2109] [node 5] ICP22 created new session: [1:7577789128588870762:2109]
2025-11-28T14:52:00.104657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789128588870762:2109] [node 5] ICS09 handshake done sender: [1:7577789124293903436:2113] self: [1:7577789124293903435:0] peer: [5:7577789128993211112:0] socket: 342 qp: -1
2025-11-28T14:52:00.104684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789128588870762:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:00.104726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:00.111525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789124698243794:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:00.145271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789124293903448:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.145587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789124293903429:2109] [node 5] ICP18 handshake (actor [1:7577789124293903448:2119]) is too old
2025-11-28T14:52:00.146203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789124698243790:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30170) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789128993211108:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.146605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789128993211115:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.147065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789128993211116:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.148177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789128993211117:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46188
2025-11-28T14:52:00.148666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789124698243727:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10880) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789128993211105:2117] held: no
2025-11-28T14:52:00.148845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789128993211108:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:00.149054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789128993211107:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:00.149388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789128993211105:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:00.149729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789124698243790:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.149780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:00.149822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789124698243790:2108] [node 4] ICP22 created new session: [5:7577789128993211119:2108]
2025-11-28T14:52:00.149880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789128993211119:2108] [node 4] ICS09 handshake done sender: [5:7577789128993211108:2120] self: [5:7577789128993211114:0] peer: [4:7577789124615908937:0] socket: 375 qp: -1
2025-11-28T14:52:00.149912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789128993211119:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:00.149952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:00.150371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789124698243728:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.150412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:00.150453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789124698243728:2049] [node 2] ICP22 created new session: [5:7577789128993211121:2049]
2025-11-28T14:52:00.150490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789128993211121:2049] [node 2] ICS09 handshake done sender: [5:7577789128993211107:2119] self: [5:7577789128993211113:0] peer: [2:7577789123646252726:0] socket: 372 qp: -1
2025-11-28T14:52:00.150520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789128993211121:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:00.150549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:00.150999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789124698243727:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.151037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:00.151073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789124698243727:2048] [node 1] ICP22 created new session: [5:7577789128993211123:2048]
2025-11-28T14:52:00.151111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789128993211123:2048] [node 1] ICS09 handshake done sender: [5:7577789128993211105:2117] self: [5:7577789128993211112:0] peer: [1:7577789124293903435:0] socket: 327 qp: -1
2025-11-28T14:52:00.151135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789128993211123:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:00.151156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.151701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789131217736603:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
2025-11-28T14:52:00.153959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789124698243799:2113] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:00.156553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789127941220044:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.183568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789123646252713:2108] [node 5] ICP18 handshake (actor [2:7577789127941220044:2120]) is too old
2025-11-28T14:52:00.204546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789124698243803:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:00.205512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789131217736603:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.205767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789126922769269:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:00.206030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789124698243803:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:00.206638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789124698243803:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:00.206894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789124698243791:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:00.206949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789131217736605:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29458
2025-11-28T14:52:00.206964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:00.207114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789124698243791:2109] [node 6] ICP22 created new session: [5:7577789128993211126:2109]
2025-11-28T14:52:00.207181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789128993211126:2109] [node 6] ICS09 handshake done sender: [5:7577789124698243803:2116] self: [5:7577789124698243802:0] peer: [6:7577789131217736604:0] socket: 349 qp: -1
2025-11-28T14:52:00.207223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789128993211126:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:00.207209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789131217736603:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:00.207263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:00.207806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789126922769269:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:00.207852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:00.207894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789126922769269:2106] [node 5] ICP22 created new session: [6:7577789131217736606:2106]
2025-11-28T14:52:00.207972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789131217736606:2106] [node 5] ICS09 handshake done sender: [6:7577789131217736603:2123] self: [6:7577789131217736604:0] peer: [5:7577789124698243802:0] socket: 351 qp: -1
2025-11-28T14:52:00.208010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789131217736606:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:00.208069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:00.208378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789126922769269:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:00.227536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:00.227799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789127941220038:2048] [node 1] ICS01 socket: 341 reason# EndOfStream
2025-11-28T14:52:00.227853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789123646252649:2048] [node 1] ICP30 unregister session Session# [2:7577789127941220038:2048] VirtualId# [2:7577789123646252730:0]
2025-11-28T14:52:00.227887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789127941220038:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.227989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:00.228041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:00.229077Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:00.229274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789123895349587:2048] [node 1] ICS01 socket: 340 reason# EndOfStream
2025-11-28T14:52:00.229320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789123895349487:2048] [node 1] ICP30 unregister session Session# [3:7577789123895349587:2048] VirtualId# [3:7577789123895349580:0]
2025-11-28T14:52:00.229354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789123895349587:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.229431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:00.229481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:00.231679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:00.231943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789128910876245:2048] [node 1] ICS07 socket disconnect 346 reason# EndOfStream
2025-11-28T14:52:00.231994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789128910876245:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.232084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:00.232134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789128910876245:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:00.232464Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:00.232719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789128993211123:2048] [node 1] ICS01 socket: 327 reason# EndOfStream
2025-11-28T14:52:00.232762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789124698243727:2048] [node 1] ICP30 unregister session Session# [5:7577789128993211123:2048] VirtualId# [5:7577789128993211112:0]
2025-11-28T14:52:00.232796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789128993211123:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.232864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:00.232924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:00.232860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789124615908848:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10880) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:00.233061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789128910876245:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:00.233098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789124615908848:2048] [node 1] ICP30 unregister session Session# [4:7577789128910876245:2048] VirtualId# [4:7577789124615908943:0]
2025-11-28T14:52:00.233149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:00.233200Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789124615908848:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10880) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:00.233225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10880) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:00.233588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:00.233781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789131217736596:2048] [node 1] ICS01 socket: 367 reason# EndOfStream
2025-11-28T14:52:00.233839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789126922769208:2048] [node 1] ICP30 unregister session Session# [6:7577789131217736596:2048] VirtualId# [6:7577789131217736588:0]
2025-11-28T14:52:00.233872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789131217736596:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.233962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:00.234114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:00.243400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789124615908848:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:00.263635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:00.263892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789128190316891:2049] [node 2] ICS07 socket disconnect 321 reason# EndOfStream
2025-11-28T14:52:00.263927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789128190316891:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.264079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:00.264145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789128190316891:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:00.265143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789123895349488:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32548) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:00.265235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789128190316891:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:00.265296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789123895349488:2049] [node 2] ICP30 unregister session Session# [3:7577789128190316891:2049] VirtualId# [3:7577789123895349583:0]
2025-11-28T14:52:00.265356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:00.265392Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789123895349488:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32548) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:00.265405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32548) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:00.266351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:00.266564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789124615908944:2049] [node 2] ICS01 socket: 325 reason# EndOfStream
2025-11-28T14:52:00.266609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789124615908849:2049] [node 2] ICP30 unregister session Session# [4:7577789124615908944:2049] VirtualId# [4:7577789124615908941:0]
2025-11-28T14:52:00.266643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789124615908944:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.266735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:00.266783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:00.267098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:00.267281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789128993211121:2049] [node 2] ICS01 socket: 372 reason# EndOfStream
2025-11-28T14:52:00.267328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789124698243728:2049] [node 2] ICP30 unregister session Session# [5:7577789128993211121:2049] VirtualId# [5:7577789128993211113:0]
2025-11-28T14:52:00.267361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789128993211121:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.267435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:00.267468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:00.267763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:00.268170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789131217736601:2049] [node 2] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-28T14:52:00.268227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789131217736601:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.268311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:00.268393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789131217736601:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:00.269552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789126922769209:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32548) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:00.269685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789131217736601:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:00.269727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789126922769209:2049] [node 2] ICP30 unregister session Session# [6:7577789131217736601:2049] VirtualId# [6:7577789131217736594:0]
2025-11-28T14:52:00.269767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:00.269799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789126922769209:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32548) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:00.269816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32548) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:00.275173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789123895349488:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:00.279217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789126922769209:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:00.307922Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:00.308198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789128910876248:2050] [node 3] ICS07 socket disconnect 319 reason# EndOfStream
2025-11-28T14:52:00.308235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789128910876248:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.308321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:00.308383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789128910876248:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:00.308997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:00.309177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789128993211109:2050] [node 3] ICS01 socket: 334 reason# EndOfStream
2025-11-28T14:52:00.309222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789124615908850:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25403) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:00.309255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789124698243729:2050] [node 3] ICP30 unregister session Session# [5:7577789128993211109:2050] VirtualId# [5:7577789124698243807:0]
2025-11-28T14:52:00.309302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789128993211109:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.309344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789128910876248:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:00.309376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789124615908850:2050] [node 3] ICP30 unregister session Session# [4:7577789128910876248:2050] VirtualId# [4:7577789124615908940:0]
2025-11-28T14:52:00.309418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:00.309430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:00.309478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:00.309480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789124615908850:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25403) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:00.309500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25403) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:00.310002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:00.310131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789131217736599:2050] [node 3] ICS01 socket: 368 reason# EndOfStream
2025-11-28T14:52:00.310159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789126922769210:2050] [node 3] ICP30 unregister session Session# [6:7577789131217736599:2050] VirtualId# [6:7577789131217736590:0]
2025-11-28T14:52:00.310181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789131217736599:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.310251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:00.310289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:00.340520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:00.340784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789128993211119:2108] [node 4] ICS01 socket: 375 reason# EndOfStream
2025-11-28T14:52:00.340835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789124698243790:2108] [node 4] ICP30 unregister session Session# [5:7577789128993211119:2108] VirtualId# [5:7577789128993211114:0]
2025-11-28T14:52:00.340879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789128993211119:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.340969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:00.341022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:00.344531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:00.344851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789131217736589:2110] [node 4] ICS01 socket: 362 reason# EndOfStream
2025-11-28T14:52:00.344911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789126922769274:2110] [node 4] ICP30 unregister session Session# [6:7577789131217736589:2110] VirtualId# [6:7577789126922769287:0]
2025-11-28T14:52:00.344943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789131217736589:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.345068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:00.345142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:00.362336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:00.362551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789131217736606:2106] [node 5] ICS07 socket disconnect 351 reason# EndOfStream
2025-11-28T14:52:00.362597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789131217736606:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:00.362681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:00.362742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789131217736606:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:00.363608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789126922769269:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27828) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:00.363727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789131217736606:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:00.363772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789126922769269:2106] [node 5] ICP30 unregister session Session# [6:7577789131217736606:2106] VirtualId# [6:7577789131217736604:0]
2025-11-28T14:52:00.363823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:00.363862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789126922769269:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27828) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:00.363875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27828) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:00.439017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789127844655478:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:00.439058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789127844655479:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:00.458942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789128747275454:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:00.459450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789128747275453:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:00.480587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789129372618689:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:00.480716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789129372618690:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:00.506789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789128829411697:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:00.506831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789128829411699:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:00.506898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789128829411698:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:00.531796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789130874699598:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:00.531840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789130874699599:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:00.532005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789130874699600:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:00.554567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789128861248218:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:00.554619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789128861248219:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:00.555437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789128861248220:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:00.859027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789128861248280:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:00.860545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789129372618750:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:00.869869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789127844655541:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:00.881036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789130874699661:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:00.946929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789127844655542:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:00.982073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789128829411760:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:00.946750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789130874699664:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:00.980247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789128747275516:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:00.982884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789127844655543:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:00.983809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789128747275517:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:00.991392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789128829411761:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:00.987802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789127844655478:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(147288443ns) {Configure:104(147281477ns) {ProcessConfigured:135(90938ns) {StartInitialHandshake:157(87015ns) {DropHandshakes:470(8385ns) {DropIncomingHandshake:442(862ns) DropOutgoingHandshake:457(516ns)} GenerateSessionVirtualId:988(1102ns) PrepareNewSessionHandshake:477(1425ns) {SwitchToState:179(539ns)}}}}}
2025-11-28T14:52:00.986215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789128747275518:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:00.984077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789128861248282:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:00.997727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789128747275454:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(148929843ns) {Configure:104(142045525ns) {ProcessConfigured:135(682633ns) {StartInitialHandshake:157(680481ns) {DropHandshakes:470(1472ns) {DropIncomingHandshake:442(427ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(725ns) PrepareNewSessionHandshake:477(1592ns) {SwitchToState:179(209ns)}}}}}
2025-11-28T14:52:00.998724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789130874699600:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(109479769ns) {Configure:104(109475824ns) {ProcessConfigured:135(8653623ns) {StartInitialHandshake:157(8651792ns) {DropHandshakes:470(2558ns) {DropIncomingHandshake:442(761ns) DropOutgoingHandshake:457(463ns)} GenerateSessionVirtualId:988(894ns) PrepareNewSessionHandshake:477(2990ns) {SwitchToState:179(364ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.010576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789129372618689:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(145289168ns) {Configure:104(145282855ns) {ProcessConfigured:135(1620436ns) {StartInitialHandshake:157(1614679ns) {DropHandshakes:470(2648ns) {DropIncomingHandshake:442(644ns) DropOutgoingHandshake:457(358ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:477(2381ns) {SwitchToState:179(877ns)}}}}}
2025-11-28T14:52:01.011569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789129372618690:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(155942343ns) {Configure:104(155935718ns) {ProcessConfigured:135(60255ns) {StartInitialHandshake:157(58108ns) {DropHandshakes:470(944ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(524ns) PrepareNewSessionHandshake:477(2711ns) {SwitchToState:179(1043ns)}}}}}
2025-11-28T14:52:01.042164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789128747275453:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(168149483ns) {Configure:104(168143835ns) {ProcessConfigured:135(1677359ns) {StartInitialHandshake:157(1676011ns) {DropHandshakes:470(1417ns) {DropIncomingHandshake:442(497ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(365ns) PrepareNewSessionHandshake:477(1810ns) {SwitchToState:179(943ns)}}}}}
2025-11-28T14:52:01.011903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789133667586062:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:01.052927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789128861248218:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(69912588ns) {Configure:104(69908202ns) {ProcessConfigured:135(8230045ns) {StartInitialHandshake:157(8226366ns) {DropHandshakes:470(4004ns) {DropIncomingHandshake:442(861ns) DropOutgoingHandshake:457(646ns)} GenerateSessionVirtualId:988(1384ns) PrepareNewSessionHandshake:477(3808ns) {SwitchToState:179(2166ns)}}}}}
2025-11-28T14:52:01.020760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789135169666970:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4699
2025-11-28T14:52:01.071970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789128747275516:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(78516525ns) {Configure:104(78510190ns) {ProcessConfigured:135(15191829ns) {StartInitialHandshake:157(15189095ns) {DropHandshakes:470(1289ns) {DropIncomingHandshake:442(291ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(575ns) PrepareNewSessionHandshake:477(2707ns) {SwitchToState:179(238ns)}}}}}
2025-11-28T14:52:01.045039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789127844655479:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(174578438ns) {Configure:104(173237082ns) {ProcessConfigured:135(4401226ns) {StartInitialHandshake:157(4389677ns) {DropHandshakes:470(3381ns) {DropIncomingHandshake:442(719ns) DropOutgoingHandshake:457(412ns)} GenerateSessionVirtualId:988(798ns) PrepareNewSessionHandshake:477(2513ns) {SwitchToState:179(1034ns)}}}}}
2025-11-28T14:52:01.071172Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789128861248280:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(90368727ns) {Configure:104(90361030ns) {ProcessConfigured:135(14382951ns) {StartInitialHandshake:157(14377519ns) {DropHandshakes:470(2635ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(439ns)} GenerateSessionVirtualId:988(1465ns) PrepareNewSessionHandshake:477(4459ns) {SwitchToState:179(2293ns)}}}}}
2025-11-28T14:52:01.046627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789130874699598:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(62998466ns) {Configure:104(62987680ns) {ProcessConfigured:135(86714ns) {StartInitialHandshake:157(82680ns) {DropHandshakes:470(3052ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(638ns)} GenerateSessionVirtualId:988(669ns) PrepareNewSessionHandshake:477(2013ns) {SwitchToState:179(744ns)}}}}}
2025-11-28T14:52:01.074312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789128829411697:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(168023676ns) {Configure:104(168017414ns) {ProcessConfigured:135(16640143ns) {StartInitialHandshake:157(16637231ns) {DropHandshakes:470(1261ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(3373ns) {SwitchToState:179(1468ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.017131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789133667586069:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:01.078943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789135169666970:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.083434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789127844655547:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.074246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789129372618750:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(187128148ns) {Configure:104(187124274ns) {ProcessConfigured:135(31308131ns) {StartInitialHandshake:157(31305285ns) {DropHandshakes:470(872ns) {DropIncomingHandshake:442(212ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(435ns) PrepareNewSessionHandshake:477(3961ns) {SwitchToState:179(1718ns)}}}}}
2025-11-28T14:52:01.079609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789130874699598:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.085961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789128829411699:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(85218094ns) {Configure:104(85213142ns) {ProcessConfigured:135(3219445ns) {StartInitialHandshake:157(3215184ns) {DropHandshakes:470(2480ns) {DropIncomingHandshake:442(586ns) DropOutgoingHandshake:457(505ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:477(3788ns) {SwitchToState:179(1504ns)}}}}}
2025-11-28T14:52:01.084729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789135169666974:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4699
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.083479Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789127844655547:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.109249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789133042242818:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.087740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789127844655541:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(82000794ns) {Configure:104(81993191ns) {ProcessConfigured:135(10316198ns) {StartInitialHandshake:157(10310815ns) {DropHandshakes:470(3036ns) {DropIncomingHandshake:442(575ns) DropOutgoingHandshake:457(427ns)} GenerateSessionVirtualId:988(1012ns) PrepareNewSessionHandshake:477(3513ns) {SwitchToState:179(1600ns)}}}}}
2025-11-28T14:52:01.109477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789128747275521:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31
2025-11-28T14:52:01.097514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215587:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.110148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789128829411698:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(105403617ns) {Configure:104(105399019ns) {ProcessConfigured:135(9718369ns) {StartInitialHandshake:157(9713516ns) {DropHandshakes:470(2959ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(677ns)} GenerateSessionVirtualId:988(656ns) PrepareNewSessionHandshake:477(3127ns) {SwitchToState:179(1309ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.088207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789127844655547:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:01.120393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789128829411761:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30942510ns) {Configure:104(30928758ns) {ProcessConfigured:135(84735ns) {StartInitialHandshake:157(81834ns) {DropHandshakes:470(3052ns) {DropIncomingHandshake:442(692ns) DropOutgoingHandshake:457(555ns)} GenerateSessionVirtualId:988(800ns) PrepareNewSessionHandshake:477(1468ns) {SwitchToState:179(429ns)}}}}}
2025-11-28T14:52:01.122278Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789133667586069:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(74030910ns) {Configure:104(74021357ns) {ProcessConfigured:135(5617357ns) {StartInitialHandshake:157(5613699ns) {DropHandshakes:470(2574ns) {DropIncomingHandshake:442(559ns) DropOutgoingHandshake:457(381ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:477(3231ns) {SwitchToState:179(794ns)}}}}}
2025-11-28T14:52:01.097770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215588:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.122337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789135169666970:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:01.121709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789128747275521:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.088629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789132139622851:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31080
2025-11-28T14:52:01.097915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215590:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.133261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789130874699598:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.123678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789133042242818:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.123161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789127844655542:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.110331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789128861248219:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(96257947ns) {Configure:104(96253264ns) {ProcessConfigured:135(3727139ns) {StartInitialHandshake:157(3723645ns) {DropHandshakes:470(2005ns) {DropIncomingHandshake:442(481ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(1322ns) PrepareNewSessionHandshake:477(2523ns) {SwitchToState:179(1601ns)}}}}}
2025-11-28T14:52:01.133324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:01.123916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789133042242821:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31
2025-11-28T14:52:01.124106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789128747275453:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.137518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789133042242821:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.123222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:01.119554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789133156215587:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.135863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789130874699598:2048] [node 1] ICP22 created new session: [5:7577789135169666976:2048]
2025-11-28T14:52:01.139060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789135169666976:2048] [node 1] ICS09 handshake done sender: [5:7577789135169666970:2114] self: [5:7577789135169666973:0] peer: [1:7577789127844655546:0] socket: 324 qp: -1
2025-11-28T14:52:01.139076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789130874699666:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.139107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789130874699666:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:01.139533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789135169666976:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:01.140039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.140949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789128829411760:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(60122482ns) {Configure:104(60115981ns) {ProcessConfigured:135(365520ns) {StartInitialHandshake:157(362062ns) {DropHandshakes:470(2308ns) {DropIncomingHandshake:442(286ns) DropOutgoingHandshake:457(580ns)} GenerateSessionVirtualId:988(570ns) PrepareNewSessionHandshake:477(1449ns) {SwitchToState:179(524ns)}}}}}
2025-11-28T14:52:01.124642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789127844655542:2108] [node 5] ICP22 created new session: [1:7577789132139622856:2108]
2025-11-28T14:52:01.141783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789133667586064:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.141894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789135169666975:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4699
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.142585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789133124379067:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8525
2025-11-28T14:52:01.141832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789133667586064:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:01.128860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789133156215588:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.124680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789132139622851:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.149022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789133042242834:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.132904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789132139622856:2108] [node 5] ICS09 handshake done sender: [1:7577789127844655547:2111] self: [1:7577789127844655546:0] peer: [5:7577789135169666973:0] socket: 320 qp: -1
2025-11-28T14:52:01.134814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789132139622856:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:01.135333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789127844655545:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.135376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789127844655545:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:01.135558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:01.143194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789127844655549:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.143214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789127844655549:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:01.145951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789132139622857:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31080
2025-11-28T14:52:01.146373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789127844655545:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:01.134689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789128861248218:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.145870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789133667586073:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64750
2025-11-28T14:52:01.151228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789127844655542:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18713338ns) {HandleHandshakeStatus:329(18709445ns) {DropIncomingHandshake:442(995ns) DropOutgoingHandshake:457(721ns) SwitchToState:179(1287ns) ProcessPendingSessionEvents:495(2190087ns) {ForwardSessionEventToSession:612(1245028ns) ForwardSessionEventToSession:612(362129ns) ForwardSessionEventToSession:612(1999ns) ForwardSessionEventToSession:612(2127ns) ForwardSessionEventToSession:612(541979ns) ForwardSessionEventToSession:612(2267ns) ForwardSessionEventToSession:612(7348ns) ForwardSessionEventToSession:612(1721ns) ForwardSessionEventToSession:612(7898ns) ForwardSessionEventToSession:612(6135ns)}}}
2025-11-28T14:52:01.150797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789133124379067:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.152255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789133042242818:2112] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:01.135854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789128861248280:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.152179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789130874699666:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:01.152439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789132139622855:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31080
2025-11-28T14:52:01.180469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789130874699661:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.180562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.151416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789133667586064:2121] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.160855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789133667586074:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64750
2025-11-28T14:52:01.136485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789133156215590:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.152594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789127844655478:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.194849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789133124379074:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8525
2025-11-28T14:52:01.180722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789128747275453:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.139313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789128861248220:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.184513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789130874699598:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18588920ns) {HandleHandshakeStatus:329(18585362ns) {DropIncomingHandshake:442(705ns) DropOutgoingHandshake:457(6985ns) SwitchToState:179(742ns) ProcessPendingSessionEvents:495(9937133ns) {ForwardSessionEventToSession:612(3048255ns) ForwardSessionEventToSession:612(9667ns) ForwardSessionEventToSession:612(2968ns) ForwardSessionEventToSession:612(3285ns) ForwardSessionEventToSession:612(590389ns) ForwardSessionEventToSession:612(4082ns) ForwardSessionEventToSession:612(3550ns) ForwardSessionEventToSession:612(5653665ns) ForwardSessionEventToSession:612(7879ns) ForwardSessionEventToSession:612(597415ns)}}}
2025-11-28T14:52:01.212566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789133124379064:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8525
2025-11-28T14:52:01.172443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789133667586073:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.181195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789133667586062:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.181243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:01.181303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789133667586062:2109] [node 6] ICP22 created new session: [3:7577789133667586079:2109]
2025-11-28T14:52:01.181383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789133667586079:2109] [node 6] ICS09 handshake done sender: [3:7577789133667586064:2121] self: [3:7577789133667586063:0] peer: [6:7577789133156215597:0] socket: 331 qp: -1
2025-11-28T14:52:01.181422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789133667586079:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:01.181473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:01.182332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789133667586076:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64750
2025-11-28T14:52:01.182525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789133667586078:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64750
2025-11-28T14:52:01.182667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789133667586062:2109] [node 6] ICP18 handshake (actor [3:7577789133667586073:2124]) is too old
2025-11-28T14:52:01.180790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:01.156330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:01.200430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789130874699661:2109] [node 6] ICP22 created new session: [5:7577789135169666980:2109]
2025-11-28T14:52:01.206523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789128747275453:2048] [node 1] ICP22 created new session: [2:7577789133042242835:2048]
2025-11-28T14:52:01.172526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789127844655478:2048] [node 2] ICP22 created new session: [1:7577789132139622864:2048]
2025-11-28T14:52:01.225720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789128861248220:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61180) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789133156215590:2116] held: no
2025-11-28T14:52:01.228751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789133124379064:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.208479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789135169666980:2109] [node 6] ICS09 handshake done sender: [5:7577789130874699666:2111] self: [5:7577789130874699665:0] peer: [6:7577789133156215596:0] socket: 322 qp: -1
2025-11-28T14:52:01.184496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789132139622864:2048] [node 2] ICS09 handshake done sender: [1:7577789127844655545:2110] self: [1:7577789127844655544:0] peer: [2:7577789133042242832:0] socket: 319 qp: -1
2025-11-28T14:52:01.212473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789133042242835:2048] [node 1] ICS09 handshake done sender: [2:7577789133042242818:2112] self: [2:7577789133042242832:0] peer: [1:7577789127844655544:0] socket: 327 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.233032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789133124379074:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:01.208568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789135169666980:2109] [node 6] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:01.217767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789133042242835:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:01.200508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789132139622864:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:01.212913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:01.218139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:01.216588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.228134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789130874699661:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(55350149ns) {HandleHandshakeStatus:329(55346379ns) {DropIncomingHandshake:442(2355ns) DropOutgoingHandshake:457(1246ns) SwitchToState:179(1159ns) ProcessPendingSessionEvents:495(2908472ns) {ForwardSessionEventToSession:612(2828069ns) ForwardSessionEventToSession:612(12238ns) ForwardSessionEventToSession:612(3946ns) ForwardSessionEventToSession:612(7645ns) ForwardSessionEventToSession:612(9397ns) ForwardSessionEventToSession:612(3135ns) ForwardSessionEventToSession:612(17470ns) ForwardSessionEventToSession:612(2325ns) ForwardSessionEventToSession:612(2518ns) ForwardSessionEventToSession:612(7456ns)}}}
2025-11-28T14:52:01.233900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789133667586077:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64750
2025-11-28T14:52:01.217287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789127844655478:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(64434835ns) {HandleHandshakeStatus:329(64431370ns) {DropIncomingHandshake:442(433ns) DropOutgoingHandshake:457(587ns) SwitchToState:179(560ns) ProcessPendingSessionEvents:495(180863ns) {ForwardSessionEventToSession:612(85501ns) ForwardSessionEventToSession:612(13470ns) ForwardSessionEventToSession:612(3190ns) ForwardSessionEventToSession:612(3159ns) ForwardSessionEventToSession:612(20356ns) ForwardSessionEventToSession:612(2443ns) ForwardSessionEventToSession:612(18724ns) ForwardSessionEventToSession:612(2776ns) ForwardSessionEventToSession:612(2537ns) ForwardSessionEventToSession:612(9081ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.233888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215598:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.234032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789133667586078:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.217488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789127844655549:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:01.226496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789132139622855:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.233008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789127844655543:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.234005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215599:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.234038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789128829411699:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.233467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:01.233953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789127844655543:2109] [node 6] ICP22 created new session: [1:7577789132139622867:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.236254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789133667586070:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.237328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789128747275453:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(52851597ns) {HandleHandshakeStatus:329(52847970ns) {DropIncomingHandshake:442(878ns) DropOutgoingHandshake:457(5613ns) SwitchToState:179(285ns) ProcessPendingSessionEvents:495(899138ns) {ForwardSessionEventToSession:612(242301ns) ForwardSessionEventToSession:612(3521ns) ForwardSessionEventToSession:612(1748ns) ForwardSessionEventToSession:612(115545ns) ForwardSessionEventToSession:612(388879ns) ForwardSessionEventToSession:612(3352ns) ForwardSessionEventToSession:612(9316ns) ForwardSessionEventToSession:612(3277ns) ForwardSessionEventToSession:612(118726ns) ForwardSessionEventToSession:612(3349ns)}}}
2025-11-28T14:52:01.234010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789132139622867:2109] [node 6] ICS09 handshake done sender: [1:7577789127844655549:2112] self: [1:7577789127844655548:0] peer: [6:7577789133156215595:0] socket: 321 qp: -1
2025-11-28T14:52:01.234865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789132139622863:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31080
2025-11-28T14:52:01.236274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789133667586070:2122] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.236202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789132139622867:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:01.237523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215600:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.236247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:01.236383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789129372618689:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.237575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789132139622850:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.237472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789133667586070:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:01.237610Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789132139622850:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:01.237635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789133156215588:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:01.237619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789127844655543:2109] [node 6] ICP18 handshake (actor [1:7577789132139622855:2116]) is too old
2025-11-28T14:52:01.237689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789129372618750:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.237651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215601:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.237731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:01.237835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789133156215590:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.237870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789129372618750:2107] [node 4] ICP22 created new session: [3:7577789133667586084:2107]
2025-11-28T14:52:01.238350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789128861248220:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.237953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789133667586084:2107] [node 4] ICS09 handshake done sender: [3:7577789133667586070:2122] self: [3:7577789133667586068:0] peer: [4:7577789133124379076:0] socket: 373 qp: -1
2025-11-28T14:52:01.238750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:01.237999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789133667586084:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:01.239720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789133124379077:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8525
2025-11-28T14:52:01.239613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789128861248220:2050] [node 3] ICP22 created new session: [6:7577789133156215603:2050]
2025-11-28T14:52:01.238119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:01.239723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789128861248280:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.239850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:01.239877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789128861248280:2106] [node 5] ICP22 created new session: [6:7577789133156215604:2106]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.240745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789132139622850:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:01.239940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789133156215603:2050] [node 3] ICS09 handshake done sender: [6:7577789133156215590:2116] self: [6:7577789133156215597:0] peer: [3:7577789133667586063:0] socket: 337 qp: -1
2025-11-28T14:52:01.239940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789133156215604:2106] [node 5] ICS09 handshake done sender: [6:7577789133156215588:2115] self: [6:7577789133156215596:0] peer: [5:7577789130874699665:0] socket: 336 qp: -1
2025-11-28T14:52:01.241304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789135169666982:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4699
2025-11-28T14:52:01.241301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789127844655479:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:01.239974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789133156215603:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:01.241484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:01.239980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789133156215604:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:01.239997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:01.241535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789127844655479:2049] [node 3] ICP22 created new session: [1:7577789132139622870:2049]
2025-11-28T14:52:01.241983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789135169666983:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4699
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.240046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:01.241594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789132139622870:2049] [node 3] ICS09 handshake done sender: [1:7577789132139622850:2113] self: [1:7577789132139622849:0] peer: [3:7577789133667586082:0] socket: 359 qp: -1
2025-11-28T14:52:01.241980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789132139622870:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:01.241973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789133156215587:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:01.242021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:01.242259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789133667586087:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64750
2025-11-28T14:52:01.242796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789128861248218:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789133156215587:2114] held: no
2025-11-28T14:52:01.243221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789128861248218:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.243443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:01.243645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789128861248218:2048] [node 1] ICP22 created new session: [6:7577789133156215608:2048]
2025-11-28T14:52:01.243774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789133156215608:2048] [node 1] ICS09 handshake done sender: [6:7577789133156215587:2114] self: [6:7577789133156215595:0] peer: [1:7577789127844655548:0] socket: 335 qp: -1
2025-11-28T14:52:01.243881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789133667586078:2128] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:01.243824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789133156215608:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:01.243894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:01.244169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789129372618689:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789133667586078:2128] held: no
2025-11-28T14:52:01.244339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789129372618689:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.244697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:01.244736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789129372618689:2048] [node 1] ICP22 created new session: [3:7577789133667586088:2048]
2025-11-28T14:52:01.244779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789133667586088:2048] [node 1] ICS09 handshake done sender: [3:7577789133667586078:2128] self: [3:7577789133667586082:0] peer: [1:7577789132139622849:0] socket: 370 qp: -1
2025-11-28T14:52:01.244804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789133667586088:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:01.244937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:01.251794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789133042242823:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:01.252635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789135169666975:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.252816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789130874699599:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.264782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789133042242823:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.265744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789133042242837:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31
2025-11-28T14:52:01.268329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789133042242820:2113] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.339159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789135169666985:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4699
2025-11-28T14:52:01.296472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789133042242823:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:01.336825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789133042242837:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.336894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789128747275517:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.337001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:01.337063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789128747275517:2108] [node 5] ICP22 created new session: [2:7577789133042242839:2108]
2025-11-28T14:52:01.337125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789133042242839:2108] [node 5] ICS09 handshake done sender: [2:7577789133042242823:2115] self: [2:7577789133042242822:0] peer: [5:7577789135169666984:0] socket: 346 qp: -1
2025-11-28T14:52:01.337171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789133042242839:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:01.337216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:01.342176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789132139622857:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.342385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789127844655478:2048] [node 2] ICP18 handshake (actor [1:7577789132139622857:2117]) is too old
2025-11-28T14:52:01.342789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789135169666975:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:01.356484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789133667586076:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.345922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789130874699671:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.356488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789130874699599:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.356565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:01.356644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789130874699599:2049] [node 2] ICP22 created new session: [5:7577789135169666986:2049]
2025-11-28T14:52:01.372513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789135169666986:2049] [node 2] ICS09 handshake done sender: [5:7577789135169666975:2117] self: [5:7577789135169666984:0] peer: [2:7577789133042242822:0] socket: 352 qp: -1
2025-11-28T14:52:01.372581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789135169666986:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:01.372656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:01.389101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789130874699599:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(32430453ns) {HandleHandshakeStatus:329(32424094ns) {DropIncomingHandshake:442(2069ns) DropOutgoingHandshake:457(6911ns) SwitchToState:179(1149ns) ProcessPendingSessionEvents:495(148647ns) {ForwardSessionEventToSession:612(95125ns) ForwardSessionEventToSession:612(7094ns) ForwardSessionEventToSession:612(2818ns) ForwardSessionEventToSession:612(10368ns) ForwardSessionEventToSession:612(4618ns) ForwardSessionEventToSession:612(2471ns) ForwardSessionEventToSession:612(6258ns) ForwardSessionEventToSession:612(2948ns) ForwardSessionEventToSession:612(2284ns) ForwardSessionEventToSession:612(4951ns)}}}
2025-11-28T14:52:01.389185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789130874699599:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:01.384819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215610:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.418796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789133124379062:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.419893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789133667586077:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.420100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789129372618750:2107] [node 4] ICP18 handshake (actor [3:7577789133667586077:2127]) is too old
2025-11-28T14:52:01.419839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789133124379077:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.420030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789133124379078:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8525
2025-11-28T14:52:01.420739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789133124379074:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:01.447075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789128829411697:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.447207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789133124379079:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8525
2025-11-28T14:52:01.447808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789128829411699:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.447862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:01.447898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789128829411699:2050] [node 3] ICP22 created new session: [4:7577789133124379081:2050]
2025-11-28T14:52:01.447937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789133124379081:2050] [node 3] ICS09 handshake done sender: [4:7577789133124379074:2117] self: [4:7577789133124379076:0] peer: [3:7577789133667586068:0] socket: 374 qp: -1
2025-11-28T14:52:01.447964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789133124379081:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:01.447993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:01.448228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789128829411699:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:01.448904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789132139622853:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.448957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789132139622853:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.450642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789132139622853:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.450656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789135169666988:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4699
2025-11-28T14:52:01.450774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789133124379084:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8525
2025-11-28T14:52:01.450948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789127844655541:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.451011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:01.451059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789127844655541:2107] [node 4] ICP22 created new session: [1:7577789132139622873:2107]
2025-11-28T14:52:01.451155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789132139622873:2107] [node 4] ICS09 handshake done sender: [1:7577789132139622853:2115] self: [1:7577789132139622852:0] peer: [4:7577789133124379080:0] socket: 342 qp: -1
2025-11-28T14:52:01.451206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789132139622873:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:01.451311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:01.467106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789133124379059:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:01.497470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789133124379077:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:01.497698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789132139622863:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.498696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789128829411697:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.498745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:01.498783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789128829411697:2048] [node 1] ICP22 created new session: [4:7577789133124379085:2048]
2025-11-28T14:52:01.498826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789133124379085:2048] [node 1] ICS09 handshake done sender: [4:7577789133124379077:2118] self: [4:7577789133124379080:0] peer: [1:7577789132139622852:0] socket: 375 qp: -1
2025-11-28T14:52:01.498858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789133124379085:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:01.498822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789127844655541:2107] [node 4] ICP18 handshake (actor [1:7577789132139622863:2118]) is too old
2025-11-28T14:52:01.498885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:01.518474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789128747275520:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:01.546436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789133667586074:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.546668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789129372618690:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.546751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789133042242844:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31
2025-11-28T14:52:01.547026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789128747275520:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.547570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789128747275520:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.547683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789133667586092:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64750
2025-11-28T14:52:01.547986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789133667586074:2125] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:01.548023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789128747275454:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.548108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:01.548312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789129372618690:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.548170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789128747275454:2049] [node 3] ICP22 created new session: [2:7577789133042242846:2049]
2025-11-28T14:52:01.548392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:01.548306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789133042242846:2049] [node 3] ICS09 handshake done sender: [2:7577789128747275520:2110] self: [2:7577789128747275519:0] peer: [3:7577789133667586091:0] socket: 344 qp: -1
2025-11-28T14:52:01.548401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789133042242846:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:01.548424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789129372618690:2049] [node 2] ICP22 created new session: [3:7577789133667586093:2049]
2025-11-28T14:52:01.548510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:01.548501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789133667586093:2049] [node 2] ICS09 handshake done sender: [3:7577789133667586074:2125] self: [3:7577789133667586091:0] peer: [2:7577789128747275519:0] socket: 354 qp: -1
2025-11-28T14:52:01.548529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789133667586093:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:01.548553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:01.559790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789133124379066:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:01.622312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789133156215600:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.622547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789128861248282:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.624632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789133124379066:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.646788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789133124379066:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:01.624829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789133667586072:2123] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:01.642888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215612:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.680589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789133667586072:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.661803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789135169666982:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.678789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789128829411761:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.685420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789128861248282:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(19713505ns)
2025-11-28T14:52:01.689030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789133156215600:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:01.683621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789133667586072:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:01.672680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789130874699600:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.678891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:01.678956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789128829411761:2109] [node 6] ICP22 created new session: [4:7577789133124379089:2109]
2025-11-28T14:52:01.679004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789133124379089:2109] [node 6] ICS09 handshake done sender: [4:7577789133124379066:2114] self: [4:7577789133124379065:0] peer: [6:7577789133156215611:0] socket: 366 qp: -1
2025-11-28T14:52:01.697118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789128861248282:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.682964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789135169666990:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4699
2025-11-28T14:52:01.679060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789133124379089:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:01.689086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789135169666982:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:01.696662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789133667586069:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.698806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:01.697904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:01.697083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789130874699600:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.679108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:01.698890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789133667586069:2108] [node 5] ICP22 created new session: [3:7577789133667586096:2108]
2025-11-28T14:52:01.697149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:01.698226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789128861248282:2109] [node 4] ICP22 created new session: [6:7577789133156215613:2109]
2025-11-28T14:52:01.698257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789130874699600:2050] [node 3] ICP22 created new session: [5:7577789135169666991:2050]
2025-11-28T14:52:01.699151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789133667586096:2108] [node 5] ICS09 handshake done sender: [3:7577789133667586072:2123] self: [3:7577789133667586071:0] peer: [5:7577789135169666989:0] socket: 378 qp: -1
2025-11-28T14:52:01.698824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789133156215613:2109] [node 4] ICS09 handshake done sender: [6:7577789133156215600:2120] self: [6:7577789133156215611:0] peer: [4:7577789133124379065:0] socket: 372 qp: -1
2025-11-28T14:52:01.698969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789133156215613:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:01.699204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789133667586096:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:01.698602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789135169666991:2050] [node 3] ICS09 handshake done sender: [5:7577789135169666982:2118] self: [5:7577789135169666989:0] peer: [3:7577789133667586071:0] socket: 379 qp: -1
2025-11-28T14:52:01.699367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:01.699168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:01.699135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789135169666991:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:01.699189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:01.703942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789133042242825:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:01.733453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789133156215610:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.733638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789128861248219:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.734940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789133042242827:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:01.740453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789133042242825:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.749046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789133042242825:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:01.749351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789128747275518:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.749418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:01.749474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789128747275518:2109] [node 6] ICP22 created new session: [2:7577789133042242849:2109]
2025-11-28T14:52:01.749519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789133042242849:2109] [node 6] ICS09 handshake done sender: [2:7577789133042242825:2116] self: [2:7577789133042242824:0] peer: [6:7577789133156215615:0] socket: 345 qp: -1
2025-11-28T14:52:01.749552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789133042242849:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:01.749587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:01.772178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789128861248219:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7936) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789133156215610:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.773611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789133124379079:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.774490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789128829411698:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.774638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789133156215616:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36618
2025-11-28T14:52:01.774824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789133042242827:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.812808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789133124379092:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8525
2025-11-28T14:52:01.801710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789133042242827:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:01.790214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789133156215610:2122] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:01.832642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789135169666988:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.832867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789130874699664:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:01.838118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789135169666994:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4699
2025-11-28T14:52:01.790590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789128861248219:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.828558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789128747275516:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.829019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789133124379069:2116] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:01.836917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789133124379079:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:01.837420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789133124379069:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.838324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789133124379069:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:01.838618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789128829411760:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:01.838678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:01.838728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789128829411760:2108] [node 5] ICP22 created new session: [4:7577789133124379094:2108]
2025-11-28T14:52:01.838772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789133124379094:2108] [node 5] ICS09 handshake done sender: [4:7577789133124379069:2116] self: [4:7577789133124379068:0] peer: [5:7577789135169666993:0] socket: 353 qp: -1
2025-11-28T14:52:01.838804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789133124379094:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:01.838854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:01.828634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:01.856458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789135169666988:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:01.856825Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789130874699664:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789135169666988:2121] held: no
2025-11-28T14:52:01.790648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:01.856549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789128829411698:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.828704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789128747275516:2107] [node 4] ICP22 created new session: [2:7577789133042242852:2107]
2025-11-28T14:52:01.828763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789133042242852:2107] [node 4] ICS09 handshake done sender: [2:7577789133042242827:2117] self: [2:7577789133042242826:0] peer: [4:7577789133124379091:0] socket: 380 qp: -1
2025-11-28T14:52:01.828822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789133042242852:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:01.828861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:01.856647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:01.860738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789128829411698:2049] [node 2] ICP22 created new session: [4:7577789133124379096:2049]
2025-11-28T14:52:01.860808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789133124379096:2049] [node 2] ICS09 handshake done sender: [4:7577789133124379079:2120] self: [4:7577789133124379091:0] peer: [2:7577789133042242826:0] socket: 318 qp: -1
2025-11-28T14:52:01.860845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789133124379096:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:01.860924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:01.870744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789128829411698:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20697779ns) {HandleHandshakeStatus:329(20694551ns) {DropIncomingHandshake:442(3136ns) DropOutgoingHandshake:457(7616ns) SwitchToState:179(793ns) ProcessPendingSessionEvents:495(9336795ns) {ForwardSessionEventToSession:612(67020ns) ForwardSessionEventToSession:612(3617ns) ForwardSessionEventToSession:612(537764ns) ForwardSessionEventToSession:612(3713ns) ForwardSessionEventToSession:612(8679716ns) ForwardSessionEventToSession:612(8862ns) ForwardSessionEventToSession:612(7760ns) ForwardSessionEventToSession:612(5906ns) ForwardSessionEventToSession:612(2442ns) ForwardSessionEventToSession:612(3802ns)}}}
2025-11-28T14:52:01.790709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789128861248219:2049] [node 2] ICP22 created new session: [6:7577789133156215617:2049]
2025-11-28T14:52:01.790759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789133156215617:2049] [node 2] ICS09 handshake done sender: [6:7577789133156215610:2122] self: [6:7577789133156215615:0] peer: [2:7577789133042242824:0] socket: 323 qp: -1
2025-11-28T14:52:01.790805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789133156215617:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:01.790842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:01.872602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789130874699664:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:01.872678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:01.881945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789130874699664:2108] [node 4] ICP22 created new session: [5:7577789135169666995:2108]
2025-11-28T14:52:01.897652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789135169666995:2108] [node 4] ICS09 handshake done sender: [5:7577789135169666988:2121] self: [5:7577789135169666993:0] peer: [4:7577789133124379068:0] socket: 358 qp: -1
2025-11-28T14:52:01.899048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789135169666995:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:01.899120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:01.899931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789130874699664:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(26713487ns) {HandleHandshakeStatus:329(26708498ns) {DropIncomingHandshake:442(1515ns) DropOutgoingHandshake:457(496ns) SwitchToState:179(1208ns) ProcessPendingSessionEvents:495(1379ns)}}
2025-11-28T14:52:01.911660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789133124379063:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:01.922105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789133042242844:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:01.922325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789128747275516:2107] [node 4] ICP18 handshake (actor [2:7577789133042242844:2120]) is too old
2025-11-28T14:52:01.943606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:01.943857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789133042242835:2048] [node 1] ICS07 socket disconnect 327 reason# EndOfStream
2025-11-28T14:52:01.943892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789133042242835:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:01.943970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:01.944012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789133042242835:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:01.947915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:01.960631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789133667586088:2048] [node 1] ICS01 socket: 370 reason# EndOfStream
2025-11-28T14:52:01.960727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789129372618689:2048] [node 1] ICP30 unregister session Session# [3:7577789133667586088:2048] VirtualId# [3:7577789133667586082:0]
2025-11-28T14:52:01.960771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789133667586088:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:01.960880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:01.960945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:01.960509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789128747275453:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26745) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:01.961667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789133042242835:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:01.972592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:01.972857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789133156215608:2048] [node 1] ICS01 socket: 335 reason# EndOfStream
2025-11-28T14:52:01.972524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:01.973204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:01.972896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789128861248218:2048] [node 1] ICP30 unregister session Session# [6:7577789133156215608:2048] VirtualId# [6:7577789133156215595:0]
2025-11-28T14:52:01.976548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789128747275453:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26745) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:01.972809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789135169666976:2048] [node 1] ICS01 socket: 324 reason# EndOfStream
2025-11-28T14:52:01.972873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789130874699598:2048] [node 1] ICP30 unregister session Session# [5:7577789135169666976:2048] VirtualId# [5:7577789135169666973:0]
2025-11-28T14:52:01.972908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789135169666976:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:01.973012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:01.973072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:01.985450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789133124379085:2048] [node 1] ICS07 socket disconnect 375 reason# EndOfStream
2025-11-28T14:52:01.992595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789133124379085:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:01.992797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:01.972932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789133156215608:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:01.993044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789133042242835:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:01.993111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789128747275453:2048] [node 1] ICP30 unregister session Session# [2:7577789133042242835:2048] VirtualId# [2:7577789133042242832:0]
2025-11-28T14:52:01.993175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:01.993210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789128747275453:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:01.993225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:01.993531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [2:7577789128747275453:2048] [node 1] ICP35 event processing took too much time StateWork:294(19851925ns) {HandleHandshakeStatus:395(19848256ns) {LogHandshakeFail:472(11100ns) {UpdateErrorStateLog:545(2383ns)} DropOutgoingHandshake:457(2473ns) ProcessPendingSessionEvents:495(498ns) UnregisterSession:538(60901ns) {ProcessPendingSessionEvents:495(424ns) SwitchToInitialState:202(8526ns) {SwitchToState:179(697ns)}} TransitToErrorState:752(57527ns) {SwitchToState:179(5073ns) ProcessPendingSessionEvents:495(911ns) DropHandshakes:470(1686ns) {DropIncomingHandshake:442(624ns) DropOutgoingHandshake:457(168ns)}}}}
2025-11-28T14:52:01.992860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789133124379085:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:01.973035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:01.973130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:01.994793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789128829411697:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26745) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:01.994932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789133124379085:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:01.995292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789128829411697:2048] [node 1] ICP30 unregister session Session# [4:7577789133124379085:2048] VirtualId# [4:7577789133124379080:0]
2025-11-28T14:52:01.996579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:01.996650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789128829411697:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:01.996665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:02.006670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789128829411697:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:02.012513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:02.012777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789133667586093:2049] [node 2] ICS01 socket: 354 reason# EndOfStream
2025-11-28T14:52:02.012827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789129372618690:2049] [node 2] ICP30 unregister session Session# [3:7577789133667586093:2049] VirtualId# [3:7577789133667586091:0]
2025-11-28T14:52:02.012866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789133667586093:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.012961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:02.013016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:02.020714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:02.020985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789135169666986:2049] [node 2] ICS01 socket: 352 reason# EndOfStream
2025-11-28T14:52:02.021037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789130874699599:2049] [node 2] ICP30 unregister session Session# [5:7577789135169666986:2049] VirtualId# [5:7577789135169666984:0]
2025-11-28T14:52:02.021069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789135169666986:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.021152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:02.021192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:02.022253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:02.022452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789133124379096:2049] [node 2] ICS01 socket: 318 reason# EndOfStream
2025-11-28T14:52:02.022491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789128829411698:2049] [node 2] ICP30 unregister session Session# [4:7577789133124379096:2049] VirtualId# [4:7577789133124379091:0]
2025-11-28T14:52:02.022520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789133124379096:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.022603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:02.022654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:02.029085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:02.029328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789133156215617:2049] [node 2] ICS01 socket: 323 reason# EndOfStream
2025-11-28T14:52:02.029369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789128861248219:2049] [node 2] ICP30 unregister session Session# [6:7577789133156215617:2049] VirtualId# [6:7577789133156215615:0]
2025-11-28T14:52:02.029401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789133156215617:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.029516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:02.029562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:02.077686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:02.077956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789135169666991:2050] [node 3] ICS01 socket: 379 reason# EndOfStream
2025-11-28T14:52:02.078003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789130874699600:2050] [node 3] ICP30 unregister session Session# [5:7577789135169666991:2050] VirtualId# [5:7577789135169666989:0]
2025-11-28T14:52:02.078038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789135169666991:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.078123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:02.078165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:02.077890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:02.078028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789133124379081:2050] [node 3] ICS07 socket disconnect 374 reason# EndOfStream
2025-11-28T14:52:02.078047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789133124379081:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.078105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:02.078133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789133124379081:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:02.078852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789128829411699:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61180) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:02.078883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789133124379081:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:02.078906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789128829411699:2050] [node 3] ICP30 unregister session Session# [4:7577789133124379081:2050] VirtualId# [4:7577789133124379076:0]
2025-11-28T14:52:02.078944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:02.078965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789128829411699:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61180) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:02.078975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61180) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:02.082104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:02.082387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789133156215603:2050] [node 3] ICS01 socket: 337 reason# EndOfStream
2025-11-28T14:52:02.082451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789128861248220:2050] [node 3] ICP30 unregister session Session# [6:7577789133156215603:2050] VirtualId# [6:7577789133156215597:0]
2025-11-28T14:52:02.082495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789133156215603:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.082633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:02.082682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:02.090982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:02.091219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789135169666995:2108] [node 4] ICS01 socket: 358 reason# EndOfStream
2025-11-28T14:52:02.091282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789130874699664:2108] [node 4] ICP30 unregister session Session# [5:7577789135169666995:2108] VirtualId# [5:7577789135169666993:0]
2025-11-28T14:52:02.091335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789135169666995:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.091416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:02.091455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:02.091747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:02.091895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789133156215613:2109] [node 4] ICS07 socket disconnect 372 reason# EndOfStream
2025-11-28T14:52:02.091931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789133156215613:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.091994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:02.092021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789133156215613:2109] [node 4] ICS15 start handshake
2025-11-28T14:52:02.092765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789128861248282:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19745) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:02.092887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789133156215613:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:02.092928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789128861248282:2109] [node 4] ICP30 unregister session Session# [6:7577789133156215613:2109] VirtualId# [6:7577789133156215611:0]
2025-11-28T14:52:02.092974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:02.093003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789128861248282:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:02.093017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:02.102993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789128861248282:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:52:02.140553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:02.140817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789133156215604:2106] [node 5] ICS07 socket disconnect 336 reason# EndOfStream
2025-11-28T14:52:02.140864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789133156215604:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:02.140999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:02.141062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789133156215604:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:02.142252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789128861248280:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23314) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:02.142335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789133156215604:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:02.142404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789128861248280:2106] [node 5] ICP30 unregister session Session# [6:7577789133156215604:2106] VirtualId# [6:7577789133156215596:0]
2025-11-28T14:52:02.142481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:02.142575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789128861248280:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23314) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:02.142611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23314) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:02.220571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789136165610506:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:02.220627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789136165610507:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:02.248063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789137676234261:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:02.248104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789137676234262:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:02.278646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789138231947544:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:02.278712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789138231947545:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:02.299409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789138999998475:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:02.299450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789138999998476:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:02.299474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789138999998477:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:02.319999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789137729695584:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:02.320036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789137729695585:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:02.320064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789137729695586:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:02.340822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789139450406203:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:02.340864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789139450406204:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:02.350248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789139450406202:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:02.431925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789139450406263:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:02.501022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789136165610569:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:02.517492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789138999998538:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:02.517899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789138231947633:2133] [node 6] ICP01 ready to work
2025-11-28T14:52:02.518300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789138999998539:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:02.518287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789139450406267:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:02.517765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789137676234326:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:02.518312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789137729695647:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:02.518233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789136165610570:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:02.518817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789138231947634:2130] [node 5] ICP01 ready to work
2025-11-28T14:52:02.518659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789137676234327:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:02.518580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789136165610571:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:02.518901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789137676234328:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:02.518793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789138231947635:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:02.521128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789137729695648:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:02.526317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789138999998476:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22237087ns) {Configure:104(22231653ns) {ProcessConfigured:135(70339ns) {StartInitialHandshake:157(68917ns) {DropHandshakes:470(2060ns) {DropIncomingHandshake:442(515ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(391ns) PrepareNewSessionHandshake:477(1230ns) {SwitchToState:179(506ns)}}}}}
2025-11-28T14:52:02.526288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789136165610506:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(94909555ns) {Configure:104(94903039ns) {ProcessConfigured:135(2035208ns) {StartInitialHandshake:157(2032836ns) {DropHandshakes:470(2833ns) {DropIncomingHandshake:442(610ns) DropOutgoingHandshake:457(478ns)} GenerateSessionVirtualId:988(964ns) PrepareNewSessionHandshake:477(2893ns) {SwitchToState:179(1244ns)}}}}}
2025-11-28T14:52:02.530405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789137676234261:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16748413ns) {Configure:104(16733910ns) {ProcessConfigured:135(567301ns) {StartInitialHandshake:157(562007ns) {DropHandshakes:470(1502ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(1028ns) PrepareNewSessionHandshake:477(1751ns) {SwitchToState:179(842ns)}}}}}
2025-11-28T14:52:02.530817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789139450406263:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(93204817ns) {Configure:104(89032383ns) {ProcessConfigured:135(776699ns) {StartInitialHandshake:157(773732ns) {DropHandshakes:470(1281ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:477(2050ns) {SwitchToState:179(598ns)}}}}}
2025-11-28T14:52:02.530448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789136165610507:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(103780659ns) {Configure:104(103770149ns) {ProcessConfigured:135(1959990ns) {StartInitialHandshake:157(1958022ns) {DropHandshakes:470(1254ns) {DropIncomingHandshake:442(299ns) DropOutgoingHandshake:457(183ns)} GenerateSessionVirtualId:988(383ns) PrepareNewSessionHandshake:477(1717ns) {SwitchToState:179(877ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.544475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789138231947647:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18036
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.546669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789136165610584:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32354
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.546909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789136165610586:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32354
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.547083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789136165610589:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32354
2025-11-28T14:52:02.547978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610586:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:02.546338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789137729695663:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8808
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.548397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789137676234344:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64580
2025-11-28T14:52:02.551629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789138231947648:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18036
2025-11-28T14:52:02.552656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789137729695662:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8808
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.558119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610584:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.548407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789137676234342:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64580
2025-11-28T14:52:02.557052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789138231947649:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18036
2025-11-28T14:52:02.557164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789137729695664:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8808
2025-11-28T14:52:02.558147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789136165610587:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32354
2025-11-28T14:52:02.558210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789137729695664:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.558450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789138231947648:2141] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.565103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789138231947650:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18036
2025-11-28T14:52:02.577184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789138231947544:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:02.584208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789138231947649:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.567632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789136165610588:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32354
2025-11-28T14:52:02.567267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789137729695663:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.557545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789137676234345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64580
2025-11-28T14:52:02.558164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789137676234343:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64580
2025-11-28T14:52:02.558497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789137676234344:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.583774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610589:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.577691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789137729695665:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8808
2025-11-28T14:52:02.564882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789137676234342:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.584488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610573:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.584544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789136165610573:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:02.589090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610577:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.589143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789136165610577:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:02.592031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789136165610573:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:02.592575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789136165610506:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:02.592658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:02.612757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789136165610506:2048] [node 2] ICP22 created new session: [1:7577789136165610595:2048]
2025-11-28T14:52:02.612770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789136165610577:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:02.619231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789136165610595:2048] [node 2] ICS09 handshake done sender: [1:7577789136165610573:2110] self: [1:7577789136165610572:0] peer: [2:7577789137676234348:0] socket: 324 qp: -1
2025-11-28T14:52:02.626441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789136165610595:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:02.626508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:02.627919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789136165610506:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(35404179ns) {HandleHandshakeStatus:329(35400862ns) {DropIncomingHandshake:442(15673ns) DropOutgoingHandshake:457(673ns) SwitchToState:179(784ns) ProcessPendingSessionEvents:495(133006ns) {ForwardSessionEventToSession:612(78237ns) ForwardSessionEventToSession:612(3734ns) ForwardSessionEventToSession:612(6328ns) ForwardSessionEventToSession:612(3080ns) ForwardSessionEventToSession:612(5436ns) ForwardSessionEventToSession:612(2617ns) ForwardSessionEventToSession:612(3023ns) ForwardSessionEventToSession:612(15704ns) ForwardSessionEventToSession:612(2390ns) ForwardSessionEventToSession:612(2468ns)}}}
2025-11-28T14:52:02.628163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789136165610507:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:02.628209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:02.628254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789136165610507:2049] [node 3] ICP22 created new session: [1:7577789136165610598:2049]
2025-11-28T14:52:02.628311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789136165610598:2049] [node 3] ICS09 handshake done sender: [1:7577789136165610577:2112] self: [1:7577789136165610574:0] peer: [3:7577789138231947653:0] socket: 328 qp: -1
2025-11-28T14:52:02.628346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789136165610598:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.589249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789137729695585:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:02.566262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789137676234346:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64580
2025-11-28T14:52:02.629240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789137729695668:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8808
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.629902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.566459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789137676234261:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:02.590689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789137676234330:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.590734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789137676234330:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:02.592465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789137676234330:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:02.626320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789137676234350:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64580
2025-11-28T14:52:02.641270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789137676234261:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789137676234342:2117] held: no
2025-11-28T14:52:02.642990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789137676234326:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:02.643067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:02.643135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789137676234326:2108] [node 5] ICP22 created new session: [2:7577789137676234351:2108]
2025-11-28T14:52:02.643226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789137676234351:2108] [node 5] ICS09 handshake done sender: [2:7577789137676234330:2112] self: [2:7577789137676234329:0] peer: [5:7577789137729695667:0] socket: 327 qp: -1
2025-11-28T14:52:02.643267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789137676234351:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:02.643314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:02.643839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789137676234342:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.647278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789139450406282:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:02.650774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789137729695663:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:02.646915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789138999998553:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47636
2025-11-28T14:52:02.647970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789137676234261:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:02.650654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789139450406282:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.651610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789138999998554:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47636
2025-11-28T14:52:02.648043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:02.648461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789138231947655:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18036
2025-11-28T14:52:02.648975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789137676234261:2048] [node 1] ICP22 created new session: [2:7577789137676234355:2048]
2025-11-28T14:52:02.649046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789137676234355:2048] [node 1] ICS09 handshake done sender: [2:7577789137676234342:2117] self: [2:7577789137676234348:0] peer: [1:7577789136165610572:0] socket: 329 qp: -1
2025-11-28T14:52:02.654091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789139450406202:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:02.649088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789137676234355:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:02.649125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:02.656122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789137729695585:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:02.656202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:02.656902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789137729695585:2049] [node 2] ICP22 created new session: [5:7577789137729695669:2049]
2025-11-28T14:52:02.656056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789138999998554:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.672737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789138231947648:2141] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.655937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610585:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.676738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789138999998476:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:02.655989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789136165610585:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:02.673112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789139450406283:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:02.660784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789137729695669:2049] [node 2] ICS09 handshake done sender: [5:7577789137729695663:2116] self: [5:7577789137729695667:0] peer: [2:7577789137676234329:0] socket: 351 qp: -1
2025-11-28T14:52:02.682480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789138231947544:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:02.676593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789136165610585:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:02.673374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789137729695669:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:02.677438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789139450406285:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:02.687247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:02.688806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789138231947544:2048] [node 1] ICP22 created new session: [3:7577789138231947656:2048]
2025-11-28T14:52:02.682328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789136165610571:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:02.682125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789137676234340:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.678008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:02.682178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789137676234340:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:02.697980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789138231947656:2048] [node 1] ICS09 handshake done sender: [3:7577789138231947648:2141] self: [3:7577789138231947653:0] peer: [1:7577789136165610574:0] socket: 350 qp: -1
2025-11-28T14:52:02.726062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789137676234340:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.704408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789139450406283:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.682422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:02.699735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789138231947656:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:02.681596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789137729695585:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(25271651ns) {HandleHandshakeStatus:329(25266236ns) {DropIncomingHandshake:442(1150ns) DropOutgoingHandshake:457(7320ns) SwitchToState:179(546ns) ProcessPendingSessionEvents:495(150487ns) {ForwardSessionEventToSession:612(76289ns) ForwardSessionEventToSession:612(8380ns) ForwardSessionEventToSession:612(11831ns) ForwardSessionEventToSession:612(2813ns) ForwardSessionEventToSession:612(15612ns) ForwardSessionEventToSession:612(3031ns) ForwardSessionEventToSession:612(4112ns) ForwardSessionEventToSession:612(9726ns) ForwardSessionEventToSession:612(2605ns) ForwardSessionEventToSession:612(6058ns)}}}
2025-11-28T14:52:02.739748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789138999998556:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47636
2025-11-28T14:52:02.729378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789137676234336:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.752469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789138999998554:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:02.704858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:02.682479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789136165610571:2109] [node 6] ICP22 created new session: [1:7577789136165610602:2109]
2025-11-28T14:52:02.709186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789139450406202:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789139450406282:2117] held: no
2025-11-28T14:52:02.688800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789136165610602:2109] [node 6] ICS09 handshake done sender: [1:7577789136165610585:2115] self: [1:7577789136165610583:0] peer: [6:7577789139450406284:0] socket: 373 qp: -1
2025-11-28T14:52:02.709407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789139450406282:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.729422Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789137676234336:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:02.729656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789137676234327:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:02.710705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789139450406203:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:02.726050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789139450406202:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:02.726128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:02.726187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789139450406202:2048] [node 1] ICP22 created new session: [6:7577789139450406287:2048]
2025-11-28T14:52:02.729710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:02.756705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789138999998476:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:02.692603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789136165610602:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:02.726247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789139450406287:2048] [node 1] ICS09 handshake done sender: [6:7577789139450406282:2117] self: [6:7577789139450406284:0] peer: [1:7577789136165610583:0] socket: 374 qp: -1
2025-11-28T14:52:02.730929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789137676234336:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:02.738930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789137676234327:2107] [node 4] ICP22 created new session: [2:7577789137676234358:2107]
2025-11-28T14:52:02.743881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789137676234328:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:02.743960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:02.744024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789137676234328:2109] [node 6] ICP22 created new session: [2:7577789137676234360:2109]
2025-11-28T14:52:02.744081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789137676234360:2109] [node 6] ICS09 handshake done sender: [2:7577789137676234336:2115] self: [2:7577789137676234335:0] peer: [6:7577789139450406286:0] socket: 353 qp: -1
2025-11-28T14:52:02.744128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789137676234360:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:02.744164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:02.749968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789137676234358:2107] [node 4] ICS09 handshake done sender: [2:7577789137676234340:2116] self: [2:7577789137676234339:0] peer: [4:7577789138999998555:0] socket: 375 qp: -1
2025-11-28T14:52:02.760419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789137676234358:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:02.761167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:02.715267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789138231947544:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32531538ns) {HandleHandshakeStatus:329(32528264ns) {DropIncomingHandshake:442(1340ns) DropOutgoingHandshake:457(6905ns) SwitchToState:179(802ns) ProcessPendingSessionEvents:495(152444ns) {ForwardSessionEventToSession:612(93573ns) ForwardSessionEventToSession:612(5266ns) ForwardSessionEventToSession:612(9727ns) ForwardSessionEventToSession:612(2660ns) ForwardSessionEventToSession:612(6877ns) ForwardSessionEventToSession:612(3201ns) ForwardSessionEventToSession:612(4134ns) ForwardSessionEventToSession:612(8237ns) ForwardSessionEventToSession:612(3427ns) ForwardSessionEventToSession:612(3788ns)}}}
2025-11-28T14:52:02.756778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:02.699587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:02.726288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789139450406287:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:02.756844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789138999998476:2049] [node 2] ICP22 created new session: [4:7577789138999998557:2049]
2025-11-28T14:52:02.726331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:02.764404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789138231947658:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18036
2025-11-28T14:52:02.730682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789139450406289:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:02.731428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789139450406283:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:02.731638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789139450406203:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:02.731709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:02.774415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789137676234327:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(44610757ns) {HandleHandshakeStatus:329(44607411ns) {DropIncomingHandshake:442(912ns) DropOutgoingHandshake:457(919ns) SwitchToState:179(1056ns) ProcessPendingSessionEvents:495(7360117ns) {ForwardSessionEventToSession:612(5940817ns) ForwardSessionEventToSession:612(1352878ns) ForwardSessionEventToSession:612(6211ns) ForwardSessionEventToSession:612(1840ns) ForwardSessionEventToSession:612(8457ns) ForwardSessionEventToSession:612(2620ns) ForwardSessionEventToSession:612(2319ns) ForwardSessionEventToSession:612(7432ns) ForwardSessionEventToSession:612(2751ns) ForwardSessionEventToSession:612(16904ns)}}}
2025-11-28T14:52:02.731761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789139450406203:2049] [node 2] ICP22 created new session: [6:7577789139450406290:2049]
2025-11-28T14:52:02.752963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789136165610571:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(71745740ns) {HandleHandshakeStatus:329(70713325ns) {DropIncomingHandshake:442(21771ns) DropOutgoingHandshake:457(331ns) SwitchToState:179(669ns) ProcessPendingSessionEvents:495(13475270ns) {ForwardSessionEventToSession:612(6749190ns) ForwardSessionEventToSession:612(6646090ns) ForwardSessionEventToSession:612(15020ns) ForwardSessionEventToSession:612(3870ns) ForwardSessionEventToSession:612(5757ns) ForwardSessionEventToSession:612(2616ns) ForwardSessionEventToSession:612(2908ns) ForwardSessionEventToSession:612(6947ns) ForwardSessionEventToSession:612(2212ns) ForwardSessionEventToSession:612(4994ns)}}}
2025-11-28T14:52:02.731832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789139450406290:2049] [node 2] ICS09 handshake done sender: [6:7577789139450406283:2118] self: [6:7577789139450406286:0] peer: [2:7577789137676234335:0] socket: 376 qp: -1
2025-11-28T14:52:02.731869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789139450406290:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:02.731909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:02.756901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789138999998557:2049] [node 2] ICS09 handshake done sender: [4:7577789138999998554:2116] self: [4:7577789138999998555:0] peer: [2:7577789137676234339:0] socket: 377 qp: -1
2025-11-28T14:52:02.756937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789138999998557:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:02.756975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:02.775287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789138231947658:2146] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.775965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789138999998559:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47636
2025-11-28T14:52:02.779970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789138999998541:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:02.780800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789138999998559:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.820319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789138999998543:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:02.834252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610587:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.834407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789137676234345:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.834550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789137676234327:2107] [node 4] ICP18 handshake (actor [2:7577789137676234345:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.853172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789137729695671:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8808
2025-11-28T14:52:02.859295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789137729695652:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:02.881065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789137676234343:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.881350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789137676234326:2108] [node 5] ICP18 handshake (actor [2:7577789137676234343:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.881791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789139450406294:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:02.886095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789136165610576:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:02.887559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789137729695662:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.887811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789137729695584:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:02.888046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610576:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.895741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789136165610576:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.896333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789136165610570:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:02.896278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789137729695674:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8808
2025-11-28T14:52:02.896428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:02.896759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789136165610570:2108] [node 5] ICP22 created new session: [1:7577789136165610606:2108]
2025-11-28T14:52:02.897461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789137729695662:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:02.897759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789136165610606:2108] [node 5] ICS09 handshake done sender: [1:7577789136165610576:2111] self: [1:7577789136165610575:0] peer: [5:7577789137729695673:0] socket: 325 qp: -1
2025-11-28T14:52:02.898033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789136165610606:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:02.898086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:02.898635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789137729695584:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:02.898710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:02.898759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789137729695584:2048] [node 1] ICP22 created new session: [5:7577789137729695675:2048]
2025-11-28T14:52:02.898822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789137729695675:2048] [node 1] ICS09 handshake done sender: [5:7577789137729695662:2115] self: [5:7577789137729695673:0] peer: [1:7577789136165610575:0] socket: 337 qp: -1
2025-11-28T14:52:02.898860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789137729695675:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:02.898899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:02.900251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789137729695584:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.901837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789138999998562:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47636
2025-11-28T14:52:02.903538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789138999998562:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.912279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789137676234332:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:02.937989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789138231947647:2140] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.938134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789138231947545:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:02.938461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789137676234332:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:02.939039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789137676234332:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:02.939219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789137676234262:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:02.939288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:02.939126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789138231947661:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18036
2025-11-28T14:52:02.939339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789137676234262:2049] [node 3] ICP22 created new session: [2:7577789137676234364:2049]
2025-11-28T14:52:02.939516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789137676234364:2049] [node 3] ICS09 handshake done sender: [2:7577789137676234332:2113] self: [2:7577789137676234331:0] peer: [3:7577789138231947660:0] socket: 326 qp: -1
2025-11-28T14:52:02.939576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789137676234364:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:02.939695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:02.939843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789138231947647:2140] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:02.940122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789138231947545:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:02.940175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:02.940239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789138231947545:2049] [node 2] ICP22 created new session: [3:7577789138231947662:2049]
2025-11-28T14:52:02.940301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789138231947662:2049] [node 2] ICS09 handshake done sender: [3:7577789138231947647:2140] self: [3:7577789138231947660:0] peer: [2:7577789137676234331:0] socket: 339 qp: -1
2025-11-28T14:52:02.940345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789138231947662:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:02.940398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:02.950162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789138999998545:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:02.978255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789138231947650:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:02.988924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789138231947642:2138] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:03.018624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789138231947638:2136] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.020466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789143745373591:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:03.020108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610588:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.021103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789136165610507:2049] [node 3] ICP18 handshake (actor [1:7577789136165610588:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.023561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789143294965860:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47636
2025-11-28T14:52:03.023002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789137676234346:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.023212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789137676234262:2049] [node 3] ICP18 handshake (actor [2:7577789137676234346:2121]) is too old
2025-11-28T14:52:03.049603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789138231947640:2137] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:03.072554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789137729695665:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.072800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789137729695586:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:03.073090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789138231947640:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.073616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789138231947640:2137] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.073864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789138231947634:2130] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:03.073950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:03.073990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789138231947634:2130] [node 5] ICP22 created new session: [3:7577789142526914964:2130]
2025-11-28T14:52:03.073983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789142024662975:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8808
2025-11-28T14:52:03.074025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789142526914964:2130] [node 5] ICS09 handshake done sender: [3:7577789138231947640:2137] self: [3:7577789138231947637:0] peer: [5:7577789142024662974:0] socket: 358 qp: -1
2025-11-28T14:52:03.074061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789142526914964:2130] [node 5] ICS10 traffic start
2025-11-28T14:52:03.074091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:03.074288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789137729695665:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:03.075398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789137729695586:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789137729695665:2118] held: no
2025-11-28T14:52:03.075554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789137729695586:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:03.075649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:03.075711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789137729695586:2050] [node 3] ICP22 created new session: [5:7577789142024662976:2050]
2025-11-28T14:52:03.075826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789142024662976:2050] [node 3] ICS09 handshake done sender: [5:7577789137729695665:2118] self: [5:7577789142024662974:0] peer: [3:7577789138231947637:0] socket: 363 qp: -1
2025-11-28T14:52:03.075877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789142024662976:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:03.075935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:03.078248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789136165610579:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:03.092725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789138999998553:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.092958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789138999998475:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:03.093251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789136165610579:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.093779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789136165610579:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.093971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789143294965862:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47636
2025-11-28T14:52:03.093968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789136165610569:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:03.096237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789138999998553:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:03.094037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:03.096031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789136165610569:2107] [node 4] ICP22 created new session: [1:7577789140460577905:2107]
2025-11-28T14:52:03.098102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789138999998475:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:03.096204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789140460577905:2107] [node 4] ICS09 handshake done sender: [1:7577789136165610579:2113] self: [1:7577789136165610578:0] peer: [4:7577789143294965861:0] socket: 370 qp: -1
2025-11-28T14:52:03.098931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789140460577905:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:03.099006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:03.099543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:03.099739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789138999998547:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:03.099865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789138999998475:2048] [node 1] ICP22 created new session: [4:7577789143294965863:2048]
2025-11-28T14:52:03.107415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789143294965863:2048] [node 1] ICS09 handshake done sender: [4:7577789138999998553:2115] self: [4:7577789143294965861:0] peer: [1:7577789136165610578:0] socket: 371 qp: -1
2025-11-28T14:52:03.107491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789143294965863:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:03.107553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:03.126891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789137729695671:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.127170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789137729695648:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:03.127299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789143745373592:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:03.127372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789138999998475:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(28974130ns) {HandleHandshakeStatus:329(28971925ns) {DropIncomingHandshake:442(834ns) DropOutgoingHandshake:457(4154ns) SwitchToState:179(532ns) ProcessPendingSessionEvents:495(726119ns) {ForwardSessionEventToSession:612(505246ns) ForwardSessionEventToSession:612(5533ns) ForwardSessionEventToSession:612(6288ns) ForwardSessionEventToSession:612(3859ns) ForwardSessionEventToSession:612(3084ns) ForwardSessionEventToSession:612(59239ns) ForwardSessionEventToSession:612(7192ns) ForwardSessionEventToSession:612(2477ns) ForwardSessionEventToSession:612(118238ns) ForwardSessionEventToSession:612(3509ns)}}}
2025-11-28T14:52:03.127415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789138999998547:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.127437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789138999998475:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:03.127968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789138999998547:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.128153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789138999998538:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:03.128136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789142024662979:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8808
2025-11-28T14:52:03.128292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:03.128401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789138999998538:2108] [node 5] ICP22 created new session: [4:7577789143294965867:2108]
2025-11-28T14:52:03.128467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789143294965867:2108] [node 5] ICS09 handshake done sender: [4:7577789138999998547:2113] self: [4:7577789138999998546:0] peer: [5:7577789142024662978:0] socket: 384 qp: -1
2025-11-28T14:52:03.128457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789137729695671:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:03.128516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789143294965867:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:03.129008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789137729695648:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:03.128560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:03.129778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:03.129846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789137729695648:2108] [node 4] ICP22 created new session: [5:7577789142024662980:2108]
2025-11-28T14:52:03.129914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789142024662980:2108] [node 4] ICS09 handshake done sender: [5:7577789137729695671:2120] self: [5:7577789142024662978:0] peer: [4:7577789138999998546:0] socket: 385 qp: -1
2025-11-28T14:52:03.129954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789142024662980:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:03.129993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:03.131048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789137729695648:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:03.133764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789137729695656:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:03.142955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789139450406294:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.143195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789139450406263:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:03.143490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789137729695656:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.144113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789137729695656:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:03.144339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789137729695647:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:03.144427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789143745373594:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:03.144443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:03.144515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789137729695647:2109] [node 6] ICP22 created new session: [5:7577789142024662983:2109]
2025-11-28T14:52:03.144580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789142024662983:2109] [node 6] ICS09 handshake done sender: [5:7577789137729695656:2113] self: [5:7577789137729695654:0] peer: [6:7577789143745373593:0] socket: 330 qp: -1
2025-11-28T14:52:03.144656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789142024662983:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:03.144710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:03.144693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789139450406294:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:03.145102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789139450406263:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:03.145530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:03.146191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789139450406263:2106] [node 5] ICP22 created new session: [6:7577789143745373595:2106]
2025-11-28T14:52:03.146275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789143745373595:2106] [node 5] ICS09 handshake done sender: [6:7577789139450406294:2121] self: [6:7577789143745373593:0] peer: [5:7577789137729695654:0] socket: 343 qp: -1
2025-11-28T14:52:03.146424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789143745373595:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:03.146486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:03.146936Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789139450406263:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:03.149054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789138231947643:2139] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:03.157782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789143745373591:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.157993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789139450406204:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:03.158249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789138231947643:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.158815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789138231947643:2139] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.158994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789143745373598:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:03.159116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789138231947633:2133] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:03.159214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:03.159287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789138231947633:2133] [node 6] ICP22 created new session: [3:7577789142526914967:2133]
2025-11-28T14:52:03.159306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789143745373591:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:03.159421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789142526914967:2133] [node 6] ICS09 handshake done sender: [3:7577789138231947643:2139] self: [3:7577789138231947641:0] peer: [6:7577789143745373597:0] socket: 388 qp: -1
2025-11-28T14:52:03.159473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789142526914967:2133] [node 6] ICS10 traffic start
2025-11-28T14:52:03.159563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789139450406204:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:03.159519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:03.159802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:03.159882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789139450406204:2050] [node 3] ICP22 created new session: [6:7577789143745373599:2050]
2025-11-28T14:52:03.159998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789143745373599:2050] [node 3] ICS09 handshake done sender: [6:7577789143745373591:2122] self: [6:7577789143745373597:0] peer: [3:7577789138231947641:0] socket: 389 qp: -1
2025-11-28T14:52:03.160048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789143745373599:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:03.160081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:03.160548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789139450406204:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:03.162529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789138231947652:2144] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:03.171502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789143294965860:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.171753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789138999998477:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:03.171984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789138231947652:2144] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.172523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789138231947652:2144] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.172749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789143294965870:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47636
2025-11-28T14:52:03.172778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789138231947635:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:03.172841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:03.172879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789138231947635:2107] [node 4] ICP22 created new session: [3:7577789142526914970:2107]
2025-11-28T14:52:03.172932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789142526914970:2107] [node 4] ICS09 handshake done sender: [3:7577789138231947652:2144] self: [3:7577789138231947651:0] peer: [4:7577789143294965869:0] socket: 347 qp: -1
2025-11-28T14:52:03.173011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789142526914970:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:03.173043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:03.173073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789143294965860:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:03.173481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789138999998477:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:03.173620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:03.173672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789138999998477:2050] [node 3] ICP22 created new session: [4:7577789143294965871:2050]
2025-11-28T14:52:03.173774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789143294965871:2050] [node 3] ICS09 handshake done sender: [4:7577789143294965860:2120] self: [4:7577789143294965869:0] peer: [3:7577789138231947651:0] socket: 356 qp: -1
2025-11-28T14:52:03.173817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789143294965871:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:03.173853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:03.174213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789138999998477:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:03.176262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789138999998549:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:03.177062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789143745373592:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:03.177294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789139450406267:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:03.177508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789138999998549:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:03.178006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789138999998549:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:03.178135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789143745373602:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45347
2025-11-28T14:52:03.178222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789138999998539:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:03.178383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:03.178448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789138999998539:2109] [node 6] ICP22 created new session: [4:7577789143294965874:2109]
2025-11-28T14:52:03.178467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789143745373592:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:03.178502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789143294965874:2109] [node 6] ICS09 handshake done sender: [4:7577789138999998549:2114] self: [4:7577789138999998548:0] peer: [6:7577789143745373601:0] socket: 346 qp: -1
2025-11-28T14:52:03.178691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789139450406267:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:03.178628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789143294965874:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:03.178684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:03.178975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:03.179037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789139450406267:2109] [node 4] ICP22 created new session: [6:7577789143745373603:2109]
2025-11-28T14:52:03.179088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789143745373603:2109] [node 4] ICS09 handshake done sender: [6:7577789143745373592:2123] self: [6:7577789143745373601:0] peer: [4:7577789138999998548:0] socket: 355 qp: -1
2025-11-28T14:52:03.179112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789143745373603:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:03.179145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:03.179863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789139450406267:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:03.213861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:03.214045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789137676234355:2048] [node 1] ICS01 socket: 329 reason# EndOfStream
2025-11-28T14:52:03.214085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789137676234261:2048] [node 1] ICP30 unregister session Session# [2:7577789137676234355:2048] VirtualId# [2:7577789137676234348:0]
2025-11-28T14:52:03.214109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789137676234355:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.214183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:03.214232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:03.215154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:03.215297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789138231947656:2048] [node 1] ICS07 socket disconnect 350 reason# EndOfStream
2025-11-28T14:52:03.215322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789138231947656:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.215395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:03.215441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789138231947656:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:03.217320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789138231947544:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:03.217345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789138231947656:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:03.217623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:03.217890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789143294965863:2048] [node 1] ICS07 socket disconnect 371 reason# EndOfStream
2025-11-28T14:52:03.217933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789143294965863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.218043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:03.218049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789138231947544:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.218106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789143294965863:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:03.218240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789138231947656:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.218339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789138231947544:2048] [node 1] ICP30 unregister session Session# [3:7577789138231947656:2048] VirtualId# [3:7577789138231947653:0]
2025-11-28T14:52:03.218391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:03.218419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789138231947544:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.218428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.218841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:03.218976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789137729695675:2048] [node 1] ICS07 socket disconnect 337 reason# EndOfStream
2025-11-28T14:52:03.218996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789137729695675:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.219054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:03.219092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789137729695675:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:03.219097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789138999998475:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.219160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789143294965863:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.219207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789138999998475:2048] [node 1] ICP30 unregister session Session# [4:7577789143294965863:2048] VirtualId# [4:7577789143294965861:0]
2025-11-28T14:52:03.219258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:03.219296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789138999998475:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.219419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.219740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789137729695584:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.219917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789137729695675:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.219982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789137729695584:2048] [node 1] ICP30 unregister session Session# [5:7577789137729695675:2048] VirtualId# [5:7577789137729695673:0]
2025-11-28T14:52:03.220065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:03.220100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789137729695584:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.220134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.220124Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:03.220351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789139450406287:2048] [node 1] ICS01 socket: 374 reason# EndOfStream
2025-11-28T14:52:03.220409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789139450406202:2048] [node 1] ICP30 unregister session Session# [6:7577789139450406287:2048] VirtualId# [6:7577789139450406284:0]
2025-11-28T14:52:03.232445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789138999998475:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:03.236573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789138231947544:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:03.220437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789139450406287:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.220574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:03.220629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:03.240812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789137729695584:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:03.253914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:03.254086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789138231947662:2049] [node 2] ICS07 socket disconnect 339 reason# EndOfStream
2025-11-28T14:52:03.254120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789138231947662:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.254210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:03.254254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789138231947662:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:03.255256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789138231947545:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17660) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.255380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789138231947662:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.255413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789138231947545:2049] [node 2] ICP30 unregister session Session# [3:7577789138231947662:2049] VirtualId# [3:7577789138231947660:0]
2025-11-28T14:52:03.255508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:03.255569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789138231947545:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.255589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.257022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:03.257212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789138999998557:2049] [node 2] ICS01 socket: 377 reason# EndOfStream
2025-11-28T14:52:03.257261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789138999998476:2049] [node 2] ICP30 unregister session Session# [4:7577789138999998557:2049] VirtualId# [4:7577789138999998555:0]
2025-11-28T14:52:03.257317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789138999998557:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.257409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:03.257464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:03.258064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:03.258266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789137729695669:2049] [node 2] ICS07 socket disconnect 351 reason# EndOfStream
2025-11-28T14:52:03.258299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789137729695669:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.258371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:03.258420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789137729695669:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:03.258976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:03.259125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789137729695585:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17660) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.259160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789139450406290:2049] [node 2] ICS01 socket: 376 reason# EndOfStream
2025-11-28T14:52:03.259237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789137729695669:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.259290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789137729695585:2049] [node 2] ICP30 unregister session Session# [5:7577789137729695669:2049] VirtualId# [5:7577789137729695667:0]
2025-11-28T14:52:03.259340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:03.259254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789139450406203:2049] [node 2] ICP30 unregister session Session# [6:7577789139450406290:2049] VirtualId# [6:7577789139450406286:0]
2025-11-28T14:52:03.259410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789137729695585:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.259459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.259283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789139450406290:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.259338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:03.259378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:03.265315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789138231947545:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:03.269325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789137729695585:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:03.271066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:03.271312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789143294965871:2050] [node 3] ICS07 socket disconnect 356 reason# EndOfStream
2025-11-28T14:52:03.271350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789143294965871:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.271466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:03.271549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789143294965871:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:03.272210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:03.272430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789142024662976:2050] [node 3] ICS01 socket: 363 reason# EndOfStream
2025-11-28T14:52:03.272524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789137729695586:2050] [node 3] ICP30 unregister session Session# [5:7577789142024662976:2050] VirtualId# [5:7577789142024662974:0]
2025-11-28T14:52:03.272409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789138999998477:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29766) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.272560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789142024662976:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.272522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789143294965871:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.272588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789138999998477:2050] [node 3] ICP30 unregister session Session# [4:7577789143294965871:2050] VirtualId# [4:7577789143294965869:0]
2025-11-28T14:52:03.272662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:03.272664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:03.272696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789138999998477:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.272742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.272735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:03.273437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:03.273656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789143745373599:2050] [node 3] ICS07 socket disconnect 389 reason# EndOfStream
2025-11-28T14:52:03.273689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789143745373599:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.273763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:03.273811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789143745373599:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:03.274593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789139450406204:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29766) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.274716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789143745373599:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.274778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789139450406204:2050] [node 3] ICP30 unregister session Session# [6:7577789143745373599:2050] VirtualId# [6:7577789143745373597:0]
2025-11-28T14:52:03.274830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:03.274885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789139450406204:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.274908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.284678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789139450406204:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:03.284769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:03.285038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789142024662980:2108] [node 4] ICS07 socket disconnect 385 reason# EndOfStream
2025-11-28T14:52:03.285080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789142024662980:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.285230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:03.285300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789142024662980:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:03.285824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:03.286007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789143745373603:2109] [node 4] ICS07 socket disconnect 355 reason# EndOfStream
2025-11-28T14:52:03.286038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789143745373603:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.286129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:03.286130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789137729695648:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5306) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.286291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789142024662980:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.286189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789143745373603:2109] [node 4] ICS15 start handshake
2025-11-28T14:52:03.286415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789137729695648:2108] [node 4] ICP30 unregister session Session# [5:7577789142024662980:2108] VirtualId# [5:7577789142024662978:0]
2025-11-28T14:52:03.286497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:03.286547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789137729695648:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5306) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.286574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5306) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.287064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789139450406267:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5306) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.287194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789143745373603:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.287229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789139450406267:2109] [node 4] ICP30 unregister session Session# [6:7577789143745373603:2109] VirtualId# [6:7577789143745373601:0]
2025-11-28T14:52:03.287274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:03.287323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789139450406267:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5306) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.287346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5306) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.296950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789139450406267:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:52:03.352611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:03.352928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789143745373595:2106] [node 5] ICS07 socket disconnect 343 reason# EndOfStream
2025-11-28T14:52:03.353012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789143745373595:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:03.353220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:03.353314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789143745373595:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:03.357250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789139450406263:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26658) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:03.357337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789143745373595:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:03.357388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789139450406263:2106] [node 5] ICP30 unregister session Session# [6:7577789143745373595:2106] VirtualId# [6:7577789143745373593:0]
2025-11-28T14:52:03.357454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:03.357521Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789139450406263:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:03.357539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:03.431842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789142825755169:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:03.431843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789142825755168:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:03.464902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789141083714974:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:03.464953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789141083714975:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:03.496661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789144202001023:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:03.496703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789144202001024:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:03.520299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789141485654920:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:03.520345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789141485654921:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:03.520398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789141485654922:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:03.553168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789142023999187:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:03.553201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789142023999188:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:03.553232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789142023999189:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:03.583936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789143235919243:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:03.791808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789141485654983:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:03.583984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789143235919244:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:03.584019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789143235919245:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:03.970462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789141485654983:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(178271421ns) {Configure:104(178265340ns) {ProcessConfigured:135(114889ns) {StartInitialHandshake:157(111656ns) {DropHandshakes:470(3954ns) {DropIncomingHandshake:442(876ns) DropOutgoingHandshake:457(571ns)} GenerateSessionVirtualId:988(1266ns) PrepareNewSessionHandshake:477(1930ns) {SwitchToState:179(669ns)}}}}}
2025-11-28T14:52:03.921084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789141083714975:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(139169144ns) {Configure:104(139164960ns) {ProcessConfigured:135(6312148ns) {StartInitialHandshake:157(6307853ns) {DropHandshakes:470(3006ns) {DropIncomingHandshake:442(616ns) DropOutgoingHandshake:457(694ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:477(5663ns) {SwitchToState:179(2785ns)}}}}}
2025-11-28T14:52:03.932399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789142023999226:2085] [node 4] ICP01 ready to work
2025-11-28T14:52:03.980794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789142825755241:2107] [node 6] ICP01 ready to work
2025-11-28T14:52:03.968681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789144202001087:2107] [node 6] ICP01 ready to work
2025-11-28T14:52:03.968922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789143235919260:2062] [node 4] ICP01 ready to work
2025-11-28T14:52:03.980720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789141485654984:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:03.926213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789141083714974:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(145090561ns) {Configure:104(145083584ns) {ProcessConfigured:135(79239ns) {StartInitialHandshake:157(76194ns) {DropHandshakes:470(2352ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(401ns)} GenerateSessionVirtualId:988(672ns) PrepareNewSessionHandshake:477(1876ns) {SwitchToState:179(626ns)}}}}}
2025-11-28T14:52:03.980967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789144202001086:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:04.000704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789142825755242:2110] [node 5] ICP01 ready to work
2025-11-28T14:52:03.987344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789141083715050:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.015170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789143235919245:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45953739ns) {Configure:104(45947921ns) {ProcessConfigured:135(43955801ns) {StartInitialHandshake:157(43951939ns) {DropHandshakes:470(4582ns) {DropIncomingHandshake:442(1015ns) DropOutgoingHandshake:457(1047ns)} GenerateSessionVirtualId:988(1200ns) PrepareNewSessionHandshake:477(4459ns) {SwitchToState:179(2471ns)}}}}}
2025-11-28T14:52:04.000547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789144202001088:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:04.008578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789141083715051:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:04.026590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789144202001087:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29920649ns) {Configure:104(29908099ns) {ProcessConfigured:135(106861ns) {StartInitialHandshake:157(102753ns) {DropHandshakes:470(3257ns) {DropIncomingHandshake:442(557ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(809ns) PrepareNewSessionHandshake:477(1389ns) {SwitchToState:179(493ns)}}}}}
2025-11-28T14:52:04.032870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789143235919244:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(231518103ns) {Configure:104(231511910ns) {ProcessConfigured:135(12714215ns) {StartInitialHandshake:157(12711555ns) {DropHandshakes:470(2366ns) {DropIncomingHandshake:442(721ns) DropOutgoingHandshake:457(370ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(9927ns) {SwitchToState:179(1414ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.036690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789146318966546:2072] [node 6] ICP01 ready to work
2025-11-28T14:52:04.036643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789147120722539:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:04.019418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789145378682348:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:04.065001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789141485654920:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(252901441ns) {Configure:104(252894132ns) {ProcessConfigured:135(136802ns) {StartInitialHandshake:157(132293ns) {DropHandshakes:470(3333ns) {DropIncomingHandshake:442(1322ns) DropOutgoingHandshake:457(417ns)} GenerateSessionVirtualId:988(4550ns) PrepareNewSessionHandshake:477(3311ns) {SwitchToState:179(2427ns)}}}}}
2025-11-28T14:52:04.045906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789143235919243:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(109801955ns) {Configure:104(109796587ns) {ProcessConfigured:135(74462ns) {StartInitialHandshake:157(71187ns) {DropHandshakes:470(2823ns) {DropIncomingHandshake:442(637ns) DropOutgoingHandshake:457(443ns)} GenerateSessionVirtualId:988(671ns) PrepareNewSessionHandshake:477(1691ns) {SwitchToState:179(715ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.033801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789145378682349:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56442
2025-11-28T14:52:04.070880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789141485654921:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(195995376ns) {Configure:104(195987957ns) {ProcessConfigured:135(1754337ns) {StartInitialHandshake:157(1749536ns) {DropHandshakes:470(3340ns) {DropIncomingHandshake:442(891ns) DropOutgoingHandshake:457(688ns)} GenerateSessionVirtualId:988(710ns) PrepareNewSessionHandshake:477(3834ns) {SwitchToState:179(1860ns)}}}}}
2025-11-28T14:52:04.050554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789147530886609:2060] [node 5] ICP01 ready to work
2025-11-28T14:52:04.078654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789142023999187:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(266006663ns) {Configure:104(266003029ns) {ProcessConfigured:135(5698649ns) {StartInitialHandshake:157(5695780ns) {DropHandshakes:470(3402836ns) {DropIncomingHandshake:442(713ns) DropOutgoingHandshake:457(2245ns)} GenerateSessionVirtualId:988(3484ns) PrepareNewSessionHandshake:477(3318ns) {SwitchToState:179(1866ns)}}}}}
2025-11-28T14:52:04.077364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789145378682350:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56442
2025-11-28T14:52:04.079755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789144202001023:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(196929584ns) {Configure:104(196921431ns) {ProcessConfigured:135(1587023ns) {StartInitialHandshake:157(1582650ns) {DropHandshakes:470(1549ns) {DropIncomingHandshake:442(373ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(2200ns) PrepareNewSessionHandshake:477(2797ns) {SwitchToState:179(1403ns)}}}}}
2025-11-28T14:52:04.081419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789142023999188:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(264611759ns) {Configure:104(264605846ns) {ProcessConfigured:135(7353366ns) {StartInitialHandshake:157(7349267ns) {DropHandshakes:470(2991ns) {DropIncomingHandshake:442(598ns) DropOutgoingHandshake:457(567ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:477(2961ns) {SwitchToState:179(1485ns)}}}}}
2025-11-28T14:52:04.079293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789145378682349:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.054307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886613:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.083183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789142023999226:2085] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(145762683ns) {Configure:104(145758997ns) {ProcessConfigured:135(6315278ns) {StartInitialHandshake:157(6313548ns) {DropHandshakes:470(1997ns) {DropIncomingHandshake:442(465ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(567ns) PrepareNewSessionHandshake:477(2934ns) {SwitchToState:179(1151ns)}}}}}
2025-11-28T14:52:04.081555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789142825755168:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(159870379ns) {Configure:104(159860344ns) {ProcessConfigured:135(58057ns) {StartInitialHandshake:157(55637ns) {DropHandshakes:470(2528ns) {DropIncomingHandshake:442(624ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(902ns) PrepareNewSessionHandshake:477(1805ns) {SwitchToState:179(298ns)}}}}}
2025-11-28T14:52:04.083757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789145378682350:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.084242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789143235919260:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(71158088ns) {Configure:104(69109199ns) {ProcessConfigured:135(65093ns) {StartInitialHandshake:157(62580ns) {DropHandshakes:470(3795ns) {DropIncomingHandshake:442(725ns) DropOutgoingHandshake:457(589ns)} GenerateSessionVirtualId:988(718ns) PrepareNewSessionHandshake:477(1530ns) {SwitchToState:179(596ns)}}}}}
2025-11-28T14:52:04.087193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789141485654922:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(208795743ns) {Configure:104(208783541ns) {ProcessConfigured:135(6629064ns) {StartInitialHandshake:157(6625985ns) {DropHandshakes:470(2960ns) {DropIncomingHandshake:442(617ns) DropOutgoingHandshake:457(486ns)} GenerateSessionVirtualId:988(838ns) PrepareNewSessionHandshake:477(2688ns) {SwitchToState:179(1165ns)}}}}}
2025-11-28T14:52:04.084899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789142825755169:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(292570227ns) {Configure:104(292564106ns) {ProcessConfigured:135(3919089ns) {StartInitialHandshake:157(3915844ns) {DropHandshakes:470(2215ns) {DropIncomingHandshake:442(445ns) DropOutgoingHandshake:457(919ns)} GenerateSessionVirtualId:988(639ns) PrepareNewSessionHandshake:477(2962ns) {SwitchToState:179(265ns)}}}}}
2025-11-28T14:52:04.087619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789141485654986:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.087181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789144202001024:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(195411771ns) {Configure:104(195403172ns) {ProcessConfigured:135(3579077ns) {StartInitialHandshake:157(3577273ns) {DropHandshakes:470(1339ns) {DropIncomingHandshake:442(357ns) DropOutgoingHandshake:457(142ns)} GenerateSessionVirtualId:988(368ns) PrepareNewSessionHandshake:477(10375ns) {SwitchToState:179(8673ns)}}}}}
No RDMA context found for GID: 2025-11-28T14:52:04.086209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789147530886613:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:04.087644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789141485654986:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.088388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789142825755241:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(87349102ns) {Configure:104(87341168ns) {ProcessConfigured:135(3431590ns) {StartInitialHandshake:157(3428805ns) {DropHandshakes:470(1372ns) {DropIncomingHandshake:442(339ns) DropOutgoingHandshake:457(402ns)} GenerateSessionVirtualId:988(735ns) PrepareNewSessionHandshake:477(2674ns) {SwitchToState:179(1089ns)}}}}}
2025-11-28T14:52:04.088267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789142023999189:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(114408772ns) {Configure:104(114400695ns) {ProcessConfigured:135(3707513ns) {StartInitialHandshake:157(3705176ns) {DropHandshakes:470(1161ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:477(2593ns) {SwitchToState:179(1332ns)}}}}}
2025-11-28T14:52:04.087259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789143235919260:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.088494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789141083715050:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(69674188ns) {Configure:104(69665634ns) {ProcessConfigured:135(1890780ns) {StartInitialHandshake:157(1887460ns) {DropHandshakes:470(1316ns) {DropIncomingHandshake:442(558ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(1010ns) PrepareNewSessionHandshake:477(4223ns) {SwitchToState:179(2294ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.087827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789145780622292:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-28T14:52:04.089436Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789144202001086:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(88455025ns) {Configure:104(88447748ns) {ProcessConfigured:135(62904ns) {StartInitialHandshake:157(49878ns) {DropHandshakes:470(1229ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(646ns) PrepareNewSessionHandshake:477(649ns) {SwitchToState:179(238ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.090867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789147120722540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4209
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.090066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789141485654984:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(79666127ns) {Configure:104(79655997ns) {ProcessConfigured:135(1655600ns) {StartInitialHandshake:157(1653356ns) {DropHandshakes:470(1886ns) {DropIncomingHandshake:442(551ns) DropOutgoingHandshake:457(373ns)} GenerateSessionVirtualId:988(560ns) PrepareNewSessionHandshake:477(2027ns) {SwitchToState:179(573ns)}}}}}
2025-11-28T14:52:04.091506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789141485654986:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.089752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789148496968389:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40301
2025-11-28T14:52:04.091429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789142825755242:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(44785442ns) {Configure:104(44777165ns) {ProcessConfigured:135(1095694ns) {StartInitialHandshake:157(1094104ns) {DropHandshakes:470(1935ns) {DropIncomingHandshake:442(831ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(306ns) PrepareNewSessionHandshake:477(1268ns) {SwitchToState:179(634ns)}}}}}
2025-11-28T14:52:04.090278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789145378682351:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56442
2025-11-28T14:52:04.091929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789141083715051:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(70471079ns) {Configure:104(70461218ns) {ProcessConfigured:135(463691ns) {StartInitialHandshake:157(461919ns) {DropHandshakes:470(933ns) {DropIncomingHandshake:442(255ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(612ns) PrepareNewSessionHandshake:477(1093ns) {SwitchToState:179(467ns)}}}}}
2025-11-28T14:52:04.091780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789147120722548:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4209
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.091372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789148496968390:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40301
2025-11-28T14:52:04.096732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789146318966546:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24159418ns) {Configure:104(24144284ns) {ProcessConfigured:135(56800ns) {StartInitialHandshake:157(54716ns) {DropHandshakes:470(1571ns) {DropIncomingHandshake:442(317ns) DropOutgoingHandshake:457(343ns)} GenerateSessionVirtualId:988(916ns) PrepareNewSessionHandshake:477(743ns) {SwitchToState:179(259ns)}}}}}
2025-11-28T14:52:04.095912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789147120722549:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4209
2025-11-28T14:52:04.096678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789148496968389:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.097276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789145378682348:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47550107ns) {Configure:104(47541206ns) {ProcessConfigured:135(1479575ns) {StartInitialHandshake:157(1476907ns) {DropHandshakes:470(2003ns) {DropIncomingHandshake:442(473ns) DropOutgoingHandshake:457(448ns)} GenerateSessionVirtualId:988(536ns) PrepareNewSessionHandshake:477(1841ns) {SwitchToState:179(895ns)}}}}}
No RDMA context found for GID: 2025-11-28T14:52:04.098391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886617:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.096809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722540:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.098616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789145780622296:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-28T14:52:04.096722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789148496968390:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.100057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789141485654983:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.098723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789147120722545:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4209
2025-11-28T14:52:04.098928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789141083715045:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.098682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789147530886609:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27291595ns) {Configure:104(27285172ns) {ProcessConfigured:135(3547731ns) {StartInitialHandshake:157(3545278ns) {DropHandshakes:470(1336ns) {DropIncomingHandshake:442(336ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(1655ns) {SwitchToState:179(830ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.096855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789144202001024:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.100443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:04.102249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789148496968401:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40301
2025-11-28T14:52:04.098965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789141083715045:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:04.100865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886618:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
2025-11-28T14:52:04.101772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789146318966561:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25650
2025-11-28T14:52:04.098989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722548:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.102267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789145378682361:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56442
2025-11-28T14:52:04.102353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886620:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
2025-11-28T14:52:04.101264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789141485654983:2109] [node 6] ICP22 created new session: [4:7577789145780622297:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.102776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789145780622297:2109] [node 6] ICS09 handshake done sender: [4:7577789141485654986:2110] self: [4:7577789141485654985:0] peer: [6:7577789147530886616:0] socket: 332 qp: -1
2025-11-28T14:52:04.102329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789141083715045:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:04.102891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789147530886618:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.101802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789147120722557:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4209
2025-11-28T14:52:04.102952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789145780622297:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:04.103008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:04.110969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789141083714975:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.118787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789143235919244:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.128474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789148496968403:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40301
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.135964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789147530886613:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:04.118786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:04.145248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789148496968404:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40301
2025-11-28T14:52:04.128574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789141083714975:2049] [node 3] ICP22 created new session: [2:7577789145378682363:2049]
2025-11-28T14:52:04.146481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789143235919260:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.146559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:04.132462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789145378682353:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.147097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789148496968389:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:04.145863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789146318966562:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25650
2025-11-28T14:52:04.152221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789144202001024:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.132518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789145378682353:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:04.152219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789141485654983:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(47053903ns) {HandleHandshakeStatus:329(47050974ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(995ns) SwitchToState:179(658ns) ProcessPendingSessionEvents:495(15095096ns) {ForwardSessionEventToSession:612(9529193ns) ForwardSessionEventToSession:612(7350ns) ForwardSessionEventToSession:612(4889695ns) ForwardSessionEventToSession:612(17252ns) ForwardSessionEventToSession:612(4385ns) ForwardSessionEventToSession:612(99027ns) ForwardSessionEventToSession:612(2852ns) ForwardSessionEventToSession:612(501495ns) ForwardSessionEventToSession:612(4857ns) ForwardSessionEventToSession:612(12660ns)}}}
2025-11-28T14:52:04.136503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789145378682363:2049] [node 3] ICS09 handshake done sender: [2:7577789141083715045:2117] self: [2:7577789141083715044:0] peer: [3:7577789148496968400:0] socket: 330 qp: -1
2025-11-28T14:52:04.147113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789143235919260:2062] [node 4] ICP22 created new session: [6:7577789147530886623:2062]
2025-11-28T14:52:04.143225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789145378682363:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:04.151495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789147530886623:2062] [node 4] ICS09 handshake done sender: [6:7577789147530886613:2113] self: [6:7577789147530886616:0] peer: [4:7577789141485654985:0] socket: 340 qp: -1
2025-11-28T14:52:04.153130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:04.143296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:04.152166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789147530886623:2062] [node 4] ICS10 traffic start
2025-11-28T14:52:04.153407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789146318966562:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.155836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789142023999188:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.152050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789145378682353:2122] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:04.152632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789141083715050:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.155787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.153192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789144202001024:2049] [node 2] ICP22 created new session: [3:7577789148496968405:2049]
2025-11-28T14:52:04.152217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.155762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789148496968405:2049] [node 2] ICS09 handshake done sender: [3:7577789148496968389:2112] self: [3:7577789148496968400:0] peer: [2:7577789141083715044:0] socket: 334 qp: -1
2025-11-28T14:52:04.157442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789143235919244:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789147530886618:2116] held: no
2025-11-28T14:52:04.155810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789148496968405:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:04.162928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886624:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.162703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789145780622300:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-28T14:52:04.156537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789141083715050:2108] [node 6] ICP22 created new session: [2:7577789145378682367:2108]
2025-11-28T14:52:04.156629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789145378682367:2108] [node 6] ICS09 handshake done sender: [2:7577789145378682353:2122] self: [2:7577789145378682352:0] peer: [6:7577789147530886622:0] socket: 360 qp: -1
2025-11-28T14:52:04.155887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:04.156695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789145378682367:2108] [node 6] ICS10 traffic start
2025-11-28T14:52:04.156751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:04.157489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789141083714975:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(42588014ns) {HandleHandshakeStatus:329(42584891ns) {DropIncomingHandshake:442(738ns) DropOutgoingHandshake:457(1223ns) SwitchToState:179(680ns) ProcessPendingSessionEvents:495(2021347ns) {ForwardSessionEventToSession:612(195979ns) ForwardSessionEventToSession:612(282862ns) ForwardSessionEventToSession:612(161812ns) ForwardSessionEventToSession:612(5938ns) ForwardSessionEventToSession:612(173155ns) ForwardSessionEventToSession:612(3746ns) ForwardSessionEventToSession:612(3611ns) ForwardSessionEventToSession:612(737048ns) ForwardSessionEventToSession:612(4279ns) ForwardSessionEventToSession:612(431785ns)}}}
2025-11-28T14:52:04.167708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789146318966564:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25650
2025-11-28T14:52:04.160611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789145378682359:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.167822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789143235919260:2062] [node 4] ICP35 event processing took too much time PendingConnection:279(16324255ns) {HandleHandshakeStatus:329(16320835ns) {DropIncomingHandshake:442(818ns) DropOutgoingHandshake:457(14642ns) SwitchToState:179(555ns) ProcessPendingSessionEvents:495(7037736ns) {ForwardSessionEventToSession:612(1657359ns) ForwardSessionEventToSession:612(3439481ns) ForwardSessionEventToSession:612(162628ns) ForwardSessionEventToSession:612(4564ns) ForwardSessionEventToSession:612(539781ns) ForwardSessionEventToSession:612(3207ns) ForwardSessionEventToSession:612(644656ns) ForwardSessionEventToSession:612(3270ns) ForwardSessionEventToSession:612(3051ns) ForwardSessionEventToSession:612(561404ns)}}}
2025-11-28T14:52:04.167942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789145780622300:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.171001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789147530886618:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:04.173130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789143235919244:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.173465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:04.173184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789141485654921:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.160659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789145378682359:2124] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.173524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789143235919244:2049] [node 2] ICP22 created new session: [6:7577789147530886627:2049]
2025-11-28T14:52:04.173337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789141083715050:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(20000525ns) {HandleHandshakeStatus:329(19997842ns) {DropIncomingHandshake:442(15539ns) DropOutgoingHandshake:457(398ns) SwitchToState:179(797ns) ProcessPendingSessionEvents:495(11022672ns) {ForwardSessionEventToSession:612(5498303ns) ForwardSessionEventToSession:612(110339ns) ForwardSessionEventToSession:612(4319ns) ForwardSessionEventToSession:612(3012ns) ForwardSessionEventToSession:612(135202ns) ForwardSessionEventToSession:612(4057ns) ForwardSessionEventToSession:612(3093426ns) ForwardSessionEventToSession:612(1515052ns) ForwardSessionEventToSession:612(9311ns) ForwardSessionEventToSession:612(11921ns)}}}
2025-11-28T14:52:04.173729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789147530886627:2049] [node 2] ICS09 handshake done sender: [6:7577789147530886618:2116] self: [6:7577789147530886622:0] peer: [2:7577789145378682352:0] socket: 362 qp: -1
2025-11-28T14:52:04.173912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789147530886627:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:04.174305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:04.173643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789146318966564:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.173411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789145378682359:2124] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:04.174611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789146318966565:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25650
2025-11-28T14:52:04.173505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789145378682357:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.173529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789145378682357:2123] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:04.174268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789145378682348:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.174312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:04.174369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789145378682348:2109] [node 5] ICP22 created new session: [2:7577789145378682371:2109]
2025-11-28T14:52:04.174813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789145378682371:2109] [node 5] ICS09 handshake done sender: [2:7577789145378682359:2124] self: [2:7577789145378682358:0] peer: [5:7577789146318966563:0] socket: 378 qp: -1
2025-11-28T14:52:04.174880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789145378682371:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:04.174940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.176184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789146318966566:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25650
2025-11-28T14:52:04.176515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789146318966562:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:04.178397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789142023999188:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.177852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789145378682357:2123] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.178378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789141083715051:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.178477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:04.178496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:04.178885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789145780622302:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-28T14:52:04.178748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789141083715051:2107] [node 4] ICP22 created new session: [2:7577789145378682374:2107]
2025-11-28T14:52:04.178756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789142023999188:2049] [node 2] ICP22 created new session: [5:7577789146318966568:2049]
2025-11-28T14:52:04.178896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789146318966568:2049] [node 2] ICS09 handshake done sender: [5:7577789146318966562:2116] self: [5:7577789146318966563:0] peer: [2:7577789145378682358:0] socket: 379 qp: -1
2025-11-28T14:52:04.178939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789146318966568:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:04.179149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789145378682374:2107] [node 4] ICS09 handshake done sender: [2:7577789145378682357:2123] self: [2:7577789145378682356:0] peer: [4:7577789145780622301:0] socket: 384 qp: -1
2025-11-28T14:52:04.179112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:04.179201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789145378682374:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:04.179237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:04.179555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789145780622300:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:04.179771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789141485654921:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789145780622300:2117] held: no
2025-11-28T14:52:04.180558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789141485654921:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.180604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:04.180681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789141485654921:2049] [node 2] ICP22 created new session: [4:7577789145780622303:2049]
2025-11-28T14:52:04.180736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789145780622303:2049] [node 2] ICS09 handshake done sender: [4:7577789145780622300:2117] self: [4:7577789145780622301:0] peer: [2:7577789145378682356:0] socket: 385 qp: -1
2025-11-28T14:52:04.180780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789145780622303:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:04.180820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.185363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789146318966570:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25650
2025-11-28T14:52:04.188991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789146318966553:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.189922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722549:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.190316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789146318966570:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.190522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789142023999226:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.190859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789145780622294:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.190900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789145780622294:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:04.204038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789146318966552:2111] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.209300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789145780622294:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.230569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789141485654984:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.230650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:04.234526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789146318966572:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25650
2025-11-28T14:52:04.230806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789141485654984:2108] [node 5] ICP22 created new session: [4:7577789145780622307:2108]
2025-11-28T14:52:04.238700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789145780622307:2108] [node 5] ICS09 handshake done sender: [4:7577789145780622294:2115] self: [4:7577789145780622293:0] peer: [5:7577789146318966571:0] socket: 345 qp: -1
2025-11-28T14:52:04.238782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789145780622307:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:04.238885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:04.238949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789145780622292:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.238972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789146318966570:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:04.239171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789142023999226:2085] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.239252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:04.239417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789142023999226:2085] [node 4] ICP22 created new session: [5:7577789146318966573:2085]
2025-11-28T14:52:04.239474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789146318966573:2085] [node 4] ICS09 handshake done sender: [5:7577789146318966570:2120] self: [5:7577789146318966571:0] peer: [4:7577789145780622293:0] socket: 346 qp: -1
2025-11-28T14:52:04.239510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789146318966573:2085] [node 4] ICS10 traffic start
2025-11-28T14:52:04.239552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:04.239635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789141485654984:2108] [node 5] ICP18 handshake (actor [4:7577789145780622292:2114]) is too old
2025-11-28T14:52:04.243231Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789148496968388:2111] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.261856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789145780622296:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.268600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789141485654922:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.268675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789148496968408:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40301
2025-11-28T14:52:04.269058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789148496968388:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.269523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789148496968388:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.269730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789144202001088:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.269786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:04.269842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789144202001088:2108] [node 4] ICP22 created new session: [3:7577789148496968410:2108]
2025-11-28T14:52:04.269814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789145780622310:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-28T14:52:04.269979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789148496968410:2108] [node 4] ICS09 handshake done sender: [3:7577789148496968388:2111] self: [3:7577789148496968387:0] peer: [4:7577789145780622309:0] socket: 358 qp: -1
2025-11-28T14:52:04.270023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789148496968410:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:04.270062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789145780622296:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:04.270081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:04.270195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789141485654922:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.270319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:04.270411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789141485654922:2050] [node 3] ICP22 created new session: [4:7577789145780622311:2050]
2025-11-28T14:52:04.270456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789145780622311:2050] [node 3] ICS09 handshake done sender: [4:7577789145780622296:2116] self: [4:7577789145780622309:0] peer: [3:7577789148496968387:0] socket: 359 qp: -1
2025-11-28T14:52:04.270512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789145780622311:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:04.270553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:04.272435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789148496968386:2110] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.273231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789147530886617:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.273410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789143235919245:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.273578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789148496968386:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.291152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886630:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
2025-11-28T14:52:04.283599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789148496968386:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:04.291338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789144202001087:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.291426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:04.291488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789144202001087:2107] [node 6] ICP22 created new session: [3:7577789148496968413:2107]
2025-11-28T14:52:04.305252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789147530886617:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:04.305274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789143235919245:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28061) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789147530886617:2115] held: no
2025-11-28T14:52:04.291538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789148496968413:2107] [node 6] ICS09 handshake done sender: [3:7577789148496968386:2110] self: [3:7577789148496968385:0] peer: [6:7577789147530886629:0] socket: 354 qp: -1
2025-11-28T14:52:04.291580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789148496968413:2107] [node 6] ICS10 traffic start
2025-11-28T14:52:04.305925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789143235919245:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.291615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:04.305971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:04.306015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789143235919245:2050] [node 3] ICP22 created new session: [6:7577789147530886631:2050]
2025-11-28T14:52:04.306058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789147530886631:2050] [node 3] ICS09 handshake done sender: [6:7577789147530886617:2115] self: [6:7577789147530886629:0] peer: [3:7577789148496968385:0] socket: 357 qp: -1
2025-11-28T14:52:04.306090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789147530886631:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:04.307703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789144202001087:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(22965372ns) {HandleHandshakeStatus:329(15502217ns) {DropIncomingHandshake:442(31330ns) DropOutgoingHandshake:457(911ns) SwitchToState:179(934ns) ProcessPendingSessionEvents:495(15039011ns) {ForwardSessionEventToSession:612(12773072ns) ForwardSessionEventToSession:612(11173ns) ForwardSessionEventToSession:612(3071ns) ForwardSessionEventToSession:612(581157ns) ForwardSessionEventToSession:612(192704ns) ForwardSessionEventToSession:612(2704ns) ForwardSessionEventToSession:612(522960ns) ForwardSessionEventToSession:612(2670ns) ForwardSessionEventToSession:612(930487ns) ForwardSessionEventToSession:612(3899ns)}}}
2025-11-28T14:52:04.306130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:04.334691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789147530886611:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.368305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722545:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.380642Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789146318966551:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.381865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789145378682351:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.408509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789145378682348:2109] [node 5] ICP18 handshake (actor [2:7577789145378682351:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.415996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886633:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
2025-11-28T14:52:04.435884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789148496968393:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.437543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722557:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.472479Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789145780622291:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.484508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789148496968404:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.484847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789144202001088:2108] [node 4] ICP18 handshake (actor [3:7577789148496968404:2119]) is too old
2025-11-28T14:52:04.487969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789147120722544:2122] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.494351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789148496968401:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.494617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789144202001023:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.495113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722544:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.495632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789147120722544:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:04.511201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789142825755169:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.516796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.517230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789142825755169:2049] [node 3] ICP22 created new session: [1:7577789147120722564:2049]
2025-11-28T14:52:04.517298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789147120722564:2049] [node 3] ICS09 handshake done sender: [1:7577789147120722544:2122] self: [1:7577789147120722543:0] peer: [3:7577789148496968415:0] socket: 369 qp: -1
2025-11-28T14:52:04.517347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789147120722564:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:04.517326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789148496968416:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40301
2025-11-28T14:52:04.517401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:04.517775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789144202001023:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789148496968401:2117] held: no
2025-11-28T14:52:04.518188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789148496968401:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.518455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789144202001023:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.518469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886634:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
2025-11-28T14:52:04.518520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:04.518625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789144202001023:2048] [node 1] ICP22 created new session: [3:7577789148496968417:2048]
2025-11-28T14:52:04.518675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789148496968417:2048] [node 1] ICS09 handshake done sender: [3:7577789148496968401:2117] self: [3:7577789148496968415:0] peer: [1:7577789147120722543:0] socket: 370 qp: -1
2025-11-28T14:52:04.518725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789148496968417:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:04.518800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:04.523182Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789147120722551:2127] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.536042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789147120722542:2121] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.545149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789146318966561:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.545427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789142023999187:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.545791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722551:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.545862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789145378682361:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.546033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789141083714974:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.546234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722542:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.546282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789147120722551:2127] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.546576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789142825755242:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.546647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:04.546703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789142825755242:2110] [node 5] ICP22 created new session: [1:7577789147120722569:2110]
2025-11-28T14:52:04.546770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789147120722569:2110] [node 5] ICS09 handshake done sender: [1:7577789147120722551:2127] self: [1:7577789147120722550:0] peer: [5:7577789146318966577:0] socket: 371 qp: -1
2025-11-28T14:52:04.546799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789147120722542:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:04.546722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789146318966578:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25650
2025-11-28T14:52:04.546807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789147120722569:2110] [node 5] ICS10 traffic start
2025-11-28T14:52:04.546841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.547003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789146318966561:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:04.547014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789142825755168:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.547358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789145378682377:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56442
2025-11-28T14:52:04.547198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:04.547254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789142825755168:2048] [node 2] ICP22 created new session: [1:7577789147120722572:2048]
2025-11-28T14:52:04.547982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789145378682361:2125] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:04.547673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789147120722572:2048] [node 2] ICS09 handshake done sender: [1:7577789147120722542:2121] self: [1:7577789147120722541:0] peer: [2:7577789145378682376:0] socket: 367 qp: -1
2025-11-28T14:52:04.547954Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789142023999187:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789146318966561:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.547702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789147120722572:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:04.548167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789142023999187:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.548440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789141083714974:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.547941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:04.548288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:04.548599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789145780622313:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-28T14:52:04.548908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:04.548588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789142023999187:2048] [node 1] ICP22 created new session: [5:7577789146318966579:2048]
2025-11-28T14:52:04.549069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789141083714974:2048] [node 1] ICP22 created new session: [2:7577789145378682378:2048]
2025-11-28T14:52:04.549652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789145378682378:2048] [node 1] ICS09 handshake done sender: [2:7577789145378682361:2125] self: [2:7577789145378682376:0] peer: [1:7577789147120722541:0] socket: 368 qp: -1
2025-11-28T14:52:04.548787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789146318966579:2048] [node 1] ICS09 handshake done sender: [5:7577789146318966561:2115] self: [5:7577789146318966577:0] peer: [1:7577789147120722550:0] socket: 372 qp: -1
2025-11-28T14:52:04.549691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789145378682378:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:04.548900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789146318966579:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:04.548941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:04.549780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:04.550080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789141083714974:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:04.552482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789148496968397:2116] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.569075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789146318966565:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.569328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789142023999189:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.569594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789148496968397:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.570245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789148496968397:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:04.570358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789146318966582:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25650
2025-11-28T14:52:04.570487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789144202001086:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.570549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:04.570606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789144202001086:2109] [node 5] ICP22 created new session: [3:7577789148496968420:2109]
2025-11-28T14:52:04.570698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789148496968420:2109] [node 5] ICS09 handshake done sender: [3:7577789148496968397:2116] self: [3:7577789148496968396:0] peer: [5:7577789146318966581:0] socket: 337 qp: -1
2025-11-28T14:52:04.570740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789148496968420:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:04.570800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:04.570919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789146318966565:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:04.571188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789142023999189:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.571231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:04.571273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789142023999189:2050] [node 3] ICP22 created new session: [5:7577789146318966583:2050]
2025-11-28T14:52:04.571314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789146318966583:2050] [node 3] ICS09 handshake done sender: [5:7577789146318966565:2118] self: [5:7577789146318966581:0] peer: [3:7577789148496968396:0] socket: 386 qp: -1
2025-11-28T14:52:04.571347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789146318966583:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:04.571385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:04.573655Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789146318966558:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.574449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789148496968408:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.574641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789144202001086:2109] [node 5] ICP18 handshake (actor [3:7577789148496968408:2120]) is too old
2025-11-28T14:52:04.607119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789146318966560:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.612682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789147530886633:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.612949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789147530886609:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.613285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789146318966560:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.614505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789146318966560:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.620147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789146318966546:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.627840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:04.628492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886636:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
2025-11-28T14:52:04.638312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789146318966546:2072] [node 6] ICP22 created new session: [5:7577789146318966586:2072]
2025-11-28T14:52:04.638706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789146318966586:2072] [node 6] ICS09 handshake done sender: [5:7577789146318966560:2114] self: [5:7577789146318966559:0] peer: [6:7577789147530886635:0] socket: 338 qp: -1
2025-11-28T14:52:04.638790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789146318966586:2072] [node 6] ICS10 traffic start
2025-11-28T14:52:04.638867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:04.638863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789147530886633:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:04.638870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789147530886609:2060] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12900) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789147530886633:2121] held: no
2025-11-28T14:52:04.639129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789147530886609:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.646220Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789147120722547:2124] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.639442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:04.639519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789147530886609:2060] [node 5] ICP22 created new session: [6:7577789147530886637:2060]
2025-11-28T14:52:04.646989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789146318966546:2072] [node 6] ICP35 event processing took too much time PendingConnection:279(26109657ns) {HandleHandshakeStatus:329(26105047ns) {DropIncomingHandshake:442(32495ns) DropOutgoingHandshake:457(816ns) SwitchToState:179(933ns) ProcessPendingSessionEvents:495(6863383ns) {ForwardSessionEventToSession:612(5191651ns) ForwardSessionEventToSession:612(8323ns) ForwardSessionEventToSession:612(14444ns) ForwardSessionEventToSession:612(3385ns) ForwardSessionEventToSession:612(2865ns) ForwardSessionEventToSession:612(5979ns) ForwardSessionEventToSession:612(3782ns) ForwardSessionEventToSession:612(6132ns) ForwardSessionEventToSession:612(1606485ns) ForwardSessionEventToSession:612(4345ns)}}}
2025-11-28T14:52:04.642191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789147530886637:2060] [node 5] ICS09 handshake done sender: [6:7577789147530886633:2121] self: [6:7577789147530886635:0] peer: [5:7577789146318966559:0] socket: 342 qp: -1
2025-11-28T14:52:04.642249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789147530886637:2060] [node 5] ICS10 traffic start
2025-11-28T14:52:04.642313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:04.662751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789147530886634:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.673271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789143235919243:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.673599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722547:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.674100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789147120722547:2124] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.674287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789142825755241:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.674444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789147530886640:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11864
2025-11-28T14:52:04.674442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:04.674735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789147530886634:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:04.674549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789142825755241:2107] [node 6] ICP22 created new session: [1:7577789147120722575:2107]
2025-11-28T14:52:04.674787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789147120722575:2107] [node 6] ICS09 handshake done sender: [1:7577789147120722547:2124] self: [1:7577789147120722546:0] peer: [6:7577789147530886639:0] socket: 366 qp: -1
2025-11-28T14:52:04.674884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789147120722575:2107] [node 6] ICS10 traffic start
2025-11-28T14:52:04.674927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:04.675735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789143235919243:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789147530886634:2122] held: no
2025-11-28T14:52:04.676128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789143235919243:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.676179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:04.676236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789143235919243:2048] [node 1] ICP22 created new session: [6:7577789147530886641:2048]
2025-11-28T14:52:04.676320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789147530886641:2048] [node 1] ICS09 handshake done sender: [6:7577789147530886634:2122] self: [6:7577789147530886639:0] peer: [1:7577789147120722546:0] socket: 376 qp: -1
2025-11-28T14:52:04.676403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789147530886641:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:04.676459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:04.679835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789147120722556:2128] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:04.680527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789145780622313:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.680759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789141485654920:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:04.680964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789147120722556:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:04.681327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789147120722556:2128] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:04.681553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789147120722539:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:04.681701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:04.681662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789145780622315:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-28T14:52:04.681759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789147120722539:2109] [node 4] ICP22 created new session: [1:7577789147120722578:2109]
2025-11-28T14:52:04.681802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789147120722578:2109] [node 4] ICS09 handshake done sender: [1:7577789147120722556:2128] self: [1:7577789147120722555:0] peer: [4:7577789145780622314:0] socket: 398 qp: -1
2025-11-28T14:52:04.681835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789147120722578:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:04.681869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:04.681965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789145780622313:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:04.682190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789141485654920:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:04.682241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:04.682275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789141485654920:2048] [node 1] ICP22 created new session: [4:7577789145780622316:2048]
2025-11-28T14:52:04.682465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789145780622316:2048] [node 1] ICS09 handshake done sender: [4:7577789145780622313:2120] self: [4:7577789145780622314:0] peer: [1:7577789147120722555:0] socket: 355 qp: -1
2025-11-28T14:52:04.682585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789145780622316:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:04.682630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:04.682941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789141485654920:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:04.687104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:04.687345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789145378682378:2048] [node 1] ICS07 socket disconnect 368 reason# EndOfStream
2025-11-28T14:52:04.687395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789145378682378:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.687486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:04.687533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789145378682378:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:04.689853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789141083714974:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:04.689897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789145378682378:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:04.690810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:04.691231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789141083714974:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:04.691387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789145378682378:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:04.691432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789141083714974:2048] [node 1] ICP30 unregister session Session# [2:7577789145378682378:2048] VirtualId# [2:7577789145378682376:0]
2025-11-28T14:52:04.691490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:04.691526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789148496968417:2048] [node 1] ICS01 socket: 370 reason# EndOfStream
2025-11-28T14:52:04.691530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789141083714974:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.691570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789144202001023:2048] [node 1] ICP30 unregister session Session# [3:7577789148496968417:2048] VirtualId# [3:7577789148496968415:0]
2025-11-28T14:52:04.691547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.691601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789148496968417:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.691681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:04.691728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:04.691672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:04.691906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789147530886641:2048] [node 1] ICS01 socket: 376 reason# EndOfStream
2025-11-28T14:52:04.691952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789143235919243:2048] [node 1] ICP30 unregister session Session# [6:7577789147530886641:2048] VirtualId# [6:7577789147530886639:0]
2025-11-28T14:52:04.691987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789147530886641:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.692058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:04.692102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:04.692869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:04.693105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789145780622316:2048] [node 1] ICS07 socket disconnect 355 reason# EndOfStream
2025-11-28T14:52:04.693145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789145780622316:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.693242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:04.693308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789145780622316:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:04.693862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:04.693997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789146318966579:2048] [node 1] ICS01 socket: 372 reason# EndOfStream
2025-11-28T14:52:04.694027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789142023999187:2048] [node 1] ICP30 unregister session Session# [5:7577789146318966579:2048] VirtualId# [5:7577789146318966577:0]
2025-11-28T14:52:04.694050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789146318966579:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.694070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789141485654920:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:04.694105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:04.694144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:04.694125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789145780622316:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:04.694173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789141485654920:2048] [node 1] ICP30 unregister session Session# [4:7577789145780622316:2048] VirtualId# [4:7577789145780622314:0]
2025-11-28T14:52:04.694239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:04.694269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789141485654920:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.694282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.703973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789141485654920:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:04.744716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:04.744890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789148496968405:2049] [node 2] ICS07 socket disconnect 334 reason# EndOfStream
2025-11-28T14:52:04.744922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789148496968405:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.745014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:04.745059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789148496968405:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:04.747636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:04.747809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789144202001024:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31452) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:04.747855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789148496968405:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:04.747867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789145780622303:2049] [node 2] ICS01 socket: 385 reason# EndOfStream
2025-11-28T14:52:04.747923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789141485654921:2049] [node 2] ICP30 unregister session Session# [4:7577789145780622303:2049] VirtualId# [4:7577789145780622301:0]
2025-11-28T14:52:04.747958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789145780622303:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.748066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:04.748133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:04.748633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789144202001024:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31452) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:04.748752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789148496968405:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:04.748799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789144202001024:2049] [node 2] ICP30 unregister session Session# [3:7577789148496968405:2049] VirtualId# [3:7577789148496968400:0]
2025-11-28T14:52:04.748845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:04.748877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789144202001024:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.748892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.748952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:04.749134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789147530886627:2049] [node 2] ICS01 socket: 362 reason# EndOfStream
2025-11-28T14:52:04.749192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789143235919244:2049] [node 2] ICP30 unregister session Session# [6:7577789147530886627:2049] VirtualId# [6:7577789147530886622:0]
2025-11-28T14:52:04.749219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789147530886627:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.749319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:04.749366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:04.749663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:04.749786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789146318966568:2049] [node 2] ICS07 socket disconnect 379 reason# EndOfStream
2025-11-28T14:52:04.749824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789146318966568:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.749902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:04.749951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789146318966568:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:04.750517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789142023999188:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31452) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:04.750555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789146318966568:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:04.750574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789142023999188:2049] [node 2] ICP30 unregister session Session# [5:7577789146318966568:2049] VirtualId# [5:7577789146318966563:0]
2025-11-28T14:52:04.750635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:04.750674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789142023999188:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.750688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.759865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:04.760039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789147530886631:2050] [node 3] ICS01 socket: 357 reason# EndOfStream
2025-11-28T14:52:04.760076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789143235919245:2050] [node 3] ICP30 unregister session Session# [6:7577789147530886631:2050] VirtualId# [6:7577789147530886629:0]
2025-11-28T14:52:04.760099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789147530886631:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.760174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:04.760221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:04.760303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789142023999188:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:04.760842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:04.761087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789145780622311:2050] [node 3] ICS07 socket disconnect 359 reason# EndOfStream
2025-11-28T14:52:04.761137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789145780622311:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.761256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:04.761313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789145780622311:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:04.761589Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:04.761813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789146318966583:2050] [node 3] ICS07 socket disconnect 386 reason# EndOfStream
2025-11-28T14:52:04.761868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789146318966583:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.761958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:04.762006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789146318966583:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:04.762054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789141485654922:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28061) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:04.762106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789145780622311:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:04.762193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789141485654922:2050] [node 3] ICP30 unregister session Session# [4:7577789145780622311:2050] VirtualId# [4:7577789145780622309:0]
2025-11-28T14:52:04.762258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:04.762287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789141485654922:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28061) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.762296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28061) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.762935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789142023999189:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28061) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:04.763098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789146318966583:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:04.763147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789142023999189:2050] [node 3] ICP30 unregister session Session# [5:7577789146318966583:2050] VirtualId# [5:7577789146318966581:0]
2025-11-28T14:52:04.763196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:04.763244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789142023999189:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28061) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.763266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28061) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.773575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789142023999189:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:04.808909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:04.809178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789146318966573:2085] [node 4] ICS07 socket disconnect 346 reason# EndOfStream
2025-11-28T14:52:04.809237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789146318966573:2085] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.809360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:04.809449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789146318966573:2085] [node 4] ICS15 start handshake
2025-11-28T14:52:04.810047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:04.810269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789147530886623:2062] [node 4] ICS07 socket disconnect 340 reason# EndOfStream
2025-11-28T14:52:04.810285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789142023999226:2085] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21043) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:04.810308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789147530886623:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.810477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:04.810414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789146318966573:2085] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:04.810498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789142023999226:2085] [node 4] ICP30 unregister session Session# [5:7577789146318966573:2085] VirtualId# [5:7577789146318966571:0]
2025-11-28T14:52:04.810559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:04.810530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789147530886623:2062] [node 4] ICS15 start handshake
2025-11-28T14:52:04.810639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789142023999226:2085] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.810661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.811725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789143235919260:2062] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21043) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:04.811846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789147530886623:2062] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:04.812031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789143235919260:2062] [node 4] ICP30 unregister session Session# [6:7577789147530886623:2062] VirtualId# [6:7577789147530886616:0]
2025-11-28T14:52:04.812096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:04.812166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789143235919260:2062] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.812206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:04.821708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789143235919260:2062] [node 4] ICP33 wake up from error state
2025-11-28T14:52:04.822735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:04.822973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789147530886637:2060] [node 5] ICS01 socket: 342 reason# EndOfStream
2025-11-28T14:52:04.823025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789147530886609:2060] [node 5] ICP30 unregister session Session# [6:7577789147530886637:2060] VirtualId# [6:7577789147530886635:0]
2025-11-28T14:52:04.823061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789147530886637:2060] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:04.823212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:04.823283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:04.906986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789148766695025:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:04.907037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789148766695026:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:04.927583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789145427624840:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:04.927627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789145427624841:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:04.948887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789148334410907:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:04.948946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789148334410908:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:04.969921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789145350824149:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:04.969965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789145350824150:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:04.969997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789145350824151:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:04.989175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789144654954118:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:04.989244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789144654954119:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:04.989311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789144654954120:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:05.012576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789152129902081:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:05.012647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789152129902082:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:05.012677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789152129902083:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:05.119318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789152129902144:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:05.121094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789152629378268:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:05.121336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789149722592199:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:05.121789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789152629378269:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:05.122280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789149722592200:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:05.122196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789149645791509:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:05.122491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789152629378270:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:05.122510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789148949921477:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:05.122691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789152129902146:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:05.122836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789149722592201:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:05.122854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789149645791510:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:05.122967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789148949921478:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:05.124449Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789148766695025:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56659675ns) {Configure:104(53776282ns) {ProcessConfigured:135(201322ns) {StartInitialHandshake:157(198332ns) {DropHandshakes:470(2043ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(408ns)} GenerateSessionVirtualId:988(3525ns) PrepareNewSessionHandshake:477(1779ns) {SwitchToState:179(781ns)}}}}}
2025-11-28T14:52:05.125273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789153061662404:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:05.125924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789153061662405:2127] [node 6] ICP01 ready to work
2025-11-28T14:52:05.126478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789153061662406:2117] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.132874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789149722592215:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55549
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.137580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152629378281:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42792
2025-11-28T14:52:05.137678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152629378282:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42792
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.138427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789148949921489:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54542
2025-11-28T14:52:05.138357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153061662415:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1892
2025-11-28T14:52:05.138542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153061662416:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1892
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.141718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153061662418:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1892
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.143684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153061662421:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1892
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.144643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153061662422:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1892
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.144971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662416:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.144981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789149645791524:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46357
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.145040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789149722592215:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.145144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789149722592216:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55549
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:05.145280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789149722592217:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55549
::ffff:127.1.0.0
2025-11-28T14:52:05.145199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789149645791525:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46357
2025-11-28T14:52:05.145286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789149645791526:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46357
2025-11-28T14:52:05.145377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789149722592218:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.145493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789149722592219:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55549
2025-11-28T14:52:05.145712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789149722592216:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.145887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789149645791524:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.146192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789145350824150:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.146700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152629378286:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42792
2025-11-28T14:52:05.146862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789149722592207:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.146898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789149722592207:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.147012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789148949921489:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.147324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152629378282:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.147661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789149722592219:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:05.147871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152629378286:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:05.147906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789149722592218:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.147973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152629378287:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42792
2025-11-28T14:52:05.148113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789148334410908:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.148413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789149722592207:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:05.148708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152629378281:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.148711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789149645791525:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.149280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789149722592205:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.149313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789145350824151:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.149309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789149722592205:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.149415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789149645791528:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46357
2025-11-28T14:52:05.150832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789149722592199:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.150936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:05.150991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789149722592199:2107] [node 4] ICP22 created new session: [2:7577789149722592225:2107]
2025-11-28T14:52:05.151027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789149645791524:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:05.151051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789149722592225:2107] [node 4] ICS09 handshake done sender: [2:7577789149722592207:2112] self: [2:7577789149722592206:0] peer: [4:7577789149645791527:0] socket: 341 qp: -1
2025-11-28T14:52:05.151093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789149722592225:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:05.151145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:05.151195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152629378276:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.151145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662415:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.151450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662418:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.151236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789152629378276:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.151554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789145350824150:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.151595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.151629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789145350824150:2049] [node 2] ICP22 created new session: [4:7577789149645791530:2049]
2025-11-28T14:52:05.151660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662422:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.151660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789149645791530:2049] [node 2] ICS09 handshake done sender: [4:7577789149645791524:2116] self: [4:7577789149645791527:0] peer: [2:7577789149722592206:0] socket: 349 qp: -1
2025-11-28T14:52:05.151682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789149645791530:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:05.151713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:05.151835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662421:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.152229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152629378276:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:05.152713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152629378287:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.153008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789152629378268:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.153113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789149722592217:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.153073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:05.153115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789152629378268:2107] [node 4] ICP22 created new session: [3:7577789152629378293:2107]
2025-11-28T14:52:05.153163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789152629378293:2107] [node 4] ICS09 handshake done sender: [3:7577789152629378276:2114] self: [3:7577789152629378275:0] peer: [4:7577789149645791529:0] socket: 361 qp: -1
2025-11-28T14:52:05.153199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789152629378293:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:05.153231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:05.153392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789149722592205:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.153738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789145427624840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.153694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789149645791526:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.153803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789145427624841:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.153839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:05.153775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789148949921494:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54542
2025-11-28T14:52:05.153865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789145427624841:2049] [node 3] ICP22 created new session: [2:7577789149722592230:2049]
2025-11-28T14:52:05.153897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789149722592230:2049] [node 3] ICS09 handshake done sender: [2:7577789149722592205:2111] self: [2:7577789149722592204:0] peer: [3:7577789152629378288:0] socket: 340 qp: -1
2025-11-28T14:52:05.153918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789149722592230:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:05.153938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.154080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789149645791532:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46357
2025-11-28T14:52:05.154514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789145350824149:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.154475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662403:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.154517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789153061662403:2128] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.155206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152629378292:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42792
2025-11-28T14:52:05.155325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789148949921495:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54542
2025-11-28T14:52:05.155375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153061662403:2128] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:05.156283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789148766695025:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.156342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.156450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789148766695025:2048] [node 2] ICP22 created new session: [1:7577789153061662429:2048]
2025-11-28T14:52:05.157088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789153061662429:2048] [node 2] ICS09 handshake done sender: [1:7577789153061662403:2128] self: [1:7577789153061662402:0] peer: [2:7577789149722592229:0] socket: 354 qp: -1
2025-11-28T14:52:05.157073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789149645791525:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:05.157137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789153061662429:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:05.157189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:05.157481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789148334410907:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.157555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789145350824151:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10407) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789149645791525:2117] held: no
2025-11-28T14:52:05.157580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789148334410908:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64984) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789152629378282:2118] held: no
2025-11-28T14:52:05.157661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789145350824151:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.157797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:05.157807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789148949921494:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.157861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789145350824151:2050] [node 3] ICP22 created new session: [4:7577789149645791535:2050]
2025-11-28T14:52:05.157833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789149722592232:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55549
2025-11-28T14:52:05.157901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789149645791535:2050] [node 3] ICS09 handshake done sender: [4:7577789149645791525:2117] self: [4:7577789149645791529:0] peer: [3:7577789152629378275:0] socket: 376 qp: -1
2025-11-28T14:52:05.157935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789149645791535:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.157934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789144654954119:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.157966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:05.158034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152629378282:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:05.158042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902160:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.158915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789148949921497:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54542
2025-11-28T14:52:05.158931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789149722592209:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.159020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789148334410908:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.159029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662410:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.158961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789149722592209:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.159084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.159131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789148334410908:2049] [node 2] ICP22 created new session: [3:7577789152629378297:2049]
2025-11-28T14:52:05.159062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789153061662410:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.159188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789152629378297:2049] [node 2] ICS09 handshake done sender: [3:7577789152629378282:2118] self: [3:7577789152629378288:0] peer: [2:7577789149722592204:0] socket: 346 qp: -1
2025-11-28T14:52:05.159382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789152629378297:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:05.159524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:05.161014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789149722592217:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:05.161370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662408:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.161354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789148949921495:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.161469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789148949921477:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.161404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789153061662408:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.162049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789145427624840:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789149722592217:2117] held: no
2025-11-28T14:52:05.162679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789145427624840:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.162737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.162800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789145427624840:2048] [node 1] ICP22 created new session: [2:7577789149722592235:2048]
2025-11-28T14:52:05.163109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789149722592235:2048] [node 1] ICS09 handshake done sender: [2:7577789149722592217:2117] self: [2:7577789149722592229:0] peer: [1:7577789153061662402:0] socket: 372 qp: -1
2025-11-28T14:52:05.163252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789149722592235:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.163373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:05.163448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789149645791518:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.163485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789149645791518:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.163612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153061662410:2130] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.163863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902161:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
2025-11-28T14:52:05.164081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902162:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.164263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902163:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
2025-11-28T14:52:05.164300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789148949921497:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.164327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153061662408:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:05.164391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789152129902161:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.164379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789149645791538:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46357
2025-11-28T14:52:05.164496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789152129902146:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.164973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789149645791518:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:05.164951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789144654954120:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.165013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789148949921499:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54542
2025-11-28T14:52:05.165017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789153061662404:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.165077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:05.165144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789153061662404:2107] [node 4] ICP22 created new session: [1:7577789153061662434:2107]
2025-11-28T14:52:05.165539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789153061662434:2107] [node 4] ICS09 handshake done sender: [1:7577789153061662410:2130] self: [1:7577789153061662409:0] peer: [4:7577789149645791534:0] socket: 369 qp: -1
2025-11-28T14:52:05.165738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789153061662434:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:05.165807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.166087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152629378300:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42792
2025-11-28T14:52:05.165962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789149645791520:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.165990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789149645791520:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.166498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789152129902162:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.166536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789149645791509:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.166559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789148949921495:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:05.166573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:05.166667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789148766695026:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.166754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789149722592209:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:05.166624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789149645791509:2108] [node 5] ICP22 created new session: [4:7577789149645791540:2108]
2025-11-28T14:52:05.166748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:05.166773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789152129902160:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.166919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789152129902083:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.167098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789148766695026:2049] [node 3] ICP22 created new session: [1:7577789153061662436:2049]
2025-11-28T14:52:05.166734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789149645791540:2108] [node 5] ICS09 handshake done sender: [4:7577789149645791518:2114] self: [4:7577789149645791517:0] peer: [5:7577789148949921498:0] socket: 345 qp: -1
2025-11-28T14:52:05.167182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789152129902082:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.167197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789148949921477:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.166854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789149645791540:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:05.167217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789153061662436:2049] [node 3] ICS09 handshake done sender: [1:7577789153061662408:2129] self: [1:7577789153061662407:0] peer: [3:7577789152629378296:0] socket: 368 qp: -1
2025-11-28T14:52:05.166898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:05.167402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789153061662436:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:05.167376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:05.167434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:05.167429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789148949921477:2108] [node 4] ICP22 created new session: [5:7577789148949921502:2108]
2025-11-28T14:52:05.167482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152629378278:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.167463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789148949921502:2108] [node 4] ICS09 handshake done sender: [5:7577789148949921495:2117] self: [5:7577789148949921498:0] peer: [4:7577789149645791517:0] socket: 373 qp: -1
2025-11-28T14:52:05.167530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789149645791526:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:05.167482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789148949921502:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:05.167518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789152629378278:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.167517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:05.167550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789149722592200:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.167662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:05.168023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789149722592200:2108] [node 5] ICP22 created new session: [2:7577789149722592237:2108]
2025-11-28T14:52:05.168134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789149722592237:2108] [node 5] ICS09 handshake done sender: [2:7577789149722592209:2113] self: [2:7577789149722592208:0] peer: [5:7577789148949921496:0] socket: 382 qp: -1
2025-11-28T14:52:05.168155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789145350824149:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.168174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789149722592237:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:05.168204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.168208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:05.168251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789145350824149:2048] [node 1] ICP22 created new session: [4:7577789149645791543:2048]
2025-11-28T14:52:05.168294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789149645791543:2048] [node 1] ICS09 handshake done sender: [4:7577789149645791526:2118] self: [4:7577789149645791534:0] peer: [1:7577789153061662409:0] socket: 377 qp: -1
2025-11-28T14:52:05.168318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789149645791543:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.168353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:05.168405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152629378280:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.168433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789152629378280:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.169212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789149722592211:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.169240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789149722592211:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.169685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152629378287:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.170228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789148949921501:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54542
2025-11-28T14:52:05.170520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789148334410907:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.170667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.170947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789148334410907:2048] [node 1] ICP22 created new session: [3:7577789152629378303:2048]
2025-11-28T14:52:05.171040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789152629378303:2048] [node 1] ICS09 handshake done sender: [3:7577789152629378287:2120] self: [3:7577789152629378296:0] peer: [1:7577789153061662407:0] socket: 370 qp: -1
2025-11-28T14:52:05.171082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789152629378303:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.171118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:05.171437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789149722592211:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:05.172822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789149645791520:2115] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.173588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789149645791510:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.173553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902167:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
2025-11-28T14:52:05.173631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:05.173652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902168:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
2025-11-28T14:52:05.173684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789149645791510:2109] [node 6] ICP22 created new session: [4:7577789149645791545:2109]
2025-11-28T14:52:05.173573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789149722592201:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.173708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:05.173738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789149645791545:2109] [node 6] ICS09 handshake done sender: [4:7577789149645791520:2115] self: [4:7577789149645791519:0] peer: [6:7577789152129902164:0] socket: 337 qp: -1
2025-11-28T14:52:05.173787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789149722592201:2109] [node 6] ICP22 created new session: [2:7577789149722592240:2109]
2025-11-28T14:52:05.173784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789149645791545:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:05.173825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:05.173849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789149722592240:2109] [node 6] ICS09 handshake done sender: [2:7577789149722592211:2114] self: [2:7577789149722592210:0] peer: [6:7577789152129902166:0] socket: 339 qp: -1
2025-11-28T14:52:05.173906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789149722592240:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:05.173995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:05.174951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789144654954119:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64984) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789148949921494:2116] held: no
2025-11-28T14:52:05.175058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152629378278:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:05.175216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789152129902161:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:05.175286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789148949921494:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.175939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152629378280:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:05.176122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902169:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
2025-11-28T14:52:05.176321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789144654954119:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.176389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.176438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789144654954119:2049] [node 2] ICP22 created new session: [5:7577789148949921506:2049]
2025-11-28T14:52:05.176498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789148949921506:2049] [node 2] ICS09 handshake done sender: [5:7577789148949921494:2116] self: [5:7577789148949921496:0] peer: [2:7577789149722592208:0] socket: 338 qp: -1
2025-11-28T14:52:05.176551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789148949921506:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:05.176583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:05.176860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789152629378269:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.176923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:05.176976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789152629378269:2108] [node 5] ICP22 created new session: [3:7577789152629378305:2108]
2025-11-28T14:52:05.177031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789152629378305:2108] [node 5] ICS09 handshake done sender: [3:7577789152629378278:2115] self: [3:7577789152629378277:0] peer: [5:7577789148949921500:0] socket: 386 qp: -1
2025-11-28T14:52:05.177024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789152129902163:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.177065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789152629378305:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:05.177063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789152129902146:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.177092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:05.177093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:05.177129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789152129902146:2109] [node 4] ICP22 created new session: [6:7577789152129902170:2109]
2025-11-28T14:52:05.177167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789152129902170:2109] [node 4] ICS09 handshake done sender: [6:7577789152129902161:2117] self: [6:7577789152129902164:0] peer: [4:7577789149645791519:0] socket: 388 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.177206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789152129902170:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:05.177232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:05.177687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789148949921504:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54542
2025-11-28T14:52:05.177876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789152629378270:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.177913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:05.177950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789152629378270:2109] [node 6] ICP22 created new session: [3:7577789152629378307:2109]
2025-11-28T14:52:05.177992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789152629378307:2109] [node 6] ICS09 handshake done sender: [3:7577789152629378280:2116] self: [3:7577789152629378279:0] peer: [6:7577789152129902165:0] socket: 387 qp: -1
2025-11-28T14:52:05.178246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789152629378307:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:05.178277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:05.178766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789148949921497:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.179517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789148949921508:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54542
2025-11-28T14:52:05.179654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789144654954120:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.179703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:05.179762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789144654954120:2050] [node 3] ICP22 created new session: [5:7577789148949921509:2050]
2025-11-28T14:52:05.179803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789148949921509:2050] [node 3] ICS09 handshake done sender: [5:7577789148949921497:2118] self: [5:7577789148949921500:0] peer: [3:7577789152629378277:0] socket: 348 qp: -1
2025-11-28T14:52:05.179835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789148949921509:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:05.179921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:05.180063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789152129902082:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64984) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789152129902160:2116] held: no
2025-11-28T14:52:05.180520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789152129902160:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:05.181064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789152129902162:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:05.181265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789152129902082:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.181297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.181326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789152129902082:2049] [node 2] ICP22 created new session: [6:7577789152129902173:2049]
2025-11-28T14:52:05.181353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789152129902173:2049] [node 2] ICS09 handshake done sender: [6:7577789152129902160:2116] self: [6:7577789152129902166:0] peer: [2:7577789149722592210:0] socket: 358 qp: -1
2025-11-28T14:52:05.181376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789152129902173:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:05.181394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:05.182053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789152129902081:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.182150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789152129902083:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.182200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:05.182224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789152129902083:2050] [node 3] ICP22 created new session: [6:7577789152129902177:2050]
2025-11-28T14:52:05.182250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789152129902177:2050] [node 3] ICS09 handshake done sender: [6:7577789152129902162:2118] self: [6:7577789152129902165:0] peer: [3:7577789152629378279:0] socket: 389 qp: -1
2025-11-28T14:52:05.182273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789152129902177:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:05.182297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.183456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789148949921508:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.183441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902172:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
2025-11-28T14:52:05.183610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789144654954120:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:05.183554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662412:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.183591Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789153061662412:2131] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.184507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789144654954118:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.184597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152629378309:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42792
2025-11-28T14:52:05.184775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789152129902172:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.184870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153061662414:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.184906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789153061662414:2132] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.185094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789152129902144:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.185927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902179:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
2025-11-28T14:52:05.186170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789148949921488:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.186167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153061662412:2131] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:05.186200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789148949921488:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.186953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789152129902163:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:05.187026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789153061662405:2127] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.187099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:05.187156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789153061662405:2127] [node 6] ICP22 created new session: [1:7577789153061662441:2127]
2025-11-28T14:52:05.187139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789148949921488:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:05.187203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789153061662441:2127] [node 6] ICS09 handshake done sender: [1:7577789153061662412:2131] self: [1:7577789153061662411:0] peer: [6:7577789152129902175:0] socket: 391 qp: -1
2025-11-28T14:52:05.187236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789153061662441:2127] [node 6] ICS10 traffic start
2025-11-28T14:52:05.187265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:05.187849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789152129902081:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.187908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.187958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789152129902081:2048] [node 1] ICP22 created new session: [6:7577789152129902182:2048]
2025-11-28T14:52:05.187996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789152129902182:2048] [node 1] ICS09 handshake done sender: [6:7577789152129902163:2119] self: [6:7577789152129902175:0] peer: [1:7577789153061662411:0] socket: 393 qp: -1
2025-11-28T14:52:05.188006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789148949921478:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.188031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789152129902182:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.188065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:05.188067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:05.188110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789148949921478:2109] [node 6] ICP22 created new session: [5:7577789148949921514:2109]
2025-11-28T14:52:05.188168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789148949921514:2109] [node 6] ICS09 handshake done sender: [5:7577789148949921488:2114] self: [5:7577789148949921487:0] peer: [6:7577789152129902180:0] socket: 360 qp: -1
2025-11-28T14:52:05.188205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789148949921514:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:05.188233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:05.189098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153061662414:2132] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.189751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789153061662406:2117] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.189797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:05.189834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789153061662406:2117] [node 5] ICP22 created new session: [1:7577789153061662443:2117]
2025-11-28T14:52:05.189755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789148949921513:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54542
2025-11-28T14:52:05.189945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789153061662443:2117] [node 5] ICS09 handshake done sender: [1:7577789153061662414:2132] self: [1:7577789153061662413:0] peer: [5:7577789148949921511:0] socket: 336 qp: -1
2025-11-28T14:52:05.189992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789153061662443:2117] [node 5] ICS10 traffic start
2025-11-28T14:52:05.190022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:05.190267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789148949921508:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:05.190565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789144654954118:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.190607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.190654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789144654954118:2048] [node 1] ICP22 created new session: [5:7577789148949921516:2048]
2025-11-28T14:52:05.190717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789148949921516:2048] [node 1] ICS09 handshake done sender: [5:7577789148949921508:2122] self: [5:7577789148949921511:0] peer: [1:7577789153061662413:0] socket: 356 qp: -1
2025-11-28T14:52:05.190755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789148949921516:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.190786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.191908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789152129902181:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31516
2025-11-28T14:52:05.192343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789152129902144:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3797) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789152129902172:2123] held: no
2025-11-28T14:52:05.192560Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789152129902172:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:05.193055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789152129902144:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.193089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:05.193183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789152129902144:2107] [node 5] ICP22 created new session: [6:7577789152129902184:2107]
2025-11-28T14:52:05.193217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789152129902184:2107] [node 5] ICS09 handshake done sender: [6:7577789152129902172:2123] self: [6:7577789152129902180:0] peer: [5:7577789148949921487:0] socket: 352 qp: -1
2025-11-28T14:52:05.193248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789152129902184:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:05.193275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:05.212867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:05.213167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789149722592235:2048] [node 1] ICS01 socket: 372 reason# EndOfStream
2025-11-28T14:52:05.213222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789145427624840:2048] [node 1] ICP30 unregister session Session# [2:7577789149722592235:2048] VirtualId# [2:7577789149722592229:0]
2025-11-28T14:52:05.213268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789149722592235:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.213363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:05.213423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:05.214518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:05.214808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789152629378303:2048] [node 1] ICS01 socket: 370 reason# EndOfStream
2025-11-28T14:52:05.214890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789148334410907:2048] [node 1] ICP30 unregister session Session# [3:7577789152629378303:2048] VirtualId# [3:7577789152629378296:0]
2025-11-28T14:52:05.214927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789152629378303:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.215028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:05.215087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:05.216541Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:05.216784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789149645791543:2048] [node 1] ICS01 socket: 377 reason# EndOfStream
2025-11-28T14:52:05.216835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789145350824149:2048] [node 1] ICP30 unregister session Session# [4:7577789149645791543:2048] VirtualId# [4:7577789149645791534:0]
2025-11-28T14:52:05.216868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789149645791543:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.216949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:05.217020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:05.217511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:05.217714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789148949921516:2048] [node 1] ICS01 socket: 356 reason# EndOfStream
2025-11-28T14:52:05.217770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789144654954118:2048] [node 1] ICP30 unregister session Session# [5:7577789148949921516:2048] VirtualId# [5:7577789148949921511:0]
2025-11-28T14:52:05.217808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789148949921516:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.217880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:05.217938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:05.218554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:05.218767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789152129902182:2048] [node 1] ICS07 socket disconnect 393 reason# EndOfStream
2025-11-28T14:52:05.218802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789152129902182:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.218876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:05.218942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789152129902182:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:05.219889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789152129902081:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25607) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:05.219942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789152129902182:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:05.219977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789152129902081:2048] [node 1] ICP30 unregister session Session# [6:7577789152129902182:2048] VirtualId# [6:7577789152129902175:0]
2025-11-28T14:52:05.220027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:05.220066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789152129902081:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25607) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:05.220081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25607) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:05.230308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789152129902081:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:05.246268Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:05.246543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789152629378297:2049] [node 2] ICS01 socket: 346 reason# EndOfStream
2025-11-28T14:52:05.246605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789148334410908:2049] [node 2] ICP30 unregister session Session# [3:7577789152629378297:2049] VirtualId# [3:7577789152629378288:0]
2025-11-28T14:52:05.246645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789152629378297:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.246771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:05.246834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:05.249161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:05.249413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789149645791530:2049] [node 2] ICS01 socket: 349 reason# EndOfStream
2025-11-28T14:52:05.249475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789145350824150:2049] [node 2] ICP30 unregister session Session# [4:7577789149645791530:2049] VirtualId# [4:7577789149645791527:0]
2025-11-28T14:52:05.249522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789149645791530:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.249635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:05.249695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:05.250028Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:05.250226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789148949921506:2049] [node 2] ICS01 socket: 338 reason# EndOfStream
2025-11-28T14:52:05.250265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789144654954119:2049] [node 2] ICP30 unregister session Session# [5:7577789148949921506:2049] VirtualId# [5:7577789148949921496:0]
2025-11-28T14:52:05.250320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789148949921506:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.250401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:05.250456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:05.250938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:05.251134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789152129902173:2049] [node 2] ICS01 socket: 358 reason# EndOfStream
2025-11-28T14:52:05.251189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789152129902082:2049] [node 2] ICP30 unregister session Session# [6:7577789152129902173:2049] VirtualId# [6:7577789152129902166:0]
2025-11-28T14:52:05.251222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789152129902173:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.251322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:05.251391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:05.277583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:05.277832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789149645791535:2050] [node 3] ICS01 socket: 376 reason# EndOfStream
2025-11-28T14:52:05.277891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789145350824151:2050] [node 3] ICP30 unregister session Session# [4:7577789149645791535:2050] VirtualId# [4:7577789149645791529:0]
2025-11-28T14:52:05.277938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789149645791535:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.278049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:05.278099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:05.278454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:05.278655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789148949921509:2050] [node 3] ICS07 socket disconnect 348 reason# EndOfStream
2025-11-28T14:52:05.278688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789148949921509:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.278763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:05.278882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789148949921509:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:05.279416Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:05.279615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789152129902177:2050] [node 3] ICS07 socket disconnect 389 reason# EndOfStream
2025-11-28T14:52:05.279650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789152129902177:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.279753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:05.279803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789152129902177:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:05.279883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789144654954120:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10407) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:05.280057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789148949921509:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:05.280104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789144654954120:2050] [node 3] ICP30 unregister session Session# [5:7577789148949921509:2050] VirtualId# [5:7577789148949921500:0]
2025-11-28T14:52:05.280161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:05.280190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789144654954120:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10407) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:05.280204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10407) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:05.280795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789152129902083:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10407) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:05.280852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789152129902177:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:05.280892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789152129902083:2050] [node 3] ICP30 unregister session Session# [6:7577789152129902177:2050] VirtualId# [6:7577789152129902165:0]
2025-11-28T14:52:05.280931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:05.280957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789152129902083:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10407) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:05.280971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10407) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:05.289662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789144654954120:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:05.290660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789152129902083:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:05.301149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:05.301390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789148949921502:2108] [node 4] ICS01 socket: 373 reason# EndOfStream
2025-11-28T14:52:05.301441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789148949921477:2108] [node 4] ICP30 unregister session Session# [5:7577789148949921502:2108] VirtualId# [5:7577789148949921498:0]
2025-11-28T14:52:05.301479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789148949921502:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.301618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:05.301692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:05.302177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:05.302367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789152129902170:2109] [node 4] ICS07 socket disconnect 388 reason# EndOfStream
2025-11-28T14:52:05.302410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789152129902170:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.302499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:05.302569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789152129902170:2109] [node 4] ICS15 start handshake
2025-11-28T14:52:05.303413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789152129902146:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5557) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:05.303488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789152129902170:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:05.303528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789152129902146:2109] [node 4] ICP30 unregister session Session# [6:7577789152129902170:2109] VirtualId# [6:7577789152129902164:0]
2025-11-28T14:52:05.303577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:05.303613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789152129902146:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:05.303629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:05.313250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789152129902146:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:52:05.330926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:05.331167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789152129902184:2107] [node 5] ICS01 socket: 352 reason# EndOfStream
2025-11-28T14:52:05.331236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789152129902144:2107] [node 5] ICP30 unregister session Session# [6:7577789152129902184:2107] VirtualId# [6:7577789152129902180:0]
2025-11-28T14:52:05.331279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789152129902184:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:05.331421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:05.331485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:05.395295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789152110749225:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:05.395367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789152110749226:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:05.416772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789150038309851:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:05.416808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789150038309852:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:05.440846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789152967593280:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:05.440864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789152967593279:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:05.463154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789152851978407:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:05.463232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789152851978408:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:05.463350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789152851978409:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:05.485215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789151609188091:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:05.485275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789151609188093:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:05.485278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789151609188092:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:05.509026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789149430756399:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:05.509089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789149430756400:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:05.509132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789149430756401:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:05.621861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789149430756460:2104] [node 5] ICP01 ready to work
2025-11-28T14:52:05.622264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789152110749288:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:05.622276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789152851978470:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:05.623037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789152851978471:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:05.623421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789151609188154:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:05.624655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789149430756463:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:05.625597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789151609188155:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:05.625135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789152110749289:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:05.626658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789151609188091:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43511777ns) {Configure:104(43505839ns) {ProcessConfigured:135(61771ns) {StartInitialHandshake:157(59037ns) {DropHandshakes:470(6622ns) {DropIncomingHandshake:442(828ns) DropOutgoingHandshake:457(491ns)} GenerateSessionVirtualId:988(652ns) PrepareNewSessionHandshake:477(1073ns) {SwitchToState:179(302ns)}}}}}
2025-11-28T14:52:05.627153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789152110749290:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:05.627428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789152110749225:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58863104ns) {Configure:104(58852048ns) {ProcessConfigured:135(65119ns) {StartInitialHandshake:157(63141ns) {DropHandshakes:470(1845ns) {DropIncomingHandshake:442(265ns) DropOutgoingHandshake:457(461ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:477(706ns) {SwitchToState:179(326ns)}}}}}
2025-11-28T14:52:05.627820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789150038309852:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(61058871ns) {Configure:104(61053444ns) {ProcessConfigured:135(84547ns) {StartInitialHandshake:157(82996ns) {DropHandshakes:470(7080ns) {DropIncomingHandshake:442(542ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:477(1065ns) {SwitchToState:179(493ns)}}}}}
2025-11-28T14:52:05.627857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789150038309851:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(58415701ns) {Configure:104(58409852ns) {ProcessConfigured:135(90862ns) {StartInitialHandshake:157(88707ns) {DropHandshakes:470(1681ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:477(935ns) {SwitchToState:179(314ns)}}}}}
2025-11-28T14:52:05.628221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789150038309920:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:05.628150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789152967593280:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51559706ns) {Configure:104(51551805ns) {ProcessConfigured:135(132688ns) {StartInitialHandshake:157(130942ns) {DropHandshakes:470(1236ns) {DropIncomingHandshake:442(404ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(491ns) PrepareNewSessionHandshake:477(977ns) {SwitchToState:179(491ns)}}}}}
2025-11-28T14:52:05.628315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789150038309921:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:05.628527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789152967593279:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(58739851ns) {Configure:104(58732799ns) {ProcessConfigured:135(53239ns) {StartInitialHandshake:157(51001ns) {DropHandshakes:470(1030ns) {DropIncomingHandshake:442(268ns) DropOutgoingHandshake:457(159ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:477(971ns) {SwitchToState:179(357ns)}}}}}
2025-11-28T14:52:05.628769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789152967593347:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:05.628877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789149430756399:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44354657ns) {Configure:104(44351388ns) {ProcessConfigured:135(198266ns) {StartInitialHandshake:157(196650ns) {DropHandshakes:470(1262ns) {DropIncomingHandshake:442(424ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(734ns) PrepareNewSessionHandshake:477(1217ns) {SwitchToState:179(459ns)}}}}}
2025-11-28T14:52:05.628882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789150038309922:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:05.629572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789152967593348:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:05.630075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789152967593349:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.647381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789150038309930:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53348
2025-11-28T14:52:05.648559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789149430756463:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16090055ns) {Configure:104(16085547ns) {ProcessConfigured:135(458373ns) {StartInitialHandshake:157(454593ns) {DropHandshakes:470(3590ns) {DropIncomingHandshake:442(779ns) DropOutgoingHandshake:457(411ns)} GenerateSessionVirtualId:988(832ns) PrepareNewSessionHandshake:477(2034ns) {SwitchToState:179(787ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.650553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789152110749301:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20251
2025-11-28T14:52:05.651957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789152967593349:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17460759ns) {Configure:104(17456805ns) {ProcessConfigured:135(755248ns) {StartInitialHandshake:157(751669ns) {DropHandshakes:470(2971ns) {DropIncomingHandshake:442(687ns) DropOutgoingHandshake:457(639ns)} GenerateSessionVirtualId:988(641ns) PrepareNewSessionHandshake:477(3223ns) {SwitchToState:179(1259ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.652247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789152851978471:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18670113ns) {Configure:104(18665588ns) {ProcessConfigured:135(1527521ns) {StartInitialHandshake:157(1525143ns) {DropHandshakes:470(1348ns) {DropIncomingHandshake:442(444ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:477(2136ns) {SwitchToState:179(906ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.651627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789152110749304:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20251
2025-11-28T14:52:05.652805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789152110749306:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20251
2025-11-28T14:52:05.652762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152967593356:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21757
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.652763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152967593357:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21757
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.661834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749301:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.660123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789151609188169:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25330
2025-11-28T14:52:05.660985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789150038309934:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53348
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.660849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152967593356:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.662756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789152110749307:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20251
2025-11-28T14:52:05.661332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152967593360:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21757
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
2025-11-28T14:52:05.664407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789152110749308:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20251
2025-11-28T14:52:05.662024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789150038309930:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.666014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789149430756477:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
2025-11-28T14:52:05.666156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152967593357:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.664167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789152851978480:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15184
2025-11-28T14:52:05.663343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789150038309933:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53348
2025-11-28T14:52:05.666297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152967593360:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.666245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749304:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.667083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789150038309934:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.667328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749307:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.665992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789152851978483:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15184
No RDMA context found for GID: 2025-11-28T14:52:05.667582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789150038309851:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.667447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789152851978483:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:05.678138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789150038309935:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53348
2025-11-28T14:52:05.667681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789152851978480:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.693712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749292:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.694026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789151609188170:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25330
2025-11-28T14:52:05.693749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789152110749290:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20308478ns) {IncomingHandshake:261(9666757ns) {DropIncomingHandshake:442(683ns) PrepareNewSessionHandshake:477(1786ns) {SwitchToState:179(985ns)} GenerateSessionVirtualId:988(815ns) IssueIncomingHandshakeReply:193(10312ns)}}
2025-11-28T14:52:05.694112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789150038309935:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.693770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789152110749292:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.695598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789152110749292:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.695811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789150038309939:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53348
2025-11-28T14:52:05.695945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789152110749225:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.696034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.696100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789152110749225:2048] [node 2] ICP22 created new session: [1:7577789152110749313:2048]
2025-11-28T14:52:05.696198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789152110749313:2048] [node 2] ICS09 handshake done sender: [1:7577789152110749292:2110] self: [1:7577789152110749291:0] peer: [2:7577789150038309937:0] socket: 353 qp: -1
2025-11-28T14:52:05.696247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789152110749313:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:05.696232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789150038309934:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:05.696299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:05.696710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789150038309851:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.696767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.696821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789150038309851:2048] [node 1] ICP22 created new session: [2:7577789150038309940:2048]
2025-11-28T14:52:05.696982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789150038309940:2048] [node 1] ICS09 handshake done sender: [2:7577789150038309934:2119] self: [2:7577789150038309937:0] peer: [1:7577789152110749291:0] socket: 359 qp: -1
2025-11-28T14:52:05.697053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789150038309940:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.697089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.698686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789151609188171:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25330
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.699304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789151609188172:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25330
2025-11-28T14:52:05.700562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789151609188172:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.700739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789151609188171:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.700933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789151609188091:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.700946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789151609188092:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.701233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789150038309926:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.701266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789150038309926:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.701256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749298:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.701291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789152110749298:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.703369Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789150038309926:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:05.703672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789150038309921:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.703742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:05.703731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789152110749298:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:05.703795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789150038309921:2108] [node 5] ICP22 created new session: [2:7577789150038309944:2108]
2025-11-28T14:52:05.703862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789150038309944:2108] [node 5] ICS09 handshake done sender: [2:7577789150038309926:2115] self: [2:7577789150038309924:0] peer: [5:7577789151609188173:0] socket: 346 qp: -1
2025-11-28T14:52:05.704068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789152110749289:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.703904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789150038309944:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:05.704154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:05.703945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:05.704311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789152110749289:2108] [node 5] ICP22 created new session: [1:7577789152110749317:2108]
2025-11-28T14:52:05.704523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789152110749317:2108] [node 5] ICS09 handshake done sender: [1:7577789152110749298:2113] self: [1:7577789152110749297:0] peer: [5:7577789151609188174:0] socket: 351 qp: -1
2025-11-28T14:52:05.704653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789152110749317:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:05.704706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.704955Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789151609188092:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25808) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789151609188172:2118] held: no
2025-11-28T14:52:05.705622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789151609188176:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25330
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.705900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789149430756478:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.706131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789151609188175:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25330
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.706455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789149430756479:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
2025-11-28T14:52:05.707614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789151609188091:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6991) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789151609188171:2117] held: no
2025-11-28T14:52:05.708638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789149430756478:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.709001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789151609188171:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:05.708949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789149430756467:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:05.709140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789149430756400:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.736570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789151609188172:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:05.740931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789151609188091:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.709401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789149430756479:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.736515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789150038309932:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.741002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749300:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.741044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789152110749300:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.740487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789149430756399:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.741072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.736589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789150038309932:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.741404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789151609188091:2048] [node 1] ICP22 created new session: [5:7577789151609188177:2048]
2025-11-28T14:52:05.741479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789151609188092:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.741451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789151609188177:2048] [node 1] ICS09 handshake done sender: [5:7577789151609188171:2117] self: [5:7577789151609188174:0] peer: [1:7577789152110749297:0] socket: 386 qp: -1
2025-11-28T14:52:05.741934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.742024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789151609188177:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.742061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789151609188092:2049] [node 2] ICP22 created new session: [5:7577789151609188179:2049]
2025-11-28T14:52:05.742094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:05.742282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789150038309932:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:05.742115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789151609188179:2049] [node 2] ICS09 handshake done sender: [5:7577789151609188172:2118] self: [5:7577789151609188173:0] peer: [2:7577789150038309924:0] socket: 387 qp: -1
2025-11-28T14:52:05.742171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789151609188179:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:05.742211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:05.742796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789151609188169:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.743092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789150038309922:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.742857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789152110749300:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.744519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:05.744813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789150038309922:2109] [node 6] ICP22 created new session: [2:7577789150038309948:2109]
2025-11-28T14:52:05.744933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789150038309948:2109] [node 6] ICS09 handshake done sender: [2:7577789150038309932:2117] self: [2:7577789150038309931:0] peer: [6:7577789149430756480:0] socket: 348 qp: -1
2025-11-28T14:52:05.744971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789150038309948:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:05.745019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:05.743417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789152110749290:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.744856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789149430756482:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
2025-11-28T14:52:05.743484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:05.744844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789152110749290:2109] [node 6] ICP22 created new session: [1:7577789152110749321:2109]
2025-11-28T14:52:05.744998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789152110749321:2109] [node 6] ICS09 handshake done sender: [1:7577789152110749300:2114] self: [1:7577789152110749299:0] peer: [6:7577789149430756481:0] socket: 343 qp: -1
2025-11-28T14:52:05.745034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789152110749321:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:05.745065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.746380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789149430756483:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
2025-11-28T14:52:05.757697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789152967593346:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:05.747605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789149430756399:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6991) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789149430756479:2117] held: no
2025-11-28T14:52:05.757743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789149430756478:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:05.775946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789149430756479:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:05.776174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789149430756400:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25808) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789149430756478:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.776312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789149430756400:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.776348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789149430756484:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
2025-11-28T14:52:05.775928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749306:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.776395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.776440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789149430756400:2049] [node 2] ICP22 created new session: [6:7577789149430756485:2049]
2025-11-28T14:52:05.776493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789149430756485:2049] [node 2] ICS09 handshake done sender: [6:7577789149430756478:2116] self: [6:7577789149430756480:0] peer: [2:7577789150038309931:0] socket: 349 qp: -1
2025-11-28T14:52:05.776533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789149430756399:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.776535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789149430756485:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:05.776573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.776575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:05.776627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789149430756399:2048] [node 1] ICP22 created new session: [6:7577789149430756488:2048]
2025-11-28T14:52:05.776676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789149430756488:2048] [node 1] ICS09 handshake done sender: [6:7577789149430756479:2117] self: [6:7577789149430756481:0] peer: [1:7577789152110749299:0] socket: 392 qp: -1
2025-11-28T14:52:05.776709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789149430756488:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.776743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:05.783942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789149430756484:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.784111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789149430756460:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.786853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789149430756492:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
2025-11-28T14:52:05.787158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789149430756484:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:05.784775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789151609188168:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.784825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789151609188168:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.786529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789151609188168:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:05.787554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789149430756460:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.786873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789151609188155:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.786940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:05.787692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:05.786985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789151609188155:2109] [node 6] ICP22 created new session: [5:7577789151609188185:2109]
2025-11-28T14:52:05.787755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789149430756460:2104] [node 5] ICP22 created new session: [6:7577789149430756493:2104]
2025-11-28T14:52:05.787033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789151609188185:2109] [node 6] ICS09 handshake done sender: [5:7577789151609188168:2114] self: [5:7577789151609188167:0] peer: [6:7577789149430756491:0] socket: 397 qp: -1
2025-11-28T14:52:05.787066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789151609188185:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:05.787091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:05.787811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789149430756493:2104] [node 5] ICS09 handshake done sender: [6:7577789149430756484:2120] self: [6:7577789149430756491:0] peer: [5:7577789151609188167:0] socket: 374 qp: -1
2025-11-28T14:52:05.787856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789149430756493:2104] [node 5] ICS10 traffic start
2025-11-28T14:52:05.787889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:05.798328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789152967593344:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:05.830756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789150038309933:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.830720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789152967593280:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.830824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789152967593279:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.831079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152967593363:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21757
2025-11-28T14:52:05.831477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789150038309918:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.831517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789150038309918:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.831719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152967593366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21757
2025-11-28T14:52:05.831970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749294:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.832006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789152110749294:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.833080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789150038309918:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:05.832503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152967593366:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.833659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789152110749294:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:05.833781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789150038309852:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.833848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.834018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789150038309852:2049] [node 3] ICP22 created new session: [2:7577789150038309952:2049]
2025-11-28T14:52:05.834077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789150038309952:2049] [node 3] ICS09 handshake done sender: [2:7577789150038309918:2112] self: [2:7577789150038309916:0] peer: [3:7577789152967593364:0] socket: 345 qp: -1
2025-11-28T14:52:05.834108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789150038309952:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:05.834146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:05.835123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152967593367:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21757
2025-11-28T14:52:05.835611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789152110749226:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.835687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:05.835735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789152110749226:2049] [node 3] ICP22 created new session: [1:7577789152110749325:2049]
2025-11-28T14:52:05.835791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789152110749325:2049] [node 3] ICS09 handshake done sender: [1:7577789152110749294:2111] self: [1:7577789152110749293:0] peer: [3:7577789152967593365:0] socket: 362 qp: -1
2025-11-28T14:52:05.835834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789152110749325:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:05.835867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.841265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789152967593280:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25808) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789152967593357:2116] held: no
2025-11-28T14:52:05.841745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152967593357:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:05.841999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789152967593369:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21757
2025-11-28T14:52:05.843729Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789152967593279:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6991) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789152967593360:2118] held: no
2025-11-28T14:52:05.843991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789152967593280:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.844062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.844122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789152967593280:2049] [node 2] ICP22 created new session: [3:7577789152967593370:2049]
2025-11-28T14:52:05.844174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789152967593370:2049] [node 2] ICS09 handshake done sender: [3:7577789152967593357:2116] self: [3:7577789152967593364:0] peer: [2:7577789150038309916:0] socket: 357 qp: -1
2025-11-28T14:52:05.844211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789152967593370:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:05.844257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:05.845262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152967593360:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:05.850199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789152967593279:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.850267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.850322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789152967593279:2048] [node 1] ICP22 created new session: [3:7577789152967593374:2048]
2025-11-28T14:52:05.850371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789152967593374:2048] [node 1] ICS09 handshake done sender: [3:7577789152967593360:2118] self: [3:7577789152967593365:0] peer: [1:7577789152110749293:0] socket: 364 qp: -1
2025-11-28T14:52:05.850411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789152967593374:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.850448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.853092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789151609188187:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25330
2025-11-28T14:52:05.861776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789152851978477:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:05.888812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789152851978407:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.888928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789152851978408:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.889526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789151609188170:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.889707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789151609188154:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:05.910676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789152851978474:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.912026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749308:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.912097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789152851978477:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.920778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789152110749296:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.930841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789150038309925:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.930924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789150038309925:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.912336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789152851978489:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15184
2025-11-28T14:52:05.920840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789152110749296:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:05.960329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789150038309925:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:05.969390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789150038309920:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.912492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789152851978490:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15184
2025-11-28T14:52:05.969429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:05.912594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789152851978491:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15184
2025-11-28T14:52:05.969473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789150038309920:2107] [node 4] ICP22 created new session: [2:7577789150038309955:2107]
2025-11-28T14:52:05.969517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789150038309955:2107] [node 4] ICS09 handshake done sender: [2:7577789150038309925:2114] self: [2:7577789150038309923:0] peer: [4:7577789152851978488:0] socket: 350 qp: -1
2025-11-28T14:52:05.969553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789150038309955:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:05.969592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:05.912921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789152851978477:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.971206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789152851978489:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.971166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789152110749296:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:05.971626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789152851978470:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.971676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:05.971862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789152110749288:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:05.971927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:05.971711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789151609188189:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25330
2025-11-28T14:52:05.971736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789152851978470:2108] [node 5] ICP22 created new session: [4:7577789152851978495:2108]
2025-11-28T14:52:05.971784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789152851978495:2108] [node 5] ICS09 handshake done sender: [4:7577789152851978477:2112] self: [4:7577789152851978476:0] peer: [5:7577789151609188188:0] socket: 378 qp: -1
2025-11-28T14:52:05.971815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789152851978495:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:05.972171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:05.972266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789152110749288:2107] [node 4] ICP22 created new session: [1:7577789152110749329:2107]
2025-11-28T14:52:05.974739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789152110749329:2107] [node 4] ICS09 handshake done sender: [1:7577789152110749296:2112] self: [1:7577789152110749295:0] peer: [4:7577789152851978487:0] socket: 363 qp: -1
2025-11-28T14:52:05.974793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789152110749329:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:05.974825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:05.974615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789151609188170:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:05.976357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789151609188154:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.976421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:05.976473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789151609188154:2108] [node 4] ICP22 created new session: [5:7577789151609188190:2108]
2025-11-28T14:52:05.976511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789151609188190:2108] [node 4] ICS09 handshake done sender: [5:7577789151609188170:2116] self: [5:7577789151609188188:0] peer: [4:7577789152851978476:0] socket: 379 qp: -1
2025-11-28T14:52:05.976543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789151609188190:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:05.976574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.979750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789152851978470:2108] [node 5] ICP18 handshake (actor [4:7577789152851978489:2117]) is too old
2025-11-28T14:52:05.979895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789152851978493:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.980715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789152851978494:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15184
2025-11-28T14:52:05.981123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789152851978407:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6991) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789152851978483:2116] held: no
2025-11-28T14:52:05.981827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789152851978483:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:05.984995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789152851978480:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:05.985469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789152851978407:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.985529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:05.985594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789152851978407:2048] [node 1] ICP22 created new session: [4:7577789152851978498:2048]
2025-11-28T14:52:05.985649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789152851978498:2048] [node 1] ICS09 handshake done sender: [4:7577789152851978483:2116] self: [4:7577789152851978487:0] peer: [1:7577789152110749295:0] socket: 367 qp: -1
2025-11-28T14:52:05.985706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789152851978498:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:05.985756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:05.986399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789152851978408:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:05.986447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:05.986497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789152851978408:2049] [node 2] ICP22 created new session: [4:7577789152851978501:2049]
2025-11-28T14:52:05.986540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789152851978501:2049] [node 2] ICS09 handshake done sender: [4:7577789152851978480:2114] self: [4:7577789152851978488:0] peer: [2:7577789150038309923:0] socket: 361 qp: -1
2025-11-28T14:52:05.986577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789152851978501:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:05.986615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:05.988162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789149430756495:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
2025-11-28T14:52:05.986039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789150038309957:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53348
2025-11-28T14:52:05.991721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789152967593352:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:05.993243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789149430756477:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:05.993425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789149430756401:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:06.044961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789152967593359:2117] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:06.073777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152967593352:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.073875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789151609188187:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.074065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789151609188093:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.074566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789153725723793:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
2025-11-28T14:52:06.074813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152967593352:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:06.075287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789149430756477:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:06.075631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789149430756401:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.075757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:06.075814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789149430756401:2050] [node 3] ICP22 created new session: [6:7577789153725723794:2050]
2025-11-28T14:52:06.075857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152967593359:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.075866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789153725723794:2050] [node 3] ICS09 handshake done sender: [6:7577789149430756477:2115] self: [6:7577789149430756496:0] peer: [3:7577789152967593350:0] socket: 372 qp: -1
2025-11-28T14:52:06.075909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789153725723794:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:06.075942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789152967593348:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.075950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:06.075998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:06.076052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789152967593348:2110] [node 6] ICP22 created new session: [3:7577789157262560673:2110]
2025-11-28T14:52:06.076102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789157262560673:2110] [node 6] ICS09 handshake done sender: [3:7577789152967593352:2113] self: [3:7577789152967593350:0] peer: [6:7577789149430756496:0] socket: 371 qp: -1
2025-11-28T14:52:06.076139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789157262560673:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:06.076181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:06.079165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789152851978475:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:06.093367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152967593359:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.093622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789155904155489:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25330
2025-11-28T14:52:06.093844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789152967593349:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.093912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789151609188187:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:06.093914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:06.093978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789152967593349:2108] [node 5] ICP22 created new session: [3:7577789157262560676:2108]
2025-11-28T14:52:06.094104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789157262560676:2108] [node 5] ICS09 handshake done sender: [3:7577789152967593359:2117] self: [3:7577789152967593358:0] peer: [5:7577789155904155488:0] socket: 355 qp: -1
2025-11-28T14:52:06.094191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789151609188093:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.094225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789157262560676:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:06.094228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:06.094279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:06.094284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789151609188093:2050] [node 3] ICP22 created new session: [5:7577789155904155490:2050]
2025-11-28T14:52:06.094321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789155904155490:2050] [node 3] ICS09 handshake done sender: [5:7577789151609188187:2121] self: [5:7577789155904155488:0] peer: [3:7577789152967593358:0] socket: 358 qp: -1
2025-11-28T14:52:06.094345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789155904155490:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:06.094365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:06.095298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152967593363:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.099166Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789149430756473:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:06.119001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789152851978490:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.124779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789152967593353:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:06.137247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789152851978491:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.137444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789152851978409:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:06.137816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789152967593353:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.138361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789152967593353:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.138635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789152967593347:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.138555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789157146945801:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15184
2025-11-28T14:52:06.138762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:06.138849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789152967593347:2107] [node 4] ICP22 created new session: [3:7577789157262560680:2107]
2025-11-28T14:52:06.139047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789157262560680:2107] [node 4] ICS09 handshake done sender: [3:7577789152967593353:2114] self: [3:7577789152967593351:0] peer: [4:7577789157146945800:0] socket: 352 qp: -1
2025-11-28T14:52:06.139083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789157262560680:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:06.139226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:06.140038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789152851978491:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:06.140456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789152851978409:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64852) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789152851978491:2119] held: no
2025-11-28T14:52:06.140560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789152851978409:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.140602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:06.140658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789152851978409:2050] [node 3] ICP22 created new session: [4:7577789157146945802:2050]
2025-11-28T14:52:06.140705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789157146945802:2050] [node 3] ICS09 handshake done sender: [4:7577789152851978491:2119] self: [4:7577789157146945800:0] peer: [3:7577789152967593351:0] socket: 403 qp: -1
2025-11-28T14:52:06.140742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789157146945802:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:06.140776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:06.143511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789152851978482:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:06.144891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789149430756495:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.145081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789149430756463:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:06.145330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789152851978482:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.145808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789152851978482:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.145988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789153725723797:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18695
2025-11-28T14:52:06.146076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789152851978471:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.146141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:06.146443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789149430756495:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:06.146377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789152851978471:2109] [node 6] ICP22 created new session: [4:7577789157146945805:2109]
2025-11-28T14:52:06.146473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789157146945805:2109] [node 6] ICS09 handshake done sender: [4:7577789152851978482:2115] self: [4:7577789152851978481:0] peer: [6:7577789153725723796:0] socket: 380 qp: -1
2025-11-28T14:52:06.146700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789149430756463:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.146743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:06.146582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789157146945805:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:06.146778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789149430756463:2106] [node 4] ICP22 created new session: [6:7577789153725723798:2106]
2025-11-28T14:52:06.146835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789153725723798:2106] [node 4] ICS09 handshake done sender: [6:7577789149430756495:2122] self: [6:7577789153725723796:0] peer: [4:7577789152851978481:0] socket: 396 qp: -1
2025-11-28T14:52:06.146622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:06.146891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789153725723798:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:06.146925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:06.189569Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:06.189855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789150038309940:2048] [node 1] ICS07 socket disconnect 359 reason# EndOfStream
2025-11-28T14:52:06.189904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789150038309940:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.190035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:06.190103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789150038309940:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:06.190617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:06.190855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789152967593374:2048] [node 1] ICS01 socket: 364 reason# EndOfStream
2025-11-28T14:52:06.190963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789152967593279:2048] [node 1] ICP30 unregister session Session# [3:7577789152967593374:2048] VirtualId# [3:7577789152967593365:0]
2025-11-28T14:52:06.191010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789152967593374:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.191114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:06.191170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:06.192466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789150038309851:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6991) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:06.192504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789150038309940:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:06.193283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:06.193580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789152851978498:2048] [node 1] ICS01 socket: 367 reason# EndOfStream
2025-11-28T14:52:06.193624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789152851978407:2048] [node 1] ICP30 unregister session Session# [4:7577789152851978498:2048] VirtualId# [4:7577789152851978487:0]
2025-11-28T14:52:06.193657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789152851978498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.193738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:06.193787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:06.193901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789150038309851:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6991) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:06.194090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789150038309940:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:06.194120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789150038309851:2048] [node 1] ICP30 unregister session Session# [2:7577789150038309940:2048] VirtualId# [2:7577789150038309937:0]
2025-11-28T14:52:06.194197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:06.194219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789150038309851:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6991) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:06.194240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6991) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:06.194469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:06.194715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789151609188177:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-28T14:52:06.194763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789151609188091:2048] [node 1] ICP30 unregister session Session# [5:7577789151609188177:2048] VirtualId# [5:7577789151609188174:0]
2025-11-28T14:52:06.194795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789151609188177:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.194871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:06.194961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:06.195464Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:06.195616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789149430756488:2048] [node 1] ICS01 socket: 392 reason# EndOfStream
2025-11-28T14:52:06.195646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789149430756399:2048] [node 1] ICP30 unregister session Session# [6:7577789149430756488:2048] VirtualId# [6:7577789149430756481:0]
2025-11-28T14:52:06.195666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789149430756488:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.195742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:06.195780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:06.217369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:06.217708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789152967593370:2049] [node 2] ICS01 socket: 357 reason# EndOfStream
2025-11-28T14:52:06.217765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789152967593280:2049] [node 2] ICP30 unregister session Session# [3:7577789152967593370:2049] VirtualId# [3:7577789152967593364:0]
2025-11-28T14:52:06.217811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789152967593370:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.217969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:06.218037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:06.219905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:06.220130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789152851978501:2049] [node 2] ICS01 socket: 361 reason# EndOfStream
2025-11-28T14:52:06.220189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789152851978408:2049] [node 2] ICP30 unregister session Session# [4:7577789152851978501:2049] VirtualId# [4:7577789152851978488:0]
2025-11-28T14:52:06.220230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789152851978501:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.220345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:06.220428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:06.220719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:06.220866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789151609188179:2049] [node 2] ICS01 socket: 387 reason# EndOfStream
2025-11-28T14:52:06.220897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789151609188092:2049] [node 2] ICP30 unregister session Session# [5:7577789151609188179:2049] VirtualId# [5:7577789151609188173:0]
2025-11-28T14:52:06.220916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789151609188179:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.220974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:06.221006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:06.221492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:06.221706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789149430756485:2049] [node 2] ICS01 socket: 349 reason# EndOfStream
2025-11-28T14:52:06.221773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789149430756400:2049] [node 2] ICP30 unregister session Session# [6:7577789149430756485:2049] VirtualId# [6:7577789149430756480:0]
2025-11-28T14:52:06.221819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789149430756485:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.221912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:06.221964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:06.230431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:06.230703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789157146945802:2050] [node 3] ICS01 socket: 403 reason# EndOfStream
2025-11-28T14:52:06.230760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789152851978409:2050] [node 3] ICP30 unregister session Session# [4:7577789157146945802:2050] VirtualId# [4:7577789157146945800:0]
2025-11-28T14:52:06.230798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789157146945802:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.230909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:06.230973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:06.231096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:06.231224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789155904155490:2050] [node 3] ICS01 socket: 358 reason# EndOfStream
2025-11-28T14:52:06.231254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789151609188093:2050] [node 3] ICP30 unregister session Session# [5:7577789155904155490:2050] VirtualId# [5:7577789155904155488:0]
2025-11-28T14:52:06.231275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789155904155490:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.231340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:06.231381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:06.231886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:06.232119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789153725723794:2050] [node 3] ICS01 socket: 372 reason# EndOfStream
2025-11-28T14:52:06.232176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789149430756401:2050] [node 3] ICP30 unregister session Session# [6:7577789153725723794:2050] VirtualId# [6:7577789149430756496:0]
2025-11-28T14:52:06.232205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789153725723794:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.232289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:06.232351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:06.242749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:06.243003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789151609188190:2108] [node 4] ICS01 socket: 379 reason# EndOfStream
2025-11-28T14:52:06.243050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789151609188154:2108] [node 4] ICP30 unregister session Session# [5:7577789151609188190:2108] VirtualId# [5:7577789151609188188:0]
2025-11-28T14:52:06.243101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789151609188190:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.243207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:06.243274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:06.243887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:06.244120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789153725723798:2106] [node 4] ICS01 socket: 396 reason# EndOfStream
2025-11-28T14:52:06.244173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789149430756463:2106] [node 4] ICP30 unregister session Session# [6:7577789153725723798:2106] VirtualId# [6:7577789153725723796:0]
2025-11-28T14:52:06.244226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789153725723798:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.244341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:06.244436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:06.291918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:06.292080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789149430756493:2104] [node 5] ICS07 socket disconnect 374 reason# EndOfStream
2025-11-28T14:52:06.292105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789149430756493:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:06.292188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:06.292226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789149430756493:2104] [node 5] ICS15 start handshake
2025-11-28T14:52:06.292932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789149430756460:2104] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62050) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:06.293076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789149430756493:2104] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:06.293146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789149430756460:2104] [node 5] ICP30 unregister session Session# [6:7577789149430756493:2104] VirtualId# [6:7577789149430756491:0]
2025-11-28T14:52:06.293192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:06.293215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789149430756460:2104] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62050) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:06.293225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62050) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:06.434337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789153623656491:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:06.434406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789153623656492:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:06.462731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789155757622906:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:06.462774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789155757622907:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:06.493212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789154742977436:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:06.493259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789154742977437:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:06.521956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789153589611494:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:06.521998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789153589611495:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:06.522030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789153589611496:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:06.546896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789154828234967:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:06.546945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789154828234968:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:06.546982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789154828234969:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:06.571573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789156959388367:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:06.571617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789156959388368:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:06.571657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789156959388369:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:06.650422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789156959388429:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:06.707869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789154742977517:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:06.707946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789155757622969:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:06.713277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789155757622970:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:06.714080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789156959388439:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:06.714671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789154742977518:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:06.714867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789154742977519:2114] [node 6] ICP01 ready to work
2025-11-28T14:52:06.715097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789155757622971:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:06.715197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789153623656555:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:06.715168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789153589611574:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:06.715359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789153589611575:2115] [node 6] ICP01 ready to work
2025-11-28T14:52:06.714657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789154828235030:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:06.715616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789154828235031:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:06.716763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789153623656558:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:06.716835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789153623656492:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(72714545ns) {Configure:104(72706487ns) {ProcessConfigured:135(808589ns) {StartInitialHandshake:157(805883ns) {DropHandshakes:470(3073ns) {DropIncomingHandshake:442(712ns) DropOutgoingHandshake:457(706ns)} GenerateSessionVirtualId:988(792ns) PrepareNewSessionHandshake:477(4492ns) {SwitchToState:179(2995ns)}}}}}
2025-11-28T14:52:06.718058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789153623656491:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(72300933ns) {Configure:104(72294760ns) {ProcessConfigured:135(381629ns) {StartInitialHandshake:157(380193ns) {DropHandshakes:470(1753ns) {DropIncomingHandshake:442(451ns) DropOutgoingHandshake:457(283ns)} GenerateSessionVirtualId:988(598ns) PrepareNewSessionHandshake:477(1344ns) {SwitchToState:179(632ns)}}}}}
2025-11-28T14:52:06.718205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789153623656561:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:06.720671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789153589611495:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47599095ns) {Configure:104(47589491ns) {ProcessConfigured:135(603013ns) {StartInitialHandshake:157(600871ns) {DropHandshakes:470(2299ns) {DropIncomingHandshake:442(430ns) DropOutgoingHandshake:457(457ns)} GenerateSessionVirtualId:988(791ns) PrepareNewSessionHandshake:477(1382ns) {SwitchToState:179(486ns)}}}}}
2025-11-28T14:52:06.726294Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789156959388367:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21618399ns) {Configure:104(21608245ns) {ProcessConfigured:135(929998ns) {StartInitialHandshake:157(927885ns) {DropHandshakes:470(1096ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(476ns) PrepareNewSessionHandshake:477(1144ns) {SwitchToState:179(152ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.728405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789155757622983:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63838
2025-11-28T14:52:06.730241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153623656571:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39947
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.735791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789154828235040:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13884
2025-11-28T14:52:06.735763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789153589611584:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30513
2025-11-28T14:52:06.738244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789153589611584:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.738159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789155757622983:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.738137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789154828235040:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.738493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656571:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.738928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789155757622906:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.739200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789154742977526:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63223
2025-11-28T14:52:06.738940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789154828234968:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.739516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789155757622987:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63838
2025-11-28T14:52:06.739518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153623656572:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39947
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.739481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789156959388448:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
2025-11-28T14:52:06.739992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789155757622976:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.741572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656560:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.741000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789156959388448:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.743028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789153589611494:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:06.740037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789155757622976:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:06.743080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789154742977532:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63223
2025-11-28T14:52:06.743394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789156959388368:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:52:06.741582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153623656573:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39947
::ffff:127.1.0.0
2025-11-28T14:52:06.740147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789155757622986:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63838
2025-11-28T14:52:06.743291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789154742977531:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63223
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.741602Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789153623656560:2112] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.743455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789155757622976:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:06.743911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789154742977532:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.746636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789154828235046:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13884
2025-11-28T14:52:06.743671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789155757622986:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.744568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656563:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.746823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789155757622989:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63838
2025-11-28T14:52:06.744610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789154742977526:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.744596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789153623656563:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:06.746844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789155757622978:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:06.746880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789155757622978:2112] [node 6] ICRDMA Non success qp response from remote side
::ffff:127.1.0.02025-11-28T14:52:06.747868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789153589611591:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30513
2025-11-28T14:52:06.746700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789154742977531:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.744734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656573:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.747736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789155757622970:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.747083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153623656574:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39947
2025-11-28T14:52:06.747199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789154742977436:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:06.748578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789154828235048:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13884
2025-11-28T14:52:06.747777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:06.747128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153623656560:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:06.748019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789154742977536:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63223
2025-11-28T14:52:06.748112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656572:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.748597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789153623656576:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39947
2025-11-28T14:52:06.748721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153623656563:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:06.748791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789153623656491:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.748806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656557:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.748829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789153623656557:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:06.748855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:06.755002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789153623656491:2048] [node 2] ICP22 created new session: [1:7577789153623656580:2048]
2025-11-28T14:52:06.747991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789155757622970:2108] [node 5] ICP22 created new session: [2:7577789155757622993:2108]
2025-11-28T14:52:06.774516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789153623656580:2048] [node 2] ICS09 handshake done sender: [1:7577789153623656560:2112] self: [1:7577789153623656559:0] peer: [2:7577789155757622988:0] socket: 349 qp: -1
2025-11-28T14:52:06.748112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789155757622993:2108] [node 5] ICS09 handshake done sender: [2:7577789155757622976:2111] self: [2:7577789155757622975:0] peer: [5:7577789154828235047:0] socket: 354 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.780019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789154828235040:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:06.748490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789155757622993:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:06.774599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789153623656580:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.787762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789153589611592:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30513
2025-11-28T14:52:06.784705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789154828234968:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.748503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789155757622987:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.784765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:06.777586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153623656557:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:06.748621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:06.784822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789154828234968:2049] [node 2] ICP22 created new session: [5:7577789154828235050:2049]
2025-11-28T14:52:06.779829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:06.788095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789154742977540:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63223
2025-11-28T14:52:06.784873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789154828235050:2049] [node 2] ICS09 handshake done sender: [5:7577789154828235040:2114] self: [5:7577789154828235047:0] peer: [2:7577789155757622975:0] socket: 360 qp: -1
2025-11-28T14:52:06.765836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789155757622978:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:06.780253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789153623656555:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.787786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789155757622994:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63838
2025-11-28T14:52:06.780335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:06.784909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789154828235050:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:06.780405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789153623656555:2107] [node 4] ICP22 created new session: [1:7577789153623656586:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.784941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:06.780728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789153623656586:2107] [node 4] ICS09 handshake done sender: [1:7577789153623656563:2113] self: [1:7577789153623656562:0] peer: [4:7577789153589611590:0] socket: 350 qp: -1
2025-11-28T14:52:06.780855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789153623656586:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:06.787944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789155757622906:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789155757622983:2115] held: no
2025-11-28T14:52:06.789365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789153589611584:2131] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:06.789843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789154828235049:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13884
2025-11-28T14:52:06.788270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789155757622970:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(39553600ns) {HandleHandshakeStatus:329(39550601ns) {DropIncomingHandshake:442(217ns) DropOutgoingHandshake:457(601ns) SwitchToState:179(673ns) ProcessPendingSessionEvents:495(24647550ns) {ForwardSessionEventToSession:612(15217467ns) ForwardSessionEventToSession:612(2250743ns) ForwardSessionEventToSession:612(21247ns) ForwardSessionEventToSession:612(3770ns) ForwardSessionEventToSession:612(767958ns) ForwardSessionEventToSession:612(4888ns) ForwardSessionEventToSession:612(3566ns) ForwardSessionEventToSession:612(3540922ns) ForwardSessionEventToSession:612(3319ns) ForwardSessionEventToSession:612(2794370ns)}}}
2025-11-28T14:52:06.781536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:06.788227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789153623656492:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.790639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789153589611494:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.788412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789155757622971:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.788276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:06.791742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:06.788425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789153623656492:2049] [node 3] ICP22 created new session: [1:7577789153623656588:2049]
2025-11-28T14:52:06.789088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789153623656588:2049] [node 3] ICS09 handshake done sender: [1:7577789153623656557:2111] self: [1:7577789153623656556:0] peer: [3:7577789154742977537:0] socket: 348 qp: -1
2025-11-28T14:52:06.788457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:06.789128Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789153623656491:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(39460380ns) {HandleHandshakeStatus:329(39455168ns) {DropIncomingHandshake:442(15891ns) DropOutgoingHandshake:457(557ns) SwitchToState:179(474ns) ProcessPendingSessionEvents:495(1988050ns) {ForwardSessionEventToSession:612(1469743ns) ForwardSessionEventToSession:612(469432ns) ForwardSessionEventToSession:612(4681ns) ForwardSessionEventToSession:612(5837ns) ForwardSessionEventToSession:612(2954ns) ForwardSessionEventToSession:612(2067ns) ForwardSessionEventToSession:612(5031ns) ForwardSessionEventToSession:612(2272ns) ForwardSessionEventToSession:612(2024ns) ForwardSessionEventToSession:612(11238ns)}}}
2025-11-28T14:52:06.788513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789155757622971:2109] [node 6] ICP22 created new session: [2:7577789155757622997:2109]
2025-11-28T14:52:06.791792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789153589611494:2048] [node 1] ICP22 created new session: [4:7577789153589611594:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.789366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789153623656588:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:06.789366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789155757622997:2109] [node 6] ICS09 handshake done sender: [2:7577789155757622978:2112] self: [2:7577789155757622977:0] peer: [6:7577789156959388454:0] socket: 355 qp: -1
2025-11-28T14:52:06.792956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789153589611594:2048] [node 1] ICS09 handshake done sender: [4:7577789153589611584:2131] self: [4:7577789153589611590:0] peer: [1:7577789153623656562:0] socket: 358 qp: -1
2025-11-28T14:52:06.793094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789153589611594:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:06.789400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:06.789847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789155757622997:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:06.793135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:06.790001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:06.794387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789153589611593:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30513
2025-11-28T14:52:06.793690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789155757622983:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:06.794540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789153589611494:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:06.794370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789155757622970:2108] [node 5] ICP18 handshake (actor [2:7577789155757622987:2117]) is too old
2025-11-28T14:52:06.794412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789155757622906:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.794718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:06.794922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789155757622906:2048] [node 1] ICP22 created new session: [2:7577789155757623000:2048]
2025-11-28T14:52:06.797495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789155757623000:2048] [node 1] ICS09 handshake done sender: [2:7577789155757622983:2115] self: [2:7577789155757622988:0] peer: [1:7577789153623656559:0] socket: 351 qp: -1
2025-11-28T14:52:06.798019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789155757623000:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:06.801026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789153589611597:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30513
2025-11-28T14:52:06.801081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789154828235053:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13884
2025-11-28T14:52:06.800230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:06.801539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789154828235053:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.801604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789153589611597:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.801686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789154828235030:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:06.801748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789153589611495:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:06.801889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789153589611583:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.801917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789153589611583:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:06.801942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789155757622981:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.801972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789155757622981:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:06.802895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789155757622981:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.803108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789155757622969:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.803154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789153589611600:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30513
2025-11-28T14:52:06.803199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:06.803248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789153589611583:2130] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:06.803355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789155757622969:2107] [node 4] ICP22 created new session: [2:7577789155757623003:2107]
2025-11-28T14:52:06.803424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789155757623003:2107] [node 4] ICS09 handshake done sender: [2:7577789155757622981:2114] self: [2:7577789155757622979:0] peer: [4:7577789153589611598:0] socket: 367 qp: -1
2025-11-28T14:52:06.803424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789153589611597:2136] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.803461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789155757623003:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:06.803708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:06.803981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789153589611495:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.804021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789154828235055:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13884
2025-11-28T14:52:06.804000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789153589611574:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.804049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:06.804050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:06.804090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789153589611495:2049] [node 2] ICP22 created new session: [4:7577789153589611601:2049]
2025-11-28T14:52:06.804176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789153589611574:2108] [node 5] ICP22 created new session: [4:7577789153589611602:2108]
2025-11-28T14:52:06.804188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789153589611601:2049] [node 2] ICS09 handshake done sender: [4:7577789153589611597:2136] self: [4:7577789153589611598:0] peer: [2:7577789155757622979:0] socket: 404 qp: -1
2025-11-28T14:52:06.804327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789154828235053:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:06.804222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789153589611601:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:06.804631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789154828235030:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.804250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:06.804878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:06.805243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789154828235030:2108] [node 4] ICP22 created new session: [5:7577789154828235056:2108]
2025-11-28T14:52:06.804259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789153589611602:2108] [node 5] ICS09 handshake done sender: [4:7577789153589611583:2130] self: [4:7577789153589611582:0] peer: [5:7577789154828235054:0] socket: 373 qp: -1
2025-11-28T14:52:06.804321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789153589611602:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:06.805384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789154828235056:2108] [node 4] ICS09 handshake done sender: [5:7577789154828235053:2119] self: [5:7577789154828235054:0] peer: [4:7577789153589611582:0] socket: 405 qp: -1
2025-11-28T14:52:06.804358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:06.805419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789154828235056:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:06.805413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789153589611495:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:06.805464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:06.809309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789154828235046:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.808651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789156959388443:2118] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.820955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789156959388455:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.822164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789156959388456:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.822252Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789156959388444:2119] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:06.837675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789156959388457:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
2025-11-28T14:52:06.848591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789156959388448:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:06.850934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789156959388456:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.850998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789156959388368:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.851001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789156959388457:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.851053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:06.851002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656574:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.851132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789156959388368:2049] [node 2] ICP22 created new session: [6:7577789156959388458:2049]
2025-11-28T14:52:06.851194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789156959388458:2049] [node 2] ICS09 handshake done sender: [6:7577789156959388448:2122] self: [6:7577789156959388454:0] peer: [2:7577789155757622977:0] socket: 361 qp: -1
2025-11-28T14:52:06.851279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789156959388458:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:06.851314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:06.852508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789153589611589:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.852530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656569:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.852537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789153589611589:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:06.851708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789156959388367:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:06.852566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789153623656569:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:06.851788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789156959388439:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.853737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789153589611606:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30513
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.854132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789156959388463:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
2025-11-28T14:52:06.854353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153623656569:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:06.855338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789156959388456:2125] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:06.854645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789153623656561:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.854727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:06.855482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789153589611589:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:06.855253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789153623656561:2109] [node 6] ICP22 created new session: [1:7577789153623656592:2109]
2025-11-28T14:52:06.855681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789153589611575:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.855446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789153623656592:2109] [node 6] ICS09 handshake done sender: [1:7577789153623656569:2115] self: [1:7577789153623656565:0] peer: [6:7577789156959388460:0] socket: 401 qp: -1
2025-11-28T14:52:06.855560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789153623656592:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:06.855654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:06.855730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:06.856120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789153589611575:2115] [node 6] ICP22 created new session: [4:7577789153589611608:2115]
2025-11-28T14:52:06.856474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789153589611608:2115] [node 6] ICS09 handshake done sender: [4:7577789153589611589:2132] self: [4:7577789153589611585:0] peer: [6:7577789156959388461:0] socket: 363 qp: -1
2025-11-28T14:52:06.856504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789153589611608:2115] [node 6] ICS10 traffic start
2025-11-28T14:52:06.856560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:06.857387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789156959388367:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.857448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:06.857501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789156959388367:2048] [node 1] ICP22 created new session: [6:7577789156959388465:2048]
2025-11-28T14:52:06.857543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789156959388465:2048] [node 1] ICS09 handshake done sender: [6:7577789156959388456:2125] self: [6:7577789156959388460:0] peer: [1:7577789153623656565:0] socket: 369 qp: -1
2025-11-28T14:52:06.857579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789156959388465:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:06.857619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:06.860065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789156959388445:2120] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.882755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789156959388367:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:06.882846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789156959388464:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
2025-11-28T14:52:06.883177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789156959388457:2126] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:06.883645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789156959388439:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.883710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:06.883765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789156959388439:2109] [node 4] ICP22 created new session: [6:7577789156959388467:2109]
2025-11-28T14:52:06.883819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789156959388467:2109] [node 4] ICS09 handshake done sender: [6:7577789156959388457:2126] self: [6:7577789156959388461:0] peer: [4:7577789153589611585:0] socket: 409 qp: -1
2025-11-28T14:52:06.883880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789156959388467:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:06.883929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:06.887772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789154742977523:2129] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.897039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656576:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.897281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789153623656492:2049] [node 3] ICP18 handshake (actor [1:7577789153623656576:2120]) is too old
2025-11-28T14:52:06.897109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789154742977536:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.897226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789154742977541:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63223
2025-11-28T14:52:06.899032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789154742977526:2131] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:06.899320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789154742977436:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789154742977526:2131] held: no
2025-11-28T14:52:06.899426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789154742977436:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.899481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:06.899535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789154742977436:2048] [node 1] ICP22 created new session: [3:7577789154742977542:2048]
2025-11-28T14:52:06.899580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789154742977542:2048] [node 1] ICS09 handshake done sender: [3:7577789154742977526:2131] self: [3:7577789154742977537:0] peer: [1:7577789153623656556:0] socket: 359 qp: -1
2025-11-28T14:52:06.899622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789154742977542:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:06.899659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:06.900221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789154742977540:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.900840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789154742977437:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.901800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789155757622980:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.901866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789155757622980:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:06.902068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789156959388469:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
2025-11-28T14:52:06.902946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789155757622980:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:06.903213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789155757622907:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.903272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:06.903322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789155757622907:2049] [node 3] ICP22 created new session: [2:7577789155757623006:2049]
2025-11-28T14:52:06.903358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789155757623006:2049] [node 3] ICS09 handshake done sender: [2:7577789155757622980:2113] self: [2:7577789155757622972:0] peer: [3:7577789154742977546:0] socket: 398 qp: -1
2025-11-28T14:52:06.903401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789155757623006:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:06.903450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:06.918403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789154742977521:2128] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.930034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789154742977547:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63223
2025-11-28T14:52:06.930435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789153589611591:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.930635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789153589611496:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:06.930560Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789154742977540:2137] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:06.930846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789154742977437:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.930893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:06.930943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789154742977437:2049] [node 2] ICP22 created new session: [3:7577789154742977548:2049]
2025-11-28T14:52:06.930990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789154742977548:2049] [node 2] ICS09 handshake done sender: [3:7577789154742977540:2137] self: [3:7577789154742977546:0] peer: [2:7577789155757622972:0] socket: 353 qp: -1
2025-11-28T14:52:06.931026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789154742977548:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:06.931069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:06.931759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789154742977521:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.932244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789154742977521:2128] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.932991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789153589611611:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30513
2025-11-28T14:52:06.933021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789154742977517:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:06.933078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:06.933122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789154742977517:2107] [node 4] ICP22 created new session: [3:7577789154742977551:2107]
2025-11-28T14:52:06.933170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789154742977551:2107] [node 4] ICS09 handshake done sender: [3:7577789154742977521:2128] self: [3:7577789154742977520:0] peer: [4:7577789153589611610:0] socket: 383 qp: -1
2025-11-28T14:52:06.933204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789154742977551:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:06.933241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:06.933478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789153589611591:2133] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:06.933607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789153589611496:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:06.933670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:06.933727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789153589611496:2050] [node 3] ICP22 created new session: [4:7577789153589611612:2050]
2025-11-28T14:52:06.933767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789153589611612:2050] [node 3] ICS09 handshake done sender: [4:7577789153589611591:2133] self: [4:7577789153589611610:0] peer: [3:7577789154742977520:0] socket: 386 qp: -1
2025-11-28T14:52:06.933886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789153589611612:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:06.933926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:06.937338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789154828235059:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13884
2025-11-28T14:52:06.940911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789154742977525:2130] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:06.948726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789155757622989:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.948991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789155757622907:2049] [node 3] ICP18 handshake (actor [2:7577789155757622989:2118]) is too old
2025-11-28T14:52:06.991352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789154828235039:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:06.992521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789153589611593:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:06.992688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789153589611574:2108] [node 5] ICP18 handshake (actor [4:7577789153589611593:2135]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.026589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789161254355766:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
2025-11-28T14:52:07.027314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789161254355766:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.027524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789156959388429:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.027868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789154828235045:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.027897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789154828235045:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.028738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789154828235045:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:07.028848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789161254355768:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
2025-11-28T14:52:07.028998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789154828235031:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.029064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:07.030082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789161254355766:2130] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:07.030996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789153623656567:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.031403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789156959388429:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.030099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789154828235031:2109] [node 6] ICP22 created new session: [5:7577789159123202358:2109]
2025-11-28T14:52:07.031683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789159123202358:2109] [node 6] ICS09 handshake done sender: [5:7577789154828235045:2115] self: [5:7577789154828235042:0] peer: [6:7577789161254355767:0] socket: 385 qp: -1
2025-11-28T14:52:07.032100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789159123202358:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:07.031487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:07.032156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:07.031537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789156959388429:2106] [node 5] ICP22 created new session: [6:7577789161254355769:2106]
2025-11-28T14:52:07.031593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789161254355769:2106] [node 5] ICS09 handshake done sender: [6:7577789161254355766:2130] self: [6:7577789161254355767:0] peer: [5:7577789154828235042:0] socket: 352 qp: -1
2025-11-28T14:52:07.032116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789161254355769:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:07.041202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:07.041608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789156959388429:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:07.063651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789154828235049:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.063809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789154828235031:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(34488224ns) {HandleHandshakeStatus:329(34485102ns) {DropIncomingHandshake:442(13818ns) DropOutgoingHandshake:457(640ns) SwitchToState:179(577ns) ProcessPendingSessionEvents:495(9609466ns) {ForwardSessionEventToSession:612(9530836ns) ForwardSessionEventToSession:612(17219ns) ForwardSessionEventToSession:612(8950ns) ForwardSessionEventToSession:612(2483ns) ForwardSessionEventToSession:612(8323ns) ForwardSessionEventToSession:612(3074ns) ForwardSessionEventToSession:612(12174ns) ForwardSessionEventToSession:612(2730ns) ForwardSessionEventToSession:612(2556ns) ForwardSessionEventToSession:612(8525ns)}}}
2025-11-28T14:52:07.063866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789154828234967:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.064133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789153623656567:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.064658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789153623656567:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.064852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789159123202361:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13884
2025-11-28T14:52:07.064873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789153623656558:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.065031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:07.065440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789153623656558:2108] [node 5] ICP22 created new session: [1:7577789157918623891:2108]
2025-11-28T14:52:07.065506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789157918623891:2108] [node 5] ICS09 handshake done sender: [1:7577789153623656567:2114] self: [1:7577789153623656564:0] peer: [5:7577789159123202360:0] socket: 400 qp: -1
2025-11-28T14:52:07.065756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789154828235049:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:07.065562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789157918623891:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:07.065916Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789154828234967:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789154828235049:2118] held: no
2025-11-28T14:52:07.065665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:07.066124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789154828234967:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.066667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:07.066741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789154828234967:2048] [node 1] ICP22 created new session: [5:7577789159123202362:2048]
2025-11-28T14:52:07.066847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789159123202362:2048] [node 1] ICS09 handshake done sender: [5:7577789154828235049:2118] self: [5:7577789159123202360:0] peer: [1:7577789153623656564:0] socket: 402 qp: -1
2025-11-28T14:52:07.066886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789159123202362:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:07.066973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:07.070603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789156959388447:2121] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.096359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789153589611606:2138] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.096631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789153589611575:2115] [node 6] ICP18 handshake (actor [4:7577789153589611606:2138]) is too old
2025-11-28T14:52:07.099954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789154742977529:2132] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.127883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789156959388469:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.128354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789154742977529:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.128117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789156959388369:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.128881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789154742977529:2132] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.129213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789161254355772:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61031
2025-11-28T14:52:07.129189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789154742977519:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.129268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:07.129323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789154742977519:2114] [node 6] ICP22 created new session: [3:7577789159037944851:2114]
2025-11-28T14:52:07.129411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789159037944851:2114] [node 6] ICS09 handshake done sender: [3:7577789154742977529:2132] self: [3:7577789154742977527:0] peer: [6:7577789161254355771:0] socket: 378 qp: -1
2025-11-28T14:52:07.129447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789159037944851:2114] [node 6] ICS10 traffic start
2025-11-28T14:52:07.129489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:07.131627Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789156959388369:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789156959388469:2129] held: no
2025-11-28T14:52:07.131737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789156959388469:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:07.131860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789156959388369:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.131923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:07.133140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789156959388369:2050] [node 3] ICP22 created new session: [6:7577789161254355773:2050]
2025-11-28T14:52:07.133205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789161254355773:2050] [node 3] ICS09 handshake done sender: [6:7577789156959388469:2129] self: [6:7577789161254355771:0] peer: [3:7577789154742977527:0] socket: 379 qp: -1
2025-11-28T14:52:07.133258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789161254355773:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:07.133317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:07.135391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789154742977530:2133] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.136105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789154828235059:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.136337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789154828234969:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.136567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789154742977530:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.137075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789154742977530:2133] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:07.137320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789154742977518:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.137456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789159123202365:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13884
2025-11-28T14:52:07.137392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:07.137464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789154742977518:2108] [node 5] ICP22 created new session: [3:7577789159037944854:2108]
2025-11-28T14:52:07.137523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789159037944854:2108] [node 5] ICS09 handshake done sender: [3:7577789154742977530:2133] self: [3:7577789154742977528:0] peer: [5:7577789159123202364:0] socket: 365 qp: -1
2025-11-28T14:52:07.137560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789159037944854:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:07.137601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:07.137822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789154828235059:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:07.138668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789154828234969:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789154828235059:2121] held: no
2025-11-28T14:52:07.138792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789154828234969:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.138829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:07.138879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789154828234969:2050] [node 3] ICP22 created new session: [5:7577789159123202366:2050]
2025-11-28T14:52:07.138929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789159123202366:2050] [node 3] ICS09 handshake done sender: [5:7577789154828235059:2121] self: [5:7577789159123202364:0] peer: [3:7577789154742977528:0] socket: 375 qp: -1
2025-11-28T14:52:07.138969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789159123202366:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:07.139010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:07.162376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:07.162663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789155757623000:2048] [node 1] ICS01 socket: 351 reason# EndOfStream
2025-11-28T14:52:07.162710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789155757622906:2048] [node 1] ICP30 unregister session Session# [2:7577789155757623000:2048] VirtualId# [2:7577789155757622988:0]
2025-11-28T14:52:07.162750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789155757623000:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.162864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:07.162937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:07.163942Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:07.164153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789154742977542:2048] [node 1] ICS01 socket: 359 reason# EndOfStream
2025-11-28T14:52:07.164206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789154742977436:2048] [node 1] ICP30 unregister session Session# [3:7577789154742977542:2048] VirtualId# [3:7577789154742977537:0]
2025-11-28T14:52:07.164239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789154742977542:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.164330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:07.164410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:07.166964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:07.167202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789153589611594:2048] [node 1] ICS07 socket disconnect 358 reason# EndOfStream
2025-11-28T14:52:07.167242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789153589611594:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.167335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:07.167388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789153589611594:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:07.168122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:07.168229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789153589611494:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2972) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:07.168330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789159123202362:2048] [node 1] ICS01 socket: 402 reason# EndOfStream
2025-11-28T14:52:07.168346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789153589611594:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:07.168398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789153589611494:2048] [node 1] ICP30 unregister session Session# [4:7577789153589611594:2048] VirtualId# [4:7577789153589611590:0]
2025-11-28T14:52:07.168476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:07.168513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789153589611494:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2972) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.168513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789154828234967:2048] [node 1] ICP30 unregister session Session# [5:7577789159123202362:2048] VirtualId# [5:7577789159123202360:0]
2025-11-28T14:52:07.168573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789159123202362:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.168527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2972) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.168668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:07.168713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:07.172642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:07.172940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789156959388465:2048] [node 1] ICS07 socket disconnect 369 reason# EndOfStream
2025-11-28T14:52:07.173015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789156959388465:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.173175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:07.173255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789156959388465:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:07.174497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789156959388367:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2972) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:07.174640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789156959388465:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:07.175171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789156959388367:2048] [node 1] ICP30 unregister session Session# [6:7577789156959388465:2048] VirtualId# [6:7577789156959388460:0]
2025-11-28T14:52:07.175252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:07.175307Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789156959388367:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2972) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.175342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2972) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.177907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789153589611494:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:07.185102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789156959388367:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:07.202072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:07.202310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789154742977548:2049] [node 2] ICS01 socket: 353 reason# EndOfStream
2025-11-28T14:52:07.202380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789154742977437:2049] [node 2] ICP30 unregister session Session# [3:7577789154742977548:2049] VirtualId# [3:7577789154742977546:0]
2025-11-28T14:52:07.202438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789154742977548:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.202569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:07.202636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:07.206931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:07.207189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789153589611601:2049] [node 2] ICS01 socket: 404 reason# EndOfStream
2025-11-28T14:52:07.207231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789153589611495:2049] [node 2] ICP30 unregister session Session# [4:7577789153589611601:2049] VirtualId# [4:7577789153589611598:0]
2025-11-28T14:52:07.207264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789153589611601:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.207363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:07.207425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:07.207956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:07.208166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789154828235050:2049] [node 2] ICS01 socket: 360 reason# EndOfStream
2025-11-28T14:52:07.208243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789154828234968:2049] [node 2] ICP30 unregister session Session# [5:7577789154828235050:2049] VirtualId# [5:7577789154828235047:0]
2025-11-28T14:52:07.208287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789154828235050:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.208383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:07.208444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:07.207581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:07.207847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789156959388458:2049] [node 2] ICS07 socket disconnect 361 reason# EndOfStream
2025-11-28T14:52:07.207891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789156959388458:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.207970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:07.208015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789156959388458:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:07.208864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789156959388368:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:07.208907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789156959388458:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:07.208979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789156959388368:2049] [node 2] ICP30 unregister session Session# [6:7577789156959388458:2049] VirtualId# [6:7577789156959388454:0]
2025-11-28T14:52:07.209033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:07.209063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789156959388368:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.209079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.218876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789156959388368:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:07.220494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:07.220534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:07.220766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789153589611612:2050] [node 3] ICS01 socket: 386 reason# EndOfStream
2025-11-28T14:52:07.220818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789153589611496:2050] [node 3] ICP30 unregister session Session# [4:7577789153589611612:2050] VirtualId# [4:7577789153589611610:0]
2025-11-28T14:52:07.220787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789159123202366:2050] [node 3] ICS01 socket: 375 reason# EndOfStream
2025-11-28T14:52:07.220844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789154828234969:2050] [node 3] ICP30 unregister session Session# [5:7577789159123202366:2050] VirtualId# [5:7577789159123202364:0]
2025-11-28T14:52:07.220856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789153589611612:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.220873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789159123202366:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.220987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:07.220986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:07.221057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:07.221060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:07.221495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:07.221722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789161254355773:2050] [node 3] ICS01 socket: 379 reason# EndOfStream
2025-11-28T14:52:07.221767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789156959388369:2050] [node 3] ICP30 unregister session Session# [6:7577789161254355773:2050] VirtualId# [6:7577789161254355771:0]
2025-11-28T14:52:07.221799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789161254355773:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.221873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:07.221925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:07.272608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:07.272878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789154828235056:2108] [node 4] ICS07 socket disconnect 405 reason# EndOfStream
2025-11-28T14:52:07.272919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789154828235056:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.273020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:07.273075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789154828235056:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:07.273849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789154828235030:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12663) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:07.273923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789154828235056:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:07.273995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789154828235030:2108] [node 4] ICP30 unregister session Session# [5:7577789154828235056:2108] VirtualId# [5:7577789154828235054:0]
2025-11-28T14:52:07.274045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:07.274097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789154828235030:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12663) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.274114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12663) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.276582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:07.276967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789156959388467:2109] [node 4] ICS07 socket disconnect 409 reason# EndOfStream
2025-11-28T14:52:07.277037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789156959388467:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.277237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:07.277352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789156959388467:2109] [node 4] ICS15 start handshake
2025-11-28T14:52:07.279147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789156959388439:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12663) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:07.279215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789156959388467:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:07.279252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789156959388439:2109] [node 4] ICP30 unregister session Session# [6:7577789156959388467:2109] VirtualId# [6:7577789156959388461:0]
2025-11-28T14:52:07.279325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:07.279420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789156959388439:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12663) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.279461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12663) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:07.289542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789156959388439:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:52:07.332555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:07.332797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789161254355769:2106] [node 5] ICS01 socket: 352 reason# EndOfStream
2025-11-28T14:52:07.332842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789156959388429:2106] [node 5] ICP30 unregister session Session# [6:7577789161254355769:2106] VirtualId# [6:7577789161254355767:0]
2025-11-28T14:52:07.332879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789161254355769:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:07.333061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:07.333165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:07.390628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789157722297145:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:07.390900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789157722297146:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:07.412609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789159407315860:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:07.412751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789159407315859:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:07.434570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789157907954313:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:07.434616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789157907954314:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:07.458420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789158159581243:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:07.458496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789158159581244:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:07.458529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789158159581245:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:07.488171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789159734542201:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:07.488223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789159734542202:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:07.488341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789159734542203:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:07.511110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789159695449361:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:07.511112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789159695449360:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:07.511138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789159695449362:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:07.594734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789157907954376:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:07.597709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789159695449421:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:07.599371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789157722297210:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:07.599383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789159734542264:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:07.599378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789159407315924:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:07.600079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789157722297211:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:07.600990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789157907954377:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:07.601107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789159734542265:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:07.599432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789159407315923:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:07.601772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789158159581306:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:07.601762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789157907954378:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:07.601905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789159695449424:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:07.600178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789159407315925:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:07.601387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789157722297212:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:07.602190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789158159581307:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:07.606481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789157722297146:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(50335783ns) {Configure:104(50331691ns) {ProcessConfigured:135(70791ns) {StartInitialHandshake:157(68755ns) {DropHandshakes:470(1828ns) {DropIncomingHandshake:442(631ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(492ns) PrepareNewSessionHandshake:477(1387ns) {SwitchToState:179(443ns)}}}}}
2025-11-28T14:52:07.606623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789157722297145:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(50731620ns) {Configure:104(50726814ns) {ProcessConfigured:135(203834ns) {StartInitialHandshake:157(201687ns) {DropHandshakes:470(1326ns) {DropIncomingHandshake:442(398ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:477(1401ns) {SwitchToState:179(520ns)}}}}}
2025-11-28T14:52:07.608417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789159407315859:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50005245ns) {Configure:104(50000601ns) {ProcessConfigured:135(256714ns) {StartInitialHandshake:157(254411ns) {DropHandshakes:470(1278ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(506ns) PrepareNewSessionHandshake:477(1656ns) {SwitchToState:179(786ns)}}}}}
2025-11-28T14:52:07.610567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789158159581243:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29499396ns) {Configure:104(29488227ns) {ProcessConfigured:135(694572ns) {StartInitialHandshake:157(604415ns) {DropHandshakes:470(1926ns) {DropIncomingHandshake:442(641ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(781ns) PrepareNewSessionHandshake:477(1919ns) {SwitchToState:179(1008ns)}}}}}
2025-11-28T14:52:07.611001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789159695449360:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21896774ns) {Configure:104(21889504ns) {ProcessConfigured:135(37719ns) {StartInitialHandshake:157(36530ns) {DropHandshakes:470(952ns) {DropIncomingHandshake:442(278ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(534ns) {SwitchToState:179(155ns)}}}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.616508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789159734542272:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40744
2025-11-28T14:52:07.616711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789158159581316:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52326
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.617703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789157907954387:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25867
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.618315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789157722297226:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23057
2025-11-28T14:52:07.618083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789159734542272:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.620331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789158159581316:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.621154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789159734542201:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.621147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789157907954388:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25867
2025-11-28T14:52:07.621283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789158159581243:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.621633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789157722297218:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.621663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789159734542275:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40744
2025-11-28T14:52:07.621670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789157722297218:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:07.622708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789159734542281:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40744
No RDMA context found for GID: No RDMA context found for GID: 2025-11-28T14:52:07.622495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789157907954387:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:07.622934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789157722297227:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23057
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.622954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789157722297217:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.625662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789158159581321:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52326
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.624773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789159695449438:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:07.624784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789157907954313:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.624754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789159407315939:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18510
2025-11-28T14:52:07.625723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789159734542275:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.622980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789157722297217:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:07.625727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789157907954392:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25867
2025-11-28T14:52:07.626310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789159695449439:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:07.623228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789157722297226:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.627682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789159734542202:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.628381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789159407315940:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18510
2025-11-28T14:52:07.626012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789157907954393:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25867
2025-11-28T14:52:07.639827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789159407315932:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.624776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789157722297229:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23057
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.627590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789157907954388:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.639879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789159407315932:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:07.651337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789159407315928:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.624822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789157722297218:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:07.658853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789159734542283:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40744
2025-11-28T14:52:07.628337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789157907954396:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25867
2025-11-28T14:52:07.639788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789157907954314:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.625705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789157722297228:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23057
2025-11-28T14:52:07.651396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789159407315928:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:07.639954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789157722297217:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:07.659196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789159407315932:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.659458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789159407315928:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.640463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789157722297211:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.660437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789159407315924:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.647394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789157722297216:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.660475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:07.660839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789158159581324:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52326
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.660560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789159407315942:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18510
2025-11-28T14:52:07.647416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:07.660579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789159407315924:2108] [node 5] ICP22 created new session: [2:7577789159407315944:2108]
2025-11-28T14:52:07.647462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789157722297216:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:07.660842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789159407315860:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.660845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789159407315944:2108] [node 5] ICS09 handshake done sender: [2:7577789159407315932:2113] self: [2:7577789159407315930:0] peer: [5:7577789159734542284:0] socket: 365 qp: -1
2025-11-28T14:52:07.651419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789157722297211:2109] [node 5] ICP22 created new session: [1:7577789157722297233:2109]
2025-11-28T14:52:07.660866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789159407315944:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:07.660885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:07.662216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789159695449440:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:07.657363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789157722297210:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.660885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:07.661125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789159407315860:2049] [node 3] ICP22 created new session: [2:7577789159407315945:2049]
2025-11-28T14:52:07.657406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:07.662349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789159734542272:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.658454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789157722297233:2109] [node 5] ICS09 handshake done sender: [1:7577789157722297218:2114] self: [1:7577789157722297215:0] peer: [5:7577789159734542282:0] socket: 355 qp: -1
2025-11-28T14:52:07.658497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789157722297233:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:07.661967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789159407315945:2049] [node 3] ICS09 handshake done sender: [2:7577789159407315928:2111] self: [2:7577789159407315927:0] peer: [3:7577789157907954398:0] socket: 361 qp: -1
2025-11-28T14:52:07.658837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789157722297210:2107] [node 4] ICP22 created new session: [1:7577789157722297234:2107]
2025-11-28T14:52:07.662054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789159407315945:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:07.658871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.662105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:07.663151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789157907954399:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25867
2025-11-28T14:52:07.659386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789157722297234:2107] [node 4] ICS09 handshake done sender: [1:7577789157722297217:2113] self: [1:7577789157722297213:0] peer: [4:7577789158159581323:0] socket: 356 qp: -1
2025-11-28T14:52:07.663489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789159407315942:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.663564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789159734542201:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.659420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789157722297234:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:07.663517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789159695449441:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:07.659455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:07.663618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789158159581316:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:07.663632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:07.663680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789159407315859:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.659960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789157722297216:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:07.663683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789159734542201:2048] [node 1] ICP22 created new session: [5:7577789159734542286:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.661672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789157722297211:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(20274476ns) {HandleHandshakeStatus:329(20271883ns) {DropIncomingHandshake:442(735ns) DropOutgoingHandshake:457(779ns) SwitchToState:179(595ns) ProcessPendingSessionEvents:495(1365058ns) {ForwardSessionEventToSession:612(461959ns) ForwardSessionEventToSession:612(172976ns) ForwardSessionEventToSession:612(255188ns) ForwardSessionEventToSession:612(2340ns) ForwardSessionEventToSession:612(191242ns) ForwardSessionEventToSession:612(2917ns) ForwardSessionEventToSession:612(119758ns) ForwardSessionEventToSession:612(2286ns) ForwardSessionEventToSession:612(1973ns) ForwardSessionEventToSession:612(141055ns)}}}
2025-11-28T14:52:07.664290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789159734542286:2048] [node 1] ICS09 handshake done sender: [5:7577789159734542272:2112] self: [5:7577789159734542282:0] peer: [1:7577789157722297215:0] socket: 359 qp: -1
2025-11-28T14:52:07.661718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789157722297146:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.664500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789159734542286:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:07.661752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:07.665673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789157907954400:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25867
2025-11-28T14:52:07.665896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789159695449441:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.664539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:07.662035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789157722297146:2049] [node 3] ICP22 created new session: [1:7577789157722297240:2049]
2025-11-28T14:52:07.666386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789158159581243:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.662237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789157722297240:2049] [node 3] ICS09 handshake done sender: [1:7577789157722297216:2112] self: [1:7577789157722297214:0] peer: [3:7577789157907954397:0] socket: 354 qp: -1
2025-11-28T14:52:07.666444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:07.666797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789159695449361:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.666637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789158159581243:2048] [node 1] ICP22 created new session: [4:7577789158159581326:2048]
2025-11-28T14:52:07.662264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789157722297240:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:07.666925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789158159581326:2048] [node 1] ICS09 handshake done sender: [4:7577789158159581316:2114] self: [4:7577789158159581323:0] peer: [1:7577789157722297213:0] socket: 358 qp: -1
2025-11-28T14:52:07.666991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789158159581326:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:07.667099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789159407315933:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.662348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:07.667119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:07.665687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789157722297220:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.667132Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789159407315933:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:07.667362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789157907954388:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:07.665717Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789157722297220:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:07.668064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789159407315951:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18510
2025-11-28T14:52:07.666905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789157722297220:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:07.667938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789159695449440:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.667225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789157722297145:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.668240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789159407315933:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:07.667284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:07.668447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789159695449360:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.667847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789157722297145:2048] [node 2] ICP22 created new session: [1:7577789157722297243:2048]
2025-11-28T14:52:07.667963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789157722297243:2048] [node 2] ICS09 handshake done sender: [1:7577789157722297220:2115] self: [1:7577789157722297219:0] peer: [2:7577789159407315950:0] socket: 400 qp: -1
2025-11-28T14:52:07.668453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789159407315942:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:07.667993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789157722297243:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:07.668219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:07.668647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789159407315923:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:07.668920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789157907954387:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:07.668826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:07.670245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789159734542285:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40744
2025-11-28T14:52:07.670397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789157722297225:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.670439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789157722297225:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:07.670251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789159695449443:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:07.668874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789159407315923:2110] [node 6] ICP22 created new session: [2:7577789159407315953:2110]
2025-11-28T14:52:07.668920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789159407315953:2110] [node 6] ICS09 handshake done sender: [2:7577789159407315933:2114] self: [2:7577789159407315931:0] peer: [6:7577789159695449442:0] socket: 410 qp: -1
2025-11-28T14:52:07.670921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789157907954314:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.669055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789159407315953:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:07.670971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:07.671014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789157907954314:2049] [node 2] ICP22 created new session: [3:7577789157907954401:2049]
2025-11-28T14:52:07.669094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.671389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789157907954401:2049] [node 2] ICS09 handshake done sender: [3:7577789157907954388:2115] self: [3:7577789157907954398:0] peer: [2:7577789159407315927:0] socket: 364 qp: -1
2025-11-28T14:52:07.671422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789157907954401:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:07.669423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789159407315859:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789159407315942:2118] held: no
2025-11-28T14:52:07.671687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789158159581325:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52326
2025-11-28T14:52:07.671456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:07.670470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789159407315859:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.670598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:07.670835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789159407315859:2048] [node 1] ICP22 created new session: [2:7577789159407315955:2048]
2025-11-28T14:52:07.670957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789159407315955:2048] [node 1] ICS09 handshake done sender: [2:7577789159407315942:2118] self: [2:7577789159407315950:0] peer: [1:7577789157722297219:0] socket: 401 qp: -1
2025-11-28T14:52:07.670994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789159407315955:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:07.671037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:07.671969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789159734542275:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:07.671986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789157722297225:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:07.672193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789158159581325:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.672414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789158159581244:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.672413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789159695449441:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:07.672516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789157722297212:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.672716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789159734542202:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.672574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:07.672866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789159407315938:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.672777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789157907954313:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.672762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:07.672793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789157722297212:2110] [node 6] ICP22 created new session: [1:7577789157722297246:2110]
2025-11-28T14:52:07.672810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:07.672901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789159407315938:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:07.672897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789157907954313:2048] [node 1] ICP22 created new session: [3:7577789157907954403:2048]
2025-11-28T14:52:07.672914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789157722297246:2110] [node 6] ICS09 handshake done sender: [1:7577789157722297225:2116] self: [1:7577789157722297224:0] peer: [6:7577789159695449444:0] socket: 404 qp: -1
2025-11-28T14:52:07.672816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789159734542202:2049] [node 2] ICP22 created new session: [5:7577789159734542288:2049]
2025-11-28T14:52:07.672951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789157907954403:2048] [node 1] ICS09 handshake done sender: [3:7577789157907954387:2114] self: [3:7577789157907954397:0] peer: [1:7577789157722297214:0] socket: 360 qp: -1
2025-11-28T14:52:07.672986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789157907954403:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:07.672964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789157722297246:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:07.672908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789159734542288:2049] [node 2] ICS09 handshake done sender: [5:7577789159734542275:2115] self: [5:7577789159734542284:0] peer: [2:7577789159407315930:0] socket: 366 qp: -1
2025-11-28T14:52:07.673029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:07.672941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789159734542288:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:07.673010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:07.673049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:07.673555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789159695449361:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.673596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:07.673618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789159695449361:2049] [node 2] ICP22 created new session: [6:7577789159695449446:2049]
2025-11-28T14:52:07.673652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789159695449446:2049] [node 2] ICS09 handshake done sender: [6:7577789159695449441:2118] self: [6:7577789159695449442:0] peer: [2:7577789159407315931:0] socket: 412 qp: -1
2025-11-28T14:52:07.673671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789159695449446:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:07.673697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:07.674230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789159407315938:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.674762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789158159581329:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52326
2025-11-28T14:52:07.674589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789159407315925:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.674773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:07.674833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789159407315925:2107] [node 4] ICP22 created new session: [2:7577789159407315958:2107]
2025-11-28T14:52:07.674930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789159695449445:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:07.674915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789159407315958:2107] [node 4] ICS09 handshake done sender: [2:7577789159407315938:2115] self: [2:7577789159407315937:0] peer: [4:7577789158159581328:0] socket: 411 qp: -1
2025-11-28T14:52:07.674954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789159407315958:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:07.674999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:07.675250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789158159581325:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:07.675563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789159695449440:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:07.675727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789158159581244:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.675778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:07.675818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789158159581244:2049] [node 2] ICP22 created new session: [4:7577789158159581330:2049]
2025-11-28T14:52:07.675975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789158159581330:2049] [node 2] ICS09 handshake done sender: [4:7577789158159581325:2118] self: [4:7577789158159581328:0] peer: [2:7577789159407315937:0] socket: 413 qp: -1
2025-11-28T14:52:07.675981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789159695449360:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.676023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789158159581330:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:07.675967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789159734542290:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40744
2025-11-28T14:52:07.676024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:07.676051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:07.676097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789159695449360:2048] [node 1] ICP22 created new session: [6:7577789159695449449:2048]
2025-11-28T14:52:07.676138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789159695449449:2048] [node 1] ICS09 handshake done sender: [6:7577789159695449440:2117] self: [6:7577789159695449444:0] peer: [1:7577789157722297224:0] socket: 407 qp: -1
2025-11-28T14:52:07.676160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789159695449449:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:07.676302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.678940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789158159581332:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52326
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.679070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789158159581333:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52326
2025-11-28T14:52:07.682017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789158159581313:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.706407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789159407315939:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.706777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789159407315925:2107] [node 4] ICP18 handshake (actor [2:7577789159407315939:2116]) is too old
2025-11-28T14:52:07.712226Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789158159581310:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.743815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789159695449429:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.772727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789159695449451:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:07.776852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789159734542271:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.814084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789159734542267:2110] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.821859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789159695449438:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.822056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789159695449421:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.822365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789159734542267:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.822986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789159734542267:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.823143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789159695449453:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:07.823381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789159734542264:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.823451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:07.823525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789159734542264:2109] [node 6] ICP22 created new session: [5:7577789159734542293:2109]
2025-11-28T14:52:07.823644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789159734542293:2109] [node 6] ICS09 handshake done sender: [5:7577789159734542267:2110] self: [5:7577789159734542266:0] peer: [6:7577789159695449452:0] socket: 378 qp: -1
2025-11-28T14:52:07.823697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789159734542293:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:07.823767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:07.823784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789159695449438:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:07.824116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789159695449421:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.824160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:07.824206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789159695449421:2106] [node 5] ICP22 created new session: [6:7577789159695449454:2106]
2025-11-28T14:52:07.824239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789159695449454:2106] [node 5] ICS09 handshake done sender: [6:7577789159695449438:2115] self: [6:7577789159695449452:0] peer: [5:7577789159734542266:0] socket: 392 qp: -1
2025-11-28T14:52:07.824259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789159695449454:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:07.824281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:07.833482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789159695449431:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.856505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789159734542281:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.856829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789159734542264:2109] [node 6] ICP18 handshake (actor [5:7577789159734542281:2117]) is too old
2025-11-28T14:52:07.869912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789158159581314:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.878872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789157722297227:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.879127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789157722297210:2107] [node 4] ICP18 handshake (actor [1:7577789157722297227:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.879944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789159734542295:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40744
2025-11-28T14:52:07.884508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789159734542273:2113] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.894522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789157722297229:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.894740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789157722297211:2109] [node 5] ICP18 handshake (actor [1:7577789157722297229:2120]) is too old
2025-11-28T14:52:07.897984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789157907954384:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.911870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789159695449430:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.917761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789159407315940:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.930923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789159407315860:2049] [node 3] ICP18 handshake (actor [2:7577789159407315940:2117]) is too old
2025-11-28T14:52:07.933717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789157907954392:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.933921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789157907954393:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.934202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789157907954396:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.937510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789157722297228:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.945867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789157907954380:2110] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.962323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789157722297212:2110] [node 6] ICP18 handshake (actor [1:7577789157722297228:2119]) is too old
2025-11-28T14:52:07.964579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [1:7577789157722297212:2110] [node 6] ICP35 event processing took too much time StateWork:294(23796472ns) {IncomingHandshake:261(23793216ns)}
2025-11-28T14:52:07.966577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789159695449439:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.966879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789159695449362:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:07.968585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789157907954380:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:07.969321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789157907954380:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:07.969664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789157907954376:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:07.969610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789159695449457:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:07.969744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:07.969912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789157907954376:2109] [node 6] ICP22 created new session: [3:7577789157907954410:2109]
2025-11-28T14:52:07.969973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789157907954410:2109] [node 6] ICS09 handshake done sender: [3:7577789157907954380:2110] self: [3:7577789157907954379:0] peer: [6:7577789159695449456:0] socket: 381 qp: -1
2025-11-28T14:52:07.970009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789157907954410:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:07.970046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:07.971591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789159695449362:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2917) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789159695449439:2116] held: no
2025-11-28T14:52:07.976882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789157907954385:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:07.971654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789159695449439:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:07.971779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789159695449362:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:07.972055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:07.972125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789159695449362:2050] [node 3] ICP22 created new session: [6:7577789159695449458:2050]
2025-11-28T14:52:07.986468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789159695449458:2050] [node 3] ICS09 handshake done sender: [6:7577789159695449439:2116] self: [6:7577789159695449456:0] peer: [3:7577789157907954379:0] socket: 393 qp: -1
2025-11-28T14:52:07.990076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789159695449458:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:07.998854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:08.000815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789159695449362:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(28881983ns) {HandleHandshakeStatus:329(28879809ns) {DropIncomingHandshake:442(863ns) DropOutgoingHandshake:457(143ns) SwitchToState:179(708ns) ProcessPendingSessionEvents:495(1675ns)}}
2025-11-28T14:52:08.002781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789157907954385:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.003435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789157907954385:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.002270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789158159581321:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.002469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789158159581245:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.003725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789162454548633:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52326
2025-11-28T14:52:08.003695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789157907954377:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.003766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:08.003818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789157907954377:2107] [node 4] ICP22 created new session: [3:7577789162202921709:2107]
2025-11-28T14:52:08.003881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789162202921709:2107] [node 4] ICS09 handshake done sender: [3:7577789157907954385:2112] self: [3:7577789157907954381:0] peer: [4:7577789162454548632:0] socket: 374 qp: -1
2025-11-28T14:52:08.003914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789162202921709:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:08.003957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:08.005561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789158159581245:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2917) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789158159581321:2115] held: no
2025-11-28T14:52:08.005709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789158159581321:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:08.008471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789157907954395:2118] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:08.006020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789158159581245:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.006061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:08.006114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789158159581245:2050] [node 3] ICP22 created new session: [4:7577789162454548634:2050]
2025-11-28T14:52:08.006157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789162454548634:2050] [node 3] ICS09 handshake done sender: [4:7577789158159581321:2115] self: [4:7577789162454548632:0] peer: [3:7577789157907954381:0] socket: 377 qp: -1
2025-11-28T14:52:08.006200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789162454548634:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:08.006247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:08.012207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789157907954395:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.011450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789159734542290:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.011676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789159734542203:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.012823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789157907954395:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.034297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789164029509593:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40744
2025-11-28T14:52:08.034598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789159734542290:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:08.034736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789159734542203:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.043492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789157907954378:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.034811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:08.043997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:08.040291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789159734542203:2050] [node 3] ICP22 created new session: [5:7577789164029509594:2050]
2025-11-28T14:52:08.043791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789164029509594:2050] [node 3] ICS09 handshake done sender: [5:7577789159734542290:2120] self: [5:7577789164029509592:0] peer: [3:7577789157907954394:0] socket: 421 qp: -1
2025-11-28T14:52:08.044076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789157907954378:2108] [node 5] ICP22 created new session: [3:7577789162202921712:2108]
2025-11-28T14:52:08.043853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789164029509594:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:08.044278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:08.044422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789162202921712:2108] [node 5] ICS09 handshake done sender: [3:7577789157907954395:2118] self: [3:7577789157907954394:0] peer: [5:7577789164029509592:0] socket: 420 qp: -1
2025-11-28T14:52:08.044484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789162202921712:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:08.044549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:08.055945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789159734542280:2116] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:08.059203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789159734542203:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(24174476ns) {HandleHandshakeStatus:329(24171004ns) {DropIncomingHandshake:442(1317ns) DropOutgoingHandshake:457(6987ns) SwitchToState:179(885ns) ProcessPendingSessionEvents:495(14388006ns) {ForwardSessionEventToSession:612(100300ns) ForwardSessionEventToSession:612(1917359ns) ForwardSessionEventToSession:612(4528ns) ForwardSessionEventToSession:612(3518ns) ForwardSessionEventToSession:612(9418059ns) ForwardSessionEventToSession:612(5073ns) ForwardSessionEventToSession:612(5675ns) ForwardSessionEventToSession:612(11207ns) ForwardSessionEventToSession:612(3228ns) ForwardSessionEventToSession:612(2893449ns)}}}
2025-11-28T14:52:08.059337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789157907954378:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18634442ns) {HandleHandshakeStatus:329(18631843ns) {DropIncomingHandshake:442(30091ns) DropOutgoingHandshake:457(409ns) SwitchToState:179(772ns) ProcessPendingSessionEvents:495(3044577ns) {ForwardSessionEventToSession:612(2952979ns) ForwardSessionEventToSession:612(5802ns) ForwardSessionEventToSession:612(2554ns) ForwardSessionEventToSession:612(6346ns) ForwardSessionEventToSession:612(23826ns) ForwardSessionEventToSession:612(12607ns) ForwardSessionEventToSession:612(2077ns) ForwardSessionEventToSession:612(2913ns) ForwardSessionEventToSession:612(14615ns) ForwardSessionEventToSession:612(2253ns)}}}
2025-11-28T14:52:08.060103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789158159581332:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.063334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789159695449437:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:08.072626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789158159581333:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.075232Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789158159581315:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:08.076269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789159695449451:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.076507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789159695449424:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.076703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789158159581315:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.083722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789158159581315:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.083920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789163990416757:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15906
2025-11-28T14:52:08.084209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789158159581306:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.084339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789159695449451:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:08.084291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:08.084623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789158159581306:2109] [node 6] ICP22 created new session: [4:7577789162454548639:2109]
2025-11-28T14:52:08.085464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789162454548639:2109] [node 6] ICS09 handshake done sender: [4:7577789158159581315:2113] self: [4:7577789158159581312:0] peer: [6:7577789163990416756:0] socket: 373 qp: -1
2025-11-28T14:52:08.084801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789159695449424:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.085517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789162454548639:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:08.084854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:08.085706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:08.085558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789159695449424:2107] [node 4] ICP22 created new session: [6:7577789163990416758:2107]
2025-11-28T14:52:08.085844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789163990416758:2107] [node 4] ICS09 handshake done sender: [6:7577789159695449451:2121] self: [6:7577789163990416756:0] peer: [4:7577789158159581312:0] socket: 375 qp: -1
2025-11-28T14:52:08.085890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789163990416758:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:08.085937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:08.089886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789158159581322:2116] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:08.090924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789159734542295:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.091216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789159734542265:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.091405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789158159581322:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.091830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789158159581322:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.092035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789158159581307:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.092073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789164029509597:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40744
2025-11-28T14:52:08.092173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:08.092223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789158159581307:2108] [node 5] ICP22 created new session: [4:7577789162454548642:2108]
2025-11-28T14:52:08.092269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789162454548642:2108] [node 5] ICS09 handshake done sender: [4:7577789158159581322:2116] self: [4:7577789158159581320:0] peer: [5:7577789164029509596:0] socket: 369 qp: -1
2025-11-28T14:52:08.092307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789162454548642:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:08.092345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:08.092399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789159734542295:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:08.092758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789159734542265:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.092859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:08.092917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789159734542265:2108] [node 4] ICP22 created new session: [5:7577789164029509598:2108]
2025-11-28T14:52:08.092966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789164029509598:2108] [node 4] ICS09 handshake done sender: [5:7577789159734542295:2121] self: [5:7577789164029509596:0] peer: [4:7577789158159581320:0] socket: 371 qp: -1
2025-11-28T14:52:08.092995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789164029509598:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:08.093033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:08.093371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789159734542265:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:08.156536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:08.156820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789157907954403:2048] [node 1] ICS07 socket disconnect 360 reason# EndOfStream
2025-11-28T14:52:08.156853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789157907954403:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.156942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:08.157045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789157907954403:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:08.158065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789157907954313:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.158137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789157907954403:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.158181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789157907954313:2048] [node 1] ICP30 unregister session Session# [3:7577789157907954403:2048] VirtualId# [3:7577789157907954397:0]
2025-11-28T14:52:08.158230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:08.158261Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789157907954313:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.158276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.156574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:08.156833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789159407315955:2048] [node 1] ICS01 socket: 401 reason# EndOfStream
2025-11-28T14:52:08.156880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789159407315859:2048] [node 1] ICP30 unregister session Session# [2:7577789159407315955:2048] VirtualId# [2:7577789159407315950:0]
2025-11-28T14:52:08.156915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789159407315955:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.157057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:08.157119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:08.160520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:08.160759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789159695449449:2048] [node 1] ICS07 socket disconnect 407 reason# EndOfStream
2025-11-28T14:52:08.160513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:08.160753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789158159581326:2048] [node 1] ICS07 socket disconnect 358 reason# EndOfStream
2025-11-28T14:52:08.160783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789158159581326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.160851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:08.160798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789159695449449:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.160887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789158159581326:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:08.160869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:08.160908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789159695449449:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:08.160513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:08.160755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789159734542286:2048] [node 1] ICS07 socket disconnect 359 reason# EndOfStream
2025-11-28T14:52:08.162334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789159695449360:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.160783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789159734542286:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.163519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789159695449449:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.160851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:08.168130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789157907954313:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:08.163554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789159695449360:2048] [node 1] ICP30 unregister session Session# [6:7577789159695449449:2048] VirtualId# [6:7577789159695449444:0]
2025-11-28T14:52:08.169938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789158159581243:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.163615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:08.160887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789159734542286:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:08.176674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789159734542201:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.176559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789158159581326:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.178016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789159734542286:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.176604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789158159581243:2048] [node 1] ICP30 unregister session Session# [4:7577789158159581326:2048] VirtualId# [4:7577789158159581323:0]
2025-11-28T14:52:08.180394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789159734542201:2048] [node 1] ICP30 unregister session Session# [5:7577789159734542286:2048] VirtualId# [5:7577789159734542282:0]
2025-11-28T14:52:08.179853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:08.180452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:08.180490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789159734542201:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.180507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.163662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789159695449360:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.163677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.177104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789159695449360:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:08.180788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789158159581243:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.180813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4442) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.191175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789159734542201:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:08.190638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789158159581243:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:08.233907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:08.234134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789157907954401:2049] [node 2] ICS07 socket disconnect 364 reason# EndOfStream
2025-11-28T14:52:08.234163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789157907954401:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.234261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:08.234304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789157907954401:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:08.238270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789157907954314:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:08.238317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789157907954401:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:08.239301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789157907954314:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.239357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789157907954401:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.239393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789157907954314:2049] [node 2] ICP30 unregister session Session# [3:7577789157907954401:2049] VirtualId# [3:7577789157907954398:0]
2025-11-28T14:52:08.239452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:08.239486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789157907954314:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.239500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.242287Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:08.242571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789158159581330:2049] [node 2] ICS07 socket disconnect 413 reason# EndOfStream
2025-11-28T14:52:08.242607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789158159581330:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.243038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:08.243193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:08.242753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:08.243252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789159734542288:2049] [node 2] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-28T14:52:08.242804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789158159581330:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:08.243279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789159734542288:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.243379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:08.243408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789159695449446:2049] [node 2] ICS07 socket disconnect 412 reason# EndOfStream
2025-11-28T14:52:08.243437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789159734542288:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:08.243433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789159695449446:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.243489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:08.243530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789159695449446:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:08.243820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789158159581244:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.243877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789158159581330:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.243966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789158159581244:2049] [node 2] ICP30 unregister session Session# [4:7577789158159581330:2049] VirtualId# [4:7577789158159581328:0]
2025-11-28T14:52:08.244013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:08.244043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789158159581244:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.244057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.244150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789159734542202:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.244257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789159734542288:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.244306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789159695449361:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.244292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789159734542202:2049] [node 2] ICP30 unregister session Session# [5:7577789159734542288:2049] VirtualId# [5:7577789159734542284:0]
2025-11-28T14:52:08.244573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789159695449446:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.244330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:08.244378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789159734542202:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.244608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789159695449361:2049] [node 2] ICP30 unregister session Session# [6:7577789159695449446:2049] VirtualId# [6:7577789159695449442:0]
2025-11-28T14:52:08.244396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.244664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:08.244706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789159695449361:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.244739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.249037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789157907954314:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:08.254149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789159734542202:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:08.254544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789159695449361:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:08.255132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789158159581244:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:08.256409Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:08.256639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789162454548634:2050] [node 3] ICS01 socket: 377 reason# EndOfStream
2025-11-28T14:52:08.256691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789158159581245:2050] [node 3] ICP30 unregister session Session# [4:7577789162454548634:2050] VirtualId# [4:7577789162454548632:0]
2025-11-28T14:52:08.256725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789162454548634:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.256810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:08.256870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:08.257496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:08.257693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789164029509594:2050] [node 3] ICS01 socket: 421 reason# EndOfStream
2025-11-28T14:52:08.257735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789159734542203:2050] [node 3] ICP30 unregister session Session# [5:7577789164029509594:2050] VirtualId# [5:7577789164029509592:0]
2025-11-28T14:52:08.257775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789164029509594:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.257853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:08.257910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:08.259788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:08.260009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789159695449458:2050] [node 3] ICS01 socket: 393 reason# EndOfStream
2025-11-28T14:52:08.260061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789159695449362:2050] [node 3] ICP30 unregister session Session# [6:7577789159695449458:2050] VirtualId# [6:7577789159695449456:0]
2025-11-28T14:52:08.260096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789159695449458:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.260193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:08.260249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:08.271534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:08.271681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789164029509598:2108] [node 4] ICS07 socket disconnect 371 reason# EndOfStream
2025-11-28T14:52:08.271699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789164029509598:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.271769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:08.271802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789164029509598:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:08.271308Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:08.271478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789163990416758:2107] [node 4] ICS07 socket disconnect 375 reason# EndOfStream
2025-11-28T14:52:08.271501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789163990416758:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.271581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:08.271617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789163990416758:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:08.272735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789159734542265:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26316) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.273178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789164029509598:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.273236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789159734542265:2108] [node 4] ICP30 unregister session Session# [5:7577789164029509598:2108] VirtualId# [5:7577789164029509596:0]
2025-11-28T14:52:08.273289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:08.273313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789159734542265:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26316) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.273326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26316) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.274274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789159695449424:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26316) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.276624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789163990416758:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.276686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789159695449424:2107] [node 4] ICP30 unregister session Session# [6:7577789163990416758:2107] VirtualId# [6:7577789163990416756:0]
2025-11-28T14:52:08.276777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:08.276855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789159695449424:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26316) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.276882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26316) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.285021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:08.285273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789159695449454:2106] [node 5] ICS07 socket disconnect 392 reason# EndOfStream
2025-11-28T14:52:08.285306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789159695449454:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:08.285377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:08.285436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789159695449454:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:08.285822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789159695449424:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:08.286457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789159695449421:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10399) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:08.286586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789159695449454:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:08.286689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789159695449421:2106] [node 5] ICP30 unregister session Session# [6:7577789159695449454:2106] VirtualId# [6:7577789159695449452:0]
2025-11-28T14:52:08.286751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:08.286792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789159695449421:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:08.286837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:08.389857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789165937032091:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:08.390178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789165937032092:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:08.421616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789161833091610:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:08.421628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789161833091609:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:08.451364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789164152931010:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:08.451411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789164152931011:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:08.476870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789163647259575:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:08.476920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789163647259576:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:08.477642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789163647259574:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:08.506579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789164519438598:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:08.506637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789164519438600:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:08.506654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789164519438599:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:08.536465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789164602451590:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:08.536576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789164602451589:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:08.536664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789164602451588:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:08.614147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789164602451649:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:08.654974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789165937032154:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:08.654993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789161833091673:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:08.673013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789161833091672:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:08.661997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789164152931073:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:08.669907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789163647259637:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:08.674330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789165937032155:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:08.675081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789161833091674:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:08.675156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789164152931075:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:08.676128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789164602451655:2110] [node 4] ICP01 ready to work
2025-11-28T14:52:08.676433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789163647259638:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:08.674717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789164152931074:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:08.675886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789164519438662:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:08.676571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789165937032156:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:08.675933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789164519438664:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:08.679251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789165937032092:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(65287394ns) {Configure:104(65281930ns) {ProcessConfigured:135(734670ns) {StartInitialHandshake:157(731257ns) {DropHandshakes:470(2249ns) {DropIncomingHandshake:442(558ns) DropOutgoingHandshake:457(316ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(1525ns) {SwitchToState:179(513ns)}}}}}
2025-11-28T14:52:08.681678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789165937032091:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(69674990ns) {Configure:104(69667733ns) {ProcessConfigured:135(1623766ns) {StartInitialHandshake:157(1620792ns) {DropHandshakes:470(1231ns) {DropIncomingHandshake:442(373ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(676ns) PrepareNewSessionHandshake:477(2014ns) {SwitchToState:179(661ns)}}}}}
2025-11-28T14:52:08.682842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789164519438598:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50843458ns) {Configure:104(50835220ns) {ProcessConfigured:135(801845ns) {StartInitialHandshake:157(798968ns) {DropHandshakes:470(1891ns) {DropIncomingHandshake:442(985ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(1705ns) {SwitchToState:179(729ns)}}}}}
2025-11-28T14:52:08.698666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789164602451589:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46569333ns) {Configure:104(46563944ns) {ProcessConfigured:135(4692095ns) {StartInitialHandshake:157(4688825ns) {DropHandshakes:470(550225ns) {DropIncomingHandshake:442(1178ns) DropOutgoingHandshake:457(404ns)} GenerateSessionVirtualId:988(1213ns) PrepareNewSessionHandshake:477(2571ns) {SwitchToState:179(1293ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.703674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789165937032156:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20712389ns) {Configure:104(20708439ns) {ProcessConfigured:135(1215790ns) {StartInitialHandshake:157(1213538ns) {DropHandshakes:470(1273ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(654ns) PrepareNewSessionHandshake:477(2071ns) {SwitchToState:179(983ns)}}}}}
2025-11-28T14:52:08.703036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789161833091609:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(76684547ns) {Configure:104(76680464ns) {ProcessConfigured:135(9904085ns) {StartInitialHandshake:157(9901396ns) {DropHandshakes:470(637ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(779ns) PrepareNewSessionHandshake:477(2170ns) {SwitchToState:179(257ns)}}}}}
2025-11-28T14:52:08.704004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789164152931011:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29162377ns) {Configure:104(29152780ns) {ProcessConfigured:135(45717ns) {StartInitialHandshake:157(44341ns) {DropHandshakes:470(966ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(298ns) PrepareNewSessionHandshake:477(1062ns) {SwitchToState:179(617ns)}}}}}
2025-11-28T14:52:08.703845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789164519438599:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21568533ns) {Configure:104(21564015ns) {ProcessConfigured:135(9255219ns) {StartInitialHandshake:157(9252303ns) {DropHandshakes:470(1190ns) {DropIncomingHandshake:442(257ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:477(3478ns) {SwitchToState:179(596ns)}}}}}
2025-11-28T14:52:08.703143Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789164602451588:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22481219ns) {Configure:104(22471576ns) {ProcessConfigured:135(7906801ns) {StartInitialHandshake:157(7904291ns) {DropHandshakes:470(1200ns) {DropIncomingHandshake:442(389ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(2413ns) {SwitchToState:179(1224ns)}}}}}
2025-11-28T14:52:08.704866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789165937032154:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29572014ns) {Configure:104(29564851ns) {ProcessConfigured:135(658398ns) {StartInitialHandshake:157(272630ns) {DropHandshakes:470(1177ns) {DropIncomingHandshake:442(585ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(789ns) PrepareNewSessionHandshake:477(1490ns) {SwitchToState:179(747ns)}}}}}
2025-11-28T14:52:08.704898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789163647259575:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25918131ns) {Configure:104(25910334ns) {ProcessConfigured:135(1042476ns) {StartInitialHandshake:157(1041401ns) {DropHandshakes:470(1122ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(209ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:477(1359ns) {SwitchToState:179(540ns)}}}}}
2025-11-28T14:52:08.703198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789161833091610:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(76995541ns) {Configure:104(76101133ns) {ProcessConfigured:135(10401832ns) {StartInitialHandshake:157(10399950ns) {DropHandshakes:470(1439ns) {DropIncomingHandshake:442(283ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(1607ns) {SwitchToState:179(181ns)}}}}}
2025-11-28T14:52:08.705631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789164152931075:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27388004ns) {Configure:104(27383885ns) {ProcessConfigured:135(5161690ns) {StartInitialHandshake:157(5159223ns) {DropHandshakes:470(1251ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(464ns) PrepareNewSessionHandshake:477(2023ns) {SwitchToState:179(495ns)}}}}}
2025-11-28T14:52:08.704448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789164519438600:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23241386ns) {Configure:104(23236715ns) {ProcessConfigured:135(1790958ns) {StartInitialHandshake:157(1787345ns) {DropHandshakes:470(1949ns) {DropIncomingHandshake:442(489ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(529ns) PrepareNewSessionHandshake:477(2266ns) {SwitchToState:179(1143ns)}}}}}
2025-11-28T14:52:08.706082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789164602451655:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21657360ns) {Configure:104(21652961ns) {ProcessConfigured:135(1819034ns) {StartInitialHandshake:157(1816514ns) {DropHandshakes:470(1209ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:477(1987ns) {SwitchToState:179(935ns)}}}}}
2025-11-28T14:52:08.704638Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789161833091673:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26296013ns) {Configure:104(26289354ns) {ProcessConfigured:135(1056075ns) {StartInitialHandshake:157(624897ns) {DropHandshakes:470(1705ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(837ns) PrepareNewSessionHandshake:477(1072ns) {SwitchToState:179(136ns)}}}}}
2025-11-28T14:52:08.705960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789164152931010:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44307070ns) {Configure:104(36495801ns) {ProcessConfigured:135(569601ns) {StartInitialHandshake:157(568179ns) {DropHandshakes:470(1160ns) {DropIncomingHandshake:442(625ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(695ns) PrepareNewSessionHandshake:477(1096ns) {SwitchToState:179(99ns)}}}}}
2025-11-28T14:52:08.706042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789163647259574:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(79068574ns) {Configure:104(79064552ns) {ProcessConfigured:135(5095282ns) {StartInitialHandshake:157(5093681ns) {DropHandshakes:470(916ns) {DropIncomingHandshake:442(159ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:477(1352ns) {SwitchToState:179(152ns)}}}}}
2025-11-28T14:52:08.705172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789164519438663:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49761
2025-11-28T14:52:08.706451Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789164152931073:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27417307ns) {Configure:104(27410864ns) {ProcessConfigured:135(3891932ns) {StartInitialHandshake:157(3889650ns) {DropHandshakes:470(1743ns) {DropIncomingHandshake:442(766ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(874ns) PrepareNewSessionHandshake:477(1914ns) {SwitchToState:179(798ns)}}}}}
2025-11-28T14:52:08.704837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789161833091674:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22745931ns) {Configure:104(22741490ns) {ProcessConfigured:135(1523844ns) {StartInitialHandshake:157(1521795ns) {DropHandshakes:470(1045ns) {DropIncomingHandshake:442(296ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:477(1072ns) {SwitchToState:179(75ns)}}}}}
2025-11-28T14:52:08.706458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789164519438662:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26797811ns) {Configure:104(26794279ns) {ProcessConfigured:135(1815457ns) {StartInitialHandshake:157(1814000ns) {DropHandshakes:470(1027ns) {DropIncomingHandshake:442(280ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(541ns) PrepareNewSessionHandshake:477(1805ns) {SwitchToState:179(622ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.706370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789163647259576:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34707117ns) {Configure:104(34698793ns) {ProcessConfigured:135(1192932ns) {StartInitialHandshake:157(1190838ns) {DropHandshakes:470(1124ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(360ns) PrepareNewSessionHandshake:477(1512ns) {SwitchToState:179(884ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.709333Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789164152931074:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29147671ns) {Configure:104(29143157ns) {ProcessConfigured:135(2186627ns) {StartInitialHandshake:157(2184757ns) {DropHandshakes:470(1191ns) {DropIncomingHandshake:442(397ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(313ns) PrepareNewSessionHandshake:477(1967ns) {SwitchToState:179(162ns)}}}}}
2025-11-28T14:52:08.707325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789161833091672:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28794822ns) {Configure:104(28444891ns) {ProcessConfigured:135(7109558ns) {StartInitialHandshake:157(6403645ns) {DropHandshakes:470(1313ns) {DropIncomingHandshake:442(581ns) DropOutgoingHandshake:457(117ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:477(1888ns) {SwitchToState:179(559ns)}}}}}
2025-11-28T14:52:08.707788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789164519438664:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25498700ns) {Configure:104(25485528ns) {ProcessConfigured:135(5007059ns) {StartInitialHandshake:157(5005072ns) {DropHandshakes:470(3647636ns) {DropIncomingHandshake:442(1843ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(1912ns) PrepareNewSessionHandshake:477(10488ns) {SwitchToState:179(9568ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.707356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789163647259637:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28614154ns) {Configure:104(28609021ns) {ProcessConfigured:135(5408288ns) {StartInitialHandshake:157(5406526ns) {DropHandshakes:470(931ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(134ns)} GenerateSessionVirtualId:988(305ns) PrepareNewSessionHandshake:477(1977ns) {SwitchToState:179(157ns)}}}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.713225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789161833091687:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1315
2025-11-28T14:52:08.710716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789164519438663:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.708667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789163647259638:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25940844ns) {Configure:104(25936669ns) {ProcessConfigured:135(5569296ns) {StartInitialHandshake:157(5567578ns) {DropHandshakes:470(735ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(99ns)} GenerateSessionVirtualId:988(444ns) PrepareNewSessionHandshake:477(1850ns) {SwitchToState:179(609ns)}}}}}
2025-11-28T14:52:08.733362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789165937032170:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62480
2025-11-28T14:52:08.741872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789164602451667:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.734223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789161833091687:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.747141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789163647259652:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47637
2025-11-28T14:52:08.737014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789161833091691:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1315
2025-11-28T14:52:08.738645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789161833091609:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.741947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789161833091690:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1315
2025-11-28T14:52:08.742071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789161833091689:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1315
2025-11-28T14:52:08.750733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789161833091691:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.751021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789161833091689:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.749581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789165937032170:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.736909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789164152931086:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1132
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.758273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789165937032160:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.758998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789164602451666:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.747807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789164152931086:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.766784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789163647259652:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.766940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789164602451667:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.769369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789164602451589:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.792462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789164519438679:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49761
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.793791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789161833091696:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1315
2025-11-28T14:52:08.758330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789165937032160:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.758331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789165937032158:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.793875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789161833091682:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:08.747977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789164152931089:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1132
2025-11-28T14:52:08.778019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789163647259575:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.748074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789164152931010:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.748520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789164152931091:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1132
2025-11-28T14:52:08.793910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789161833091682:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:08.758359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789165937032158:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:08.808471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789163647259653:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47637
2025-11-28T14:52:08.751003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789164152931091:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.796590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789161833091683:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.808480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789164602451669:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
2025-11-28T14:52:08.759232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789165937032172:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62480
2025-11-28T14:52:08.800601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789164152931093:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1132
2025-11-28T14:52:08.796641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789161833091683:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.777455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789165937032160:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:08.803830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789161833091693:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1315
2025-11-28T14:52:08.778040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789165937032171:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62480
2025-11-28T14:52:08.808972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789161833091682:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.816260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789164152931095:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1132
2025-11-28T14:52:08.796700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789165937032158:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.825851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789164602451667:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:08.813211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789161833091687:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:08.800672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789165937032091:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.826869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789164519438680:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49761
2025-11-28T14:52:08.813540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789161833091674:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.825998Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789164602451589:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8965) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789164602451667:2116] held: no
2025-11-28T14:52:08.826421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789163647259655:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47637
2025-11-28T14:52:08.826913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789164152931086:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:08.830619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789164152931010:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.830698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:08.830758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789164152931010:2048] [node 1] ICP22 created new session: [3:7577789164152931096:2048]
2025-11-28T14:52:08.816047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:08.800750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.826747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789164602451589:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.827091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789163647259656:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47637
2025-11-28T14:52:08.800813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789165937032091:2048] [node 2] ICP22 created new session: [1:7577789165937032176:2048]
2025-11-28T14:52:08.808456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789165937032092:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.831436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789164152931096:2048] [node 1] ICS09 handshake done sender: [3:7577789164152931086:2115] self: [3:7577789164152931092:0] peer: [1:7577789165937032157:0] socket: 370 qp: -1
2025-11-28T14:52:08.808527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:08.832447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789164152931096:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:08.808650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789165937032176:2048] [node 2] ICS09 handshake done sender: [1:7577789165937032160:2111] self: [1:7577789165937032159:0] peer: [2:7577789161833091692:0] socket: 365 qp: -1
2025-11-28T14:52:08.832727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:08.808669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789165937032092:2049] [node 3] ICP22 created new session: [1:7577789165937032178:2049]
2025-11-28T14:52:08.809048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789165937032176:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:08.816110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789161833091674:2109] [node 6] ICP22 created new session: [2:7577789161833091698:2109]
2025-11-28T14:52:08.833042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789163647259652:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:08.817346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789161833091698:2109] [node 6] ICS09 handshake done sender: [2:7577789161833091682:2112] self: [2:7577789161833091678:0] peer: [6:7577789164602451668:0] socket: 378 qp: -1
2025-11-28T14:52:08.826790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:08.809090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:08.833373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789164519438681:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49761
2025-11-28T14:52:08.809157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789165937032178:2049] [node 3] ICS09 handshake done sender: [1:7577789165937032158:2110] self: [1:7577789165937032157:0] peer: [3:7577789164152931092:0] socket: 363 qp: -1
2025-11-28T14:52:08.812753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789165937032178:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:08.834665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789163647259575:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8965) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789163647259652:2115] held: no
2025-11-28T14:52:08.826849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789164602451589:2049] [node 2] ICP22 created new session: [6:7577789164602451670:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.840642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789164152931098:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1132
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.817396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789161833091698:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:08.812811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:08.827792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789165937032091:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(25325737ns) {HandleHandshakeStatus:329(25307916ns) {DropIncomingHandshake:442(639ns) DropOutgoingHandshake:457(798ns) SwitchToState:179(957ns) ProcessPendingSessionEvents:495(12542083ns) {ForwardSessionEventToSession:612(2957674ns) ForwardSessionEventToSession:612(8251784ns) ForwardSessionEventToSession:612(523722ns) ForwardSessionEventToSession:612(15350ns) ForwardSessionEventToSession:612(352936ns) ForwardSessionEventToSession:612(3571ns) ForwardSessionEventToSession:612(248397ns) ForwardSessionEventToSession:612(4147ns) ForwardSessionEventToSession:612(3081ns) ForwardSessionEventToSession:612(137318ns)}}}
2025-11-28T14:52:08.828567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789164602451670:2049] [node 2] ICS09 handshake done sender: [6:7577789164602451667:2116] self: [6:7577789164602451668:0] peer: [2:7577789161833091678:0] socket: 380 qp: -1
2025-11-28T14:52:08.835710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789164519438681:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.817441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:08.835071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789163647259575:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.830465Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789165937032092:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(22241756ns) {HandleHandshakeStatus:329(22238374ns) {DropIncomingHandshake:442(2065ns) DropOutgoingHandshake:457(1041ns) SwitchToState:179(611ns) ProcessPendingSessionEvents:495(4640200ns) {ForwardSessionEventToSession:612(3726237ns) ForwardSessionEventToSession:612(4441ns) ForwardSessionEventToSession:612(288764ns) ForwardSessionEventToSession:612(235836ns) ForwardSessionEventToSession:612(167192ns) ForwardSessionEventToSession:612(4642ns) ForwardSessionEventToSession:612(3302ns) ForwardSessionEventToSession:612(181767ns) ForwardSessionEventToSession:612(4835ns) ForwardSessionEventToSession:612(3455ns)}}}
2025-11-28T14:52:08.831130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789164602451670:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:08.831404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:08.841044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789164602451673:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
2025-11-28T14:52:08.835588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:08.818118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789161833091609:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.836424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789164519438599:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.839374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789163647259575:2049] [node 2] ICP22 created new session: [4:7577789163647259657:2049]
2025-11-28T14:52:08.840306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789163647259657:2049] [node 2] ICS09 handshake done sender: [4:7577789163647259652:2115] self: [4:7577789163647259654:0] peer: [2:7577789161833091681:0] socket: 384 qp: -1
2025-11-28T14:52:08.840547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789163647259657:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.818166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:08.840598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:08.847202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789164519438683:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49761
2025-11-28T14:52:08.819758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789161833091609:2048] [node 1] ICP22 created new session: [2:7577789161833091701:2048]
2025-11-28T14:52:08.823087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789161833091701:2048] [node 1] ICS09 handshake done sender: [2:7577789161833091687:2115] self: [2:7577789161833091692:0] peer: [1:7577789165937032159:0] socket: 372 qp: -1
2025-11-28T14:52:08.823132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789161833091701:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:08.823173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:08.825804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789161833091683:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:08.826958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789161833091673:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.827124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:08.827181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789161833091673:2107] [node 4] ICP22 created new session: [2:7577789161833091705:2107]
2025-11-28T14:52:08.827231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789161833091705:2107] [node 4] ICS09 handshake done sender: [2:7577789161833091683:2113] self: [2:7577789161833091681:0] peer: [4:7577789163647259654:0] socket: 376 qp: -1
2025-11-28T14:52:08.827990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789161833091705:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:08.828050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:08.842705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789161833091684:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.842738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789161833091684:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:08.844809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789161833091684:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:08.845219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789161833091672:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.847209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789161833091680:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:08.847233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:08.847454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789161833091672:2108] [node 5] ICP22 created new session: [2:7577789161833091708:2108]
2025-11-28T14:52:08.849075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789161833091708:2108] [node 5] ICS09 handshake done sender: [2:7577789161833091684:2114] self: [2:7577789161833091679:0] peer: [5:7577789164519438682:0] socket: 414 qp: -1
2025-11-28T14:52:08.849143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789161833091708:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:08.849202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:08.849306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789164152931089:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.849858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789164152931011:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.849977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789164519438681:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:08.856417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789161833091680:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.856596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789164519438599:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.856702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:08.856762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789164519438599:2049] [node 2] ICP22 created new session: [5:7577789164519438684:2049]
2025-11-28T14:52:08.856817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789164519438684:2049] [node 2] ICS09 handshake done sender: [5:7577789164519438681:2119] self: [5:7577789164519438682:0] peer: [2:7577789161833091679:0] socket: 415 qp: -1
2025-11-28T14:52:08.856875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789164519438684:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:08.856959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.857211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789161833091680:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:08.857309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789164152931100:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1132
2025-11-28T14:52:08.857518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789161833091610:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.857622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:08.857674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789161833091610:2049] [node 3] ICP22 created new session: [2:7577789161833091711:2049]
2025-11-28T14:52:08.857652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789164152931089:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:08.857835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789161833091711:2049] [node 3] ICS09 handshake done sender: [2:7577789161833091680:2111] self: [2:7577789161833091676:0] peer: [3:7577789164152931099:0] socket: 381 qp: -1
2025-11-28T14:52:08.857949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789161833091711:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:08.857954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789164152931011:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.857994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:08.858063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:08.858123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789164152931011:2049] [node 2] ICP22 created new session: [3:7577789164152931101:2049]
2025-11-28T14:52:08.858166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789164152931101:2049] [node 2] ICS09 handshake done sender: [3:7577789164152931089:2116] self: [3:7577789164152931099:0] peer: [2:7577789161833091676:0] socket: 388 qp: -1
2025-11-28T14:52:08.858188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789164152931101:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:08.858225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:08.859881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789164519438669:2113] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:08.877240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789161833091690:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.877481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789161833091672:2108] [node 5] ICP18 handshake (actor [2:7577789161833091690:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.878542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789164602451674:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
2025-11-28T14:52:08.881379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789164152931081:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:08.911051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789165937032172:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.911222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789165937032092:2049] [node 3] ICP18 handshake (actor [1:7577789165937032172:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.912265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789164519438687:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49761
2025-11-28T14:52:08.913317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789165937032166:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:08.953040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789164602451666:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.953246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789164602451588:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:08.953699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789165937032166:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.956098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789165937032166:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.956447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789165937032156:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:08.956822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:08.957110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789165937032156:2109] [node 6] ICP22 created new session: [1:7577789165937032183:2109]
2025-11-28T14:52:08.957157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789165937032183:2109] [node 6] ICS09 handshake done sender: [1:7577789165937032166:2113] self: [1:7577789165937032164:0] peer: [6:7577789164602451675:0] socket: 369 qp: -1
2025-11-28T14:52:08.957317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789165937032183:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:08.957359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:08.956652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789164602451676:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
2025-11-28T14:52:08.962627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789164602451666:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:08.963835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789164602451588:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789164602451666:2115] held: no
2025-11-28T14:52:08.963952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789164602451588:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:08.964008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:08.964062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789164602451588:2048] [node 1] ICP22 created new session: [6:7577789164602451677:2048]
2025-11-28T14:52:08.964104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789164602451677:2048] [node 1] ICS09 handshake done sender: [6:7577789164602451666:2115] self: [6:7577789164602451675:0] peer: [1:7577789165937032164:0] socket: 371 qp: -1
2025-11-28T14:52:08.964145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789164602451677:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:08.964180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:08.968903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789163647259660:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47637
2025-11-28T14:52:08.969642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789163647259660:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:08.990761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789164519438666:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.013457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789165937032171:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.025446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789163647259647:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.026463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789164519438679:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.026661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789164519438664:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.027133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789163647259647:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.034020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789163647259647:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.040406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789163647259638:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.041015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:09.041085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789163647259638:2108] [node 5] ICP22 created new session: [4:7577789167942226960:2108]
2025-11-28T14:52:09.037294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789168814405985:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49761
2025-11-28T14:52:09.042785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789167942226960:2108] [node 5] ICS09 handshake done sender: [4:7577789163647259647:2113] self: [4:7577789163647259640:0] peer: [5:7577789168814405984:0] socket: 390 qp: -1
2025-11-28T14:52:09.043828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789167942226960:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:09.043877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:09.043363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789164519438679:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.043783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789164519438664:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.043858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:09.043912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789164519438664:2108] [node 4] ICP22 created new session: [5:7577789168814405986:2108]
2025-11-28T14:52:09.044008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789168814405986:2108] [node 4] ICS09 handshake done sender: [5:7577789164519438679:2117] self: [5:7577789168814405984:0] peer: [4:7577789163647259640:0] socket: 392 qp: -1
2025-11-28T14:52:09.046346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789168814405986:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:09.046411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:09.049205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789170231999482:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62480
2025-11-28T14:52:09.058657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789164519438670:2114] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.091099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789164152931093:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.096571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789164152931077:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.106221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789161833091693:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.106535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789161833091610:2049] [node 3] ICP18 handshake (actor [2:7577789161833091693:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.108462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789168897418976:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
2025-11-28T14:52:09.116484Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789164152931084:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.117377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789163647259653:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.117549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789163647259576:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.117815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789164152931084:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.118396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789164152931084:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:09.118347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789167942226963:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47637
2025-11-28T14:52:09.118804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789163647259653:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:09.119229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789163647259576:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.120243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:09.120446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789163647259576:2050] [node 3] ICP22 created new session: [4:7577789167942226964:2050]
2025-11-28T14:52:09.118716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789164152931075:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.118797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:09.118850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789164152931075:2107] [node 4] ICP22 created new session: [3:7577789168447898403:2107]
2025-11-28T14:52:09.118893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789168447898403:2107] [node 4] ICS09 handshake done sender: [3:7577789164152931084:2113] self: [3:7577789164152931079:0] peer: [4:7577789167942226962:0] socket: 397 qp: -1
2025-11-28T14:52:09.118925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789168447898403:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:09.119011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:09.120745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789167942226964:2050] [node 3] ICS09 handshake done sender: [4:7577789163647259653:2116] self: [4:7577789167942226962:0] peer: [3:7577789164152931079:0] socket: 399 qp: -1
2025-11-28T14:52:09.120776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789167942226964:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:09.120813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:09.136977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789163647259576:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17690666ns) {HandleHandshakeStatus:329(17687820ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(5954ns) SwitchToState:179(291ns) ProcessPendingSessionEvents:495(13000588ns) {ForwardSessionEventToSession:612(12916928ns) ForwardSessionEventToSession:612(20333ns) ForwardSessionEventToSession:612(5371ns) ForwardSessionEventToSession:612(8727ns) ForwardSessionEventToSession:612(3743ns) ForwardSessionEventToSession:612(7823ns) ForwardSessionEventToSession:612(3929ns) ForwardSessionEventToSession:612(4196ns) ForwardSessionEventToSession:612(11919ns) ForwardSessionEventToSession:612(3450ns)}}}
2025-11-28T14:52:09.143722Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789165937032167:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.146496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789165937032167:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.144995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789163647259655:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.145196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789163647259574:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.152733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789165937032167:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:09.154535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789165937032154:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.154616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:09.154540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789167942226967:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47637
2025-11-28T14:52:09.154677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789165937032154:2107] [node 4] ICP22 created new session: [1:7577789170231999484:2107]
2025-11-28T14:52:09.155076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789163647259655:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:09.154905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789170231999484:2107] [node 4] ICS09 handshake done sender: [1:7577789165937032167:2114] self: [1:7577789165937032165:0] peer: [4:7577789167942226966:0] socket: 408 qp: -1
2025-11-28T14:52:09.155497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789163647259574:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.156217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:09.156282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789163647259574:2048] [node 1] ICP22 created new session: [4:7577789167942226968:2048]
2025-11-28T14:52:09.156341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789167942226968:2048] [node 1] ICS09 handshake done sender: [4:7577789163647259655:2117] self: [4:7577789167942226966:0] peer: [1:7577789165937032165:0] socket: 409 qp: -1
2025-11-28T14:52:09.156404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789167942226968:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:09.156450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:09.154970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789170231999484:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:09.155026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:09.160770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789165937032163:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.164687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789164519438680:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.164991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789164519438598:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.165363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789165937032163:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.169893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789165937032163:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:09.170126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789168814405989:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49761
2025-11-28T14:52:09.170788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789164519438680:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:09.170497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789165937032155:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.171362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:09.171426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789165937032155:2108] [node 5] ICP22 created new session: [1:7577789170231999487:2108]
2025-11-28T14:52:09.171484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789170231999487:2108] [node 5] ICS09 handshake done sender: [1:7577789165937032163:2112] self: [1:7577789165937032161:0] peer: [5:7577789168814405988:0] socket: 410 qp: -1
2025-11-28T14:52:09.171525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789170231999487:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:09.171570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:09.171326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789164519438598:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.171386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:09.171442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789164519438598:2048] [node 1] ICP22 created new session: [5:7577789168814405990:2048]
2025-11-28T14:52:09.172295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789168814405990:2048] [node 1] ICS09 handshake done sender: [5:7577789164519438680:2118] self: [5:7577789168814405988:0] peer: [1:7577789165937032161:0] socket: 411 qp: -1
2025-11-28T14:52:09.172344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789168814405990:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:09.172399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:09.178345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789164602451658:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.190395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789164152931098:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.192391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789164602451673:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.192604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789164602451655:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.191324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789163647259648:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.195964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789163647259648:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.201558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789163647259648:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.201873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789163647259637:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.206752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789164519438674:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.202035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:09.202137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789163647259637:2109] [node 6] ICP22 created new session: [4:7577789167942226971:2109]
2025-11-28T14:52:09.207314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789167942226971:2109] [node 6] ICS09 handshake done sender: [4:7577789163647259648:2114] self: [4:7577789163647259642:0] peer: [6:7577789168897418977:0] socket: 386 qp: -1
2025-11-28T14:52:09.207371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789167942226971:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:09.202153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789168897418978:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
2025-11-28T14:52:09.211220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789164602451674:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.221064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789164602451649:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.221841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789164602451655:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789164602451673:2118] held: no
2025-11-28T14:52:09.222172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789164519438674:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.221321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:09.235835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789164152931085:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.222012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789164602451673:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:09.244864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789163647259637:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(42733198ns) {HandleHandshakeStatus:329(42728824ns) {DropIncomingHandshake:442(25304ns) DropOutgoingHandshake:457(598ns) SwitchToState:179(1015ns) ProcessPendingSessionEvents:495(21735799ns) {ForwardSessionEventToSession:612(9249429ns) ForwardSessionEventToSession:612(14218ns) ForwardSessionEventToSession:612(12424725ns) ForwardSessionEventToSession:612(5045ns) ForwardSessionEventToSession:612(3347ns) ForwardSessionEventToSession:612(5402ns) ForwardSessionEventToSession:612(3776ns) ForwardSessionEventToSession:612(5777ns) ForwardSessionEventToSession:612(2804ns) ForwardSessionEventToSession:612(3421ns)}}}
2025-11-28T14:52:09.231052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789164519438674:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:09.245706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789164519438662:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.222297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789164602451655:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.249123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:09.222354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:09.222412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789164602451655:2110] [node 4] ICP22 created new session: [6:7577789168897418980:2110]
2025-11-28T14:52:09.249193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789164519438662:2109] [node 6] ICP22 created new session: [5:7577789168814405993:2109]
2025-11-28T14:52:09.281001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789164152931085:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.281481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789164152931085:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:09.281782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789164152931074:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.281873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:09.282667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789164152931074:2108] [node 5] ICP22 created new session: [3:7577789168447898407:2108]
2025-11-28T14:52:09.282723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789168447898407:2108] [node 5] ICS09 handshake done sender: [3:7577789164152931085:2114] self: [3:7577789164152931083:0] peer: [5:7577789168814405995:0] socket: 379 qp: -1
2025-11-28T14:52:09.282761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789168447898407:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:09.282803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:09.249260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789168814405993:2109] [node 6] ICS09 handshake done sender: [5:7577789164519438674:2116] self: [5:7577789164519438671:0] peer: [6:7577789168897418979:0] socket: 374 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.222462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789168897418980:2110] [node 4] ICS09 handshake done sender: [6:7577789164602451673:2118] self: [6:7577789168897418977:0] peer: [4:7577789163647259642:0] socket: 391 qp: -1
2025-11-28T14:52:09.222990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789168897418980:2110] [node 4] ICS10 traffic start
2025-11-28T14:52:09.231398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:09.232127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789168897418981:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
2025-11-28T14:52:09.246115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789164602451674:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:09.248731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789164602451655:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(26301016ns) {HandleHandshakeStatus:329(26289960ns) {DropIncomingHandshake:442(1295ns) DropOutgoingHandshake:457(423ns) SwitchToState:179(815ns) ProcessPendingSessionEvents:495(937ns)}}
2025-11-28T14:52:09.254064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789164602451649:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.322232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789170231999482:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.254251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:09.254333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789164602451649:2107] [node 5] ICP22 created new session: [6:7577789168897418983:2107]
2025-11-28T14:52:09.254388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789168897418983:2107] [node 5] ICS09 handshake done sender: [6:7577789164602451674:2119] self: [6:7577789168897418979:0] peer: [5:7577789164519438671:0] socket: 420 qp: -1
2025-11-28T14:52:09.254435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789168897418983:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:09.254489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:09.254888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789164602451649:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:09.321184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789163647259645:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.249305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789168814405993:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:09.322447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789165937032154:2107] [node 4] ICP18 handshake (actor [1:7577789170231999482:2118]) is too old
2025-11-28T14:52:09.249364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:09.262854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789164519438687:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.276920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789164519438600:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.316501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789164519438600:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27652) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789164519438687:2121] held: no
2025-11-28T14:52:09.316902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789164519438600:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27824173ns) {HandleHandshakeStatus:395(10945092ns) {LogHandshakeFail:472(23471ns) {UpdateErrorStateLog:545(6383ns)} DropOutgoingHandshake:457(4761ns) ProcessPendingSessionEvents:495(126637ns) {DropSessionEvent:512(30439ns) DropSessionEvent:512(7632ns) DropSessionEvent:512(7882ns) DropSessionEvent:512(6770ns) DropSessionEvent:512(18901ns) DropSessionEvent:512(5093ns) DropSessionEvent:512(6145ns) DropSessionEvent:512(6705ns) DropSessionEvent:512(4570ns) DropSessionEvent:512(6790ns)}}}
2025-11-28T14:52:09.317495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789168814405996:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49761
2025-11-28T14:52:09.323258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789164519438687:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:09.330152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789164519438600:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.351435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:09.351521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789164519438600:2050] [node 3] ICP22 created new session: [5:7577789168814405997:2050]
2025-11-28T14:52:09.351580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789168814405997:2050] [node 3] ICS09 handshake done sender: [5:7577789164519438687:2121] self: [5:7577789168814405995:0] peer: [3:7577789164152931083:0] socket: 394 qp: -1
2025-11-28T14:52:09.351622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789168814405997:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:09.351689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:09.352038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789164519438600:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(21823551ns) {HandleHandshakeStatus:329(21819975ns) {DropIncomingHandshake:442(1581ns) DropOutgoingHandshake:457(642ns) SwitchToState:179(977ns) ProcessPendingSessionEvents:495(1830ns)}}
2025-11-28T14:52:09.355978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789164152931082:2112] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:09.364322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789168897418976:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.364542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789164602451590:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.364946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789164152931082:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.366018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789164152931082:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:09.366514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789164152931073:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.366606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:09.366652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789164152931073:2109] [node 6] ICP22 created new session: [3:7577789168447898410:2109]
2025-11-28T14:52:09.366693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789168447898410:2109] [node 6] ICS09 handshake done sender: [3:7577789164152931082:2112] self: [3:7577789164152931078:0] peer: [6:7577789168897418985:0] socket: 396 qp: -1
2025-11-28T14:52:09.366734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789168447898410:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:09.366766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:09.365746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789168897418986:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51293
2025-11-28T14:52:09.368181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789168897418976:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:09.368406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789164602451590:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.368455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:09.368500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789164602451590:2050] [node 3] ICP22 created new session: [6:7577789168897418987:2050]
2025-11-28T14:52:09.368548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789168897418987:2050] [node 3] ICS09 handshake done sender: [6:7577789168897418976:2121] self: [6:7577789168897418985:0] peer: [3:7577789164152931078:0] socket: 426 qp: -1
2025-11-28T14:52:09.368577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789168897418987:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:09.368630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:09.424545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:09.424783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789161833091701:2048] [node 1] ICS01 socket: 372 reason# EndOfStream
2025-11-28T14:52:09.424838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789161833091609:2048] [node 1] ICP30 unregister session Session# [2:7577789161833091701:2048] VirtualId# [2:7577789161833091692:0]
2025-11-28T14:52:09.424898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789161833091701:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.424984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:09.425028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:09.425493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:09.447378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789164152931096:2048] [node 1] ICS07 socket disconnect 370 reason# EndOfStream
2025-11-28T14:52:09.447447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789164152931096:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.447579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:09.447625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789164152931096:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:09.450303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789164152931010:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4340) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:09.450357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789164152931096:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:09.451680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789164152931010:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:09.451744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789164152931096:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:09.451786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789164152931010:2048] [node 1] ICP30 unregister session Session# [3:7577789164152931096:2048] VirtualId# [3:7577789164152931092:0]
2025-11-28T14:52:09.451840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:09.451870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789164152931010:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.451883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.453330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:09.453736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789167942226968:2048] [node 1] ICS01 socket: 409 reason# EndOfStream
2025-11-28T14:52:09.453779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789163647259574:2048] [node 1] ICP30 unregister session Session# [4:7577789167942226968:2048] VirtualId# [4:7577789167942226966:0]
2025-11-28T14:52:09.453816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789167942226968:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.453924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:09.453968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:09.456598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:09.457017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789164602451677:2048] [node 1] ICS01 socket: 371 reason# EndOfStream
2025-11-28T14:52:09.457075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789164602451588:2048] [node 1] ICP30 unregister session Session# [6:7577789164602451677:2048] VirtualId# [6:7577789164602451675:0]
2025-11-28T14:52:09.457112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789164602451677:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.457236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:09.457287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:09.460568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:09.461289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789164152931010:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:09.461063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789168814405990:2048] [node 1] ICS07 socket disconnect 411 reason# EndOfStream
2025-11-28T14:52:09.461143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789168814405990:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.461330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:09.461406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789168814405990:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:09.472564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789164519438598:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:09.476541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789168814405990:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:09.476628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789164519438598:2048] [node 1] ICP30 unregister session Session# [5:7577789168814405990:2048] VirtualId# [5:7577789168814405988:0]
2025-11-28T14:52:09.476710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:09.476754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789164519438598:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.476773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.487527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789164519438598:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:09.496133Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:09.496396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789164152931101:2049] [node 2] ICS07 socket disconnect 388 reason# EndOfStream
2025-11-28T14:52:09.496426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789164152931101:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.496526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:09.496567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789164152931101:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:09.501140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789164152931011:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8965) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:09.501189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789164152931101:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:09.502302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789164152931011:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8965) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:09.502392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789164152931101:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:09.502450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789164152931011:2049] [node 2] ICP30 unregister session Session# [3:7577789164152931101:2049] VirtualId# [3:7577789164152931099:0]
2025-11-28T14:52:09.502522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:09.502579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789164152931011:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8965) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.502631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8965) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.505843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:09.511004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:09.516523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789164602451670:2049] [node 2] ICS01 socket: 380 reason# EndOfStream
2025-11-28T14:52:09.514135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789164152931011:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:09.523010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789164602451589:2049] [node 2] ICP30 unregister session Session# [6:7577789164602451670:2049] VirtualId# [6:7577789164602451668:0]
2025-11-28T14:52:09.523074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789164602451670:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.523214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:09.523265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:09.512484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:09.506164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789163647259657:2049] [node 2] ICS01 socket: 384 reason# EndOfStream
2025-11-28T14:52:09.517555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789163647259575:2049] [node 2] ICP30 unregister session Session# [4:7577789163647259657:2049] VirtualId# [4:7577789163647259654:0]
2025-11-28T14:52:09.517654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789163647259657:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.517803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:09.517861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:09.512740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789164519438684:2049] [node 2] ICS07 socket disconnect 415 reason# EndOfStream
2025-11-28T14:52:09.512773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789164519438684:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.512927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:09.517197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789164519438684:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:09.532876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789164519438599:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8965) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:09.535911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789164519438684:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:09.535974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789164519438599:2049] [node 2] ICP30 unregister session Session# [5:7577789164519438684:2049] VirtualId# [5:7577789164519438682:0]
2025-11-28T14:52:09.536033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:09.536071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789164519438599:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8965) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.536087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8965) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.539750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:09.540011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789167942226964:2050] [node 3] ICS07 socket disconnect 399 reason# EndOfStream
2025-11-28T14:52:09.540045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789167942226964:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.540163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:09.540207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789167942226964:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:09.540719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:09.540952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789168814405997:2050] [node 3] ICS01 socket: 394 reason# EndOfStream
2025-11-28T14:52:09.544420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:09.540993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789164519438600:2050] [node 3] ICP30 unregister session Session# [5:7577789168814405997:2050] VirtualId# [5:7577789168814405995:0]
2025-11-28T14:52:09.544735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789168897418987:2050] [node 3] ICS01 socket: 426 reason# EndOfStream
2025-11-28T14:52:09.541030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789168814405997:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.544779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789164602451590:2050] [node 3] ICP30 unregister session Session# [6:7577789168897418987:2050] VirtualId# [6:7577789168897418985:0]
2025-11-28T14:52:09.543868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789163647259576:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27652) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:09.544717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789167942226964:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:09.544766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789163647259576:2050] [node 3] ICP30 unregister session Session# [4:7577789167942226964:2050] VirtualId# [4:7577789167942226962:0]
2025-11-28T14:52:09.544815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789168897418987:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.544938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:09.541115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:09.544829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:09.544894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789163647259576:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27652) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.544913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27652) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.544993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:09.541161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:09.546959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789164519438599:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:09.594997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:09.595268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789168814405986:2108] [node 4] ICS07 socket disconnect 392 reason# EndOfStream
2025-11-28T14:52:09.595301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789168814405986:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.595422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:09.595487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789168814405986:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:09.596696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789164519438664:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5562) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:09.596986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789168814405986:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:09.597045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789164519438664:2108] [node 4] ICP30 unregister session Session# [5:7577789168814405986:2108] VirtualId# [5:7577789168814405984:0]
2025-11-28T14:52:09.597094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:09.597132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789164519438664:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5562) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.596124Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:09.597150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5562) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:09.596393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789168897418980:2110] [node 4] ICS01 socket: 391 reason# EndOfStream
2025-11-28T14:52:09.596445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789164602451655:2110] [node 4] ICP30 unregister session Session# [6:7577789168897418980:2110] VirtualId# [6:7577789168897418977:0]
2025-11-28T14:52:09.596485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789168897418980:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.596587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:09.596648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:09.600885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789164519438664:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:09.608448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:09.608714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789168897418983:2107] [node 5] ICS01 socket: 420 reason# EndOfStream
2025-11-28T14:52:09.608768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789164602451649:2107] [node 5] ICP30 unregister session Session# [6:7577789168897418983:2107] VirtualId# [6:7577789168897418979:0]
2025-11-28T14:52:09.608802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789168897418983:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:09.608880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:09.608994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:09.739195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789169827978309:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:09.739243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789169827978310:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:09.765037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789166997100126:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:09.765094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789166997100127:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:09.789623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789166523273168:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:09.789655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789166523273169:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:09.814439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789167230278295:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:09.814497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789167230278296:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:09.814578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789167230278297:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:09.837810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789170310039414:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:09.837844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789170310039415:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:09.837893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789170310039416:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:09.858851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789167510063897:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:09.858891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789167510063898:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:09.858921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789167510063899:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:09.906736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789167510063956:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:09.941439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789169827978372:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:09.960282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789169827978373:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:09.960966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789166523273232:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:09.961051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789170310039477:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:09.961168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789167230278359:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:09.961367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789166523273233:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:09.961443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789170310039478:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:09.961868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789166523273234:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:09.961956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789167230278360:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:09.960876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789166997100190:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:09.961366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789166997100191:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:09.961887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789166997100192:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:09.963074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789167510063956:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(55976613ns) {Configure:104(55971608ns) {ProcessConfigured:135(105302ns) {StartInitialHandshake:157(102518ns) {DropHandshakes:470(7067ns) {DropIncomingHandshake:442(996ns) DropOutgoingHandshake:457(1555ns)} GenerateSessionVirtualId:988(1767ns) PrepareNewSessionHandshake:477(1028ns) {SwitchToState:179(316ns)}}}}}
2025-11-28T14:52:09.963551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789167510063963:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:09.964117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789169827978376:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.970666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789166997100203:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1348
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.973198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789166523273245:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21346
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.973405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789166523273246:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21346
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.975999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789167230278371:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7964
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.977442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789167230278372:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7964
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.978046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789170310039492:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5198
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.979957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789170310039492:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.980164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789169827978385:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6980
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.980426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789169827978386:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6980
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.980575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789169827978387:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6980
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.981327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789166997100207:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1348
2025-11-28T14:52:09.980992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789169827978388:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6980
2025-11-28T14:52:09.982599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789166997100203:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.982638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789166523273245:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.982752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789166997100207:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.982785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978385:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.982917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978387:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.982976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789166523273250:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21346
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.983006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978388:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.983087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789166997100208:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1348
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.983108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978386:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:09.983179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789166523273246:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.983199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789169827978389:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6980
2025-11-28T14:52:09.983233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789166997100209:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1348
2025-11-28T14:52:09.983295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789166523273251:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21346
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.983553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789166997100210:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1348
2025-11-28T14:52:09.983616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789166523273169:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.983754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789166997100208:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.983971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789166997100198:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.983997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789166997100198:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:09.984781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789166997100198:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.985444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789166523273254:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21346
2025-11-28T14:52:09.985625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789166997100127:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.985731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:09.985780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789166997100127:2049] [node 3] ICP22 created new session: [2:7577789166997100215:2049]
2025-11-28T14:52:09.985854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789166997100215:2049] [node 3] ICS09 handshake done sender: [2:7577789166997100198:2113] self: [2:7577789166997100197:0] peer: [3:7577789166523273253:0] socket: 373 qp: -1
2025-11-28T14:52:09.985893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789166997100215:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:09.985926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:09.985980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789167230278371:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.986489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789166523273246:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:09.986835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789166523273169:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.986902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:09.986955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789166523273169:2049] [node 2] ICP22 created new session: [3:7577789166523273255:2049]
2025-11-28T14:52:09.987001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789166523273255:2049] [node 2] ICS09 handshake done sender: [3:7577789166523273246:2117] self: [3:7577789166523273253:0] peer: [2:7577789166997100197:0] socket: 381 qp: -1
2025-11-28T14:52:09.987030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789166523273255:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:09.987059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789166997100209:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.987065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:09.987093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789167230278372:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.987296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789167230278296:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.987478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789167230278376:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7964
2025-11-28T14:52:09.987728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789170310039494:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5198
2025-11-28T14:52:09.987781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789166997100194:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.987677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789167230278297:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.987812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789166997100194:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:09.990014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789166523273250:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.990241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789166523273240:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.990271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789166523273240:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:09.990257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789170310039494:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.990396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789166997100194:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:09.990495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789170310039415:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.991380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789166997100190:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.991524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:09.991556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789166997100190:2107] [node 4] ICP22 created new session: [2:7577789166997100220:2107]
2025-11-28T14:52:09.991544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789167230278379:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7964
2025-11-28T14:52:09.991538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789166523273169:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:09.991593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789166997100220:2107] [node 4] ICS09 handshake done sender: [2:7577789166997100194:2111] self: [2:7577789166997100193:0] peer: [4:7577789167230278377:0] socket: 371 qp: -1
2025-11-28T14:52:09.991621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789166997100220:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:09.991645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:09.991725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978389:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.992125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789166997100200:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.992145Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789166997100200:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:09.992293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789167230278371:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:09.992358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789166523273251:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.992741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789166523273240:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:09.992894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789167230278376:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.993032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789166997100210:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.992955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789167230278296:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.993037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:09.993091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789167230278296:2049] [node 2] ICP22 created new session: [4:7577789167230278381:2049]
2025-11-28T14:52:09.993134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789167230278381:2049] [node 2] ICS09 handshake done sender: [4:7577789167230278371:2116] self: [4:7577789167230278377:0] peer: [2:7577789166997100193:0] socket: 384 qp: -1
2025-11-28T14:52:09.993143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789166523273232:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.993163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789167230278381:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:09.993200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:09.993196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:09.993250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789166523273232:2107] [node 4] ICP22 created new session: [3:7577789166523273260:2107]
2025-11-28T14:52:09.993294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789166523273260:2107] [node 4] ICS09 handshake done sender: [3:7577789166523273240:2113] self: [3:7577789166523273239:0] peer: [4:7577789167230278378:0] socket: 386 qp: -1
2025-11-28T14:52:09.993325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789166523273260:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:09.993360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:09.993429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789166997100126:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.993818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789166997100200:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.993918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978375:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.993951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789169827978375:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:09.993871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789167510063975:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:09.994533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789166523273168:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.994551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789170310039496:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5198
2025-11-28T14:52:09.994788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789170310039497:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5198
2025-11-28T14:52:09.995034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789169827978375:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:09.995095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789166997100191:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.995150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:09.995287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789166997100191:2108] [node 5] ICP22 created new session: [2:7577789166997100225:2108]
No RDMA context found for GID: 2025-11-28T14:52:09.995503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789166997100225:2108] [node 5] ICS09 handshake done sender: [2:7577789166997100200:2114] self: [2:7577789166997100199:0] peer: [5:7577789170310039495:0] socket: 398 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:09.995626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789166997100225:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:09.995768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:09.995972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789167230278380:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7964
2025-11-28T14:52:09.996001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789170310039494:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:09.996025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789169827978309:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:09.996085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:09.996504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789170310039415:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:09.996125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789169827978309:2048] [node 2] ICP22 created new session: [1:7577789169827978399:2048]
2025-11-28T14:52:09.996562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:09.996648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789170310039415:2049] [node 2] ICP22 created new session: [5:7577789170310039498:2049]
2025-11-28T14:52:09.996401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789169827978399:2048] [node 2] ICS09 handshake done sender: [1:7577789169827978375:2110] self: [1:7577789169827978374:0] peer: [2:7577789166997100224:0] socket: 401 qp: -1
2025-11-28T14:52:09.996730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789170310039498:2049] [node 2] ICS09 handshake done sender: [5:7577789170310039494:2116] self: [5:7577789170310039495:0] peer: [2:7577789166997100199:0] socket: 410 qp: -1
2025-11-28T14:52:09.996763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789170310039498:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:09.996474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789169827978399:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:09.996796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:09.996806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789167230278296:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:09.996513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:09.997944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978378:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:09.997979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789169827978378:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:09.998785Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789167230278297:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789167230278372:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:09.999211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789167230278295:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:09.999285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789167230278372:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:09.999219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789167510063976:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:10.000152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789169827978378:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:10.000804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789167510063976:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.000846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789169827978310:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.000841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789167230278297:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.000975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789167510063899:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.000965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:10.000909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:10.001070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789167230278297:2050] [node 3] ICP22 created new session: [4:7577789167230278386:2050]
No RDMA context found for GID: 2025-11-28T14:52:10.000957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789169827978310:2049] [node 3] ICP22 created new session: [1:7577789169827978403:2049]
::ffff:127.1.0.0
2025-11-28T14:52:10.001108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789167230278386:2050] [node 3] ICS09 handshake done sender: [4:7577789167230278372:2117] self: [4:7577789167230278378:0] peer: [3:7577789166523273239:0] socket: 387 qp: -1
2025-11-28T14:52:10.001035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789169827978403:2049] [node 3] ICS09 handshake done sender: [1:7577789169827978378:2111] self: [1:7577789169827978377:0] peer: [3:7577789166523273262:0] socket: 402 qp: -1
2025-11-28T14:52:10.001064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789169827978403:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:10.001147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789167230278386:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:10.001178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789170310039500:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5198
2025-11-28T14:52:10.001095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:10.001184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:10.001443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789170310039496:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.001445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789166523273244:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.001480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789166523273244:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.002037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789166523273264:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21346
2025-11-28T14:52:10.002096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789170310039416:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.002271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789166997100126:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789166997100210:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.002354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789167510063975:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.002333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978380:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.002566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789166997100227:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1348
2025-11-28T14:52:10.002360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789169827978380:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.002785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789167510063898:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.003703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789166523273242:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.003766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789166997100210:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:10.003745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789166523273242:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.004836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789166997100126:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.004779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789171805031274:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:10.004888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.004838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789174605006799:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5198
2025-11-28T14:52:10.004936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789166997100126:2048] [node 1] ICP22 created new session: [2:7577789171292067524:2048]
2025-11-28T14:52:10.005034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789171292067524:2048] [node 1] ICS09 handshake done sender: [2:7577789166997100210:2120] self: [2:7577789166997100224:0] peer: [1:7577789169827978374:0] socket: 404 qp: -1
2025-11-28T14:52:10.005083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789171292067524:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.005122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.005276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789171805031276:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:10.005993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789166997100202:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.006023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789166523273251:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:10.006015Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789166997100202:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.007247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789166523273168:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789166523273251:2119] held: no
2025-11-28T14:52:10.007354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789166523273168:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.007336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789171525245684:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7964
2025-11-28T14:52:10.007398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.007436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789166523273168:2048] [node 1] ICP22 created new session: [3:7577789170818240563:2048]
2025-11-28T14:52:10.007476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789170818240563:2048] [node 1] ICS09 handshake done sender: [3:7577789166523273251:2119] self: [3:7577789166523273262:0] peer: [1:7577789169827978377:0] socket: 403 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.007501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789170818240563:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.007530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:10.007623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789171805031277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:10.007693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789166997100202:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:10.007729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789169827978380:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:10.007717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789170310039500:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.007977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789171525245684:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.007841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789174605006799:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.008203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789170310039414:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.008386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789170310039477:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.008473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789171805031274:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.008520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789171525245685:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7964
2025-11-28T14:52:10.008600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789171805031278:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:10.008650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789169827978372:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.008634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789166523273244:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:10.008384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789166997100192:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.008721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:10.008454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.008766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789169827978372:2107] [node 4] ICP22 created new session: [1:7577789174122945703:2107]
2025-11-28T14:52:10.008503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789166997100192:2109] [node 6] ICP22 created new session: [2:7577789171292067527:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.008540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789171292067527:2109] [node 6] ICS09 handshake done sender: [2:7577789166997100202:2115] self: [2:7577789166997100201:0] peer: [6:7577789171805031275:0] socket: 415 qp: -1
2025-11-28T14:52:10.008825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789174122945703:2107] [node 4] ICS09 handshake done sender: [1:7577789169827978380:2112] self: [1:7577789169827978379:0] peer: [4:7577789167230278383:0] socket: 405 qp: -1
2025-11-28T14:52:10.008580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789171292067527:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:10.008928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789174122945703:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:10.008964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:10.009246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789174605006802:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5198
2025-11-28T14:52:10.008624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.009356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789166523273234:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.009613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.009664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789166523273234:2110] [node 6] ICP22 created new session: [3:7577789170818240565:2110]
2025-11-28T14:52:10.009700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789170818240565:2110] [node 6] ICS09 handshake done sender: [3:7577789166523273244:2115] self: [3:7577789166523273243:0] peer: [6:7577789167510063977:0] socket: 370 qp: -1
2025-11-28T14:52:10.009734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789170818240565:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:10.009754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.009890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789167230278368:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.009928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789167230278368:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.010125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978382:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.010115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789167510063963:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.010157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789169827978382:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.010233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789167510063975:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:10.011205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789167230278376:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:10.011439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789166523273242:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:10.011776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789166523273233:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.011821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:10.011854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789166523273233:2109] [node 5] ICP22 created new session: [3:7577789170818240567:2109]
2025-11-28T14:52:10.011889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789170818240567:2109] [node 5] ICS09 handshake done sender: [3:7577789166523273242:2114] self: [3:7577789166523273241:0] peer: [5:7577789174605006798:0] socket: 414 qp: -1
2025-11-28T14:52:10.011912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789170818240567:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:10.012019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:10.013876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789167230278295:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.013990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.014043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789167230278295:2048] [node 1] ICP22 created new session: [4:7577789171525245688:2048]
2025-11-28T14:52:10.013985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789170310039496:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:10.014008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789167510063976:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:10.014092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789171525245688:2048] [node 1] ICS09 handshake done sender: [4:7577789167230278376:2118] self: [4:7577789167230278383:0] peer: [1:7577789169827978379:0] socket: 409 qp: -1
2025-11-28T14:52:10.014129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789171525245688:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.014180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:10.014446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789167510063898:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.014147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789169827978382:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:10.014784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789169827978373:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.014761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:10.014849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789170310039416:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.014809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789167510063898:2049] [node 2] ICP22 created new session: [6:7577789171805031280:2049]
2025-11-28T14:52:10.014865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:10.014896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:10.014860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789171805031280:2049] [node 2] ICS09 handshake done sender: [6:7577789167510063975:2115] self: [6:7577789171805031275:0] peer: [2:7577789166997100201:0] socket: 418 qp: -1
2025-11-28T14:52:10.014902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789167230278295:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:10.014912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789169827978373:2108] [node 5] ICP22 created new session: [1:7577789174122945706:2108]
2025-11-28T14:52:10.014885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789171805031280:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:10.014933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789170310039416:2050] [node 3] ICP22 created new session: [5:7577789174605006805:2050]
2025-11-28T14:52:10.014958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789174122945706:2108] [node 5] ICS09 handshake done sender: [1:7577789169827978382:2113] self: [1:7577789169827978381:0] peer: [5:7577789174605006800:0] socket: 400 qp: -1
2025-11-28T14:52:10.014915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:10.014990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789174605006805:2050] [node 3] ICS09 handshake done sender: [5:7577789170310039496:2117] self: [5:7577789174605006798:0] peer: [3:7577789166523273241:0] socket: 416 qp: -1
2025-11-28T14:52:10.015003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789174122945706:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:10.015020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789174605006805:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:10.015032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:10.015051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:10.015226Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789167510063898:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:10.015365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789167230278370:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.015393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789167230278370:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.015760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789167510063899:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.015799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:10.015839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789167510063899:2050] [node 3] ICP22 created new session: [6:7577789171805031282:2050]
2025-11-28T14:52:10.015879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789171805031282:2050] [node 3] ICS09 handshake done sender: [6:7577789167510063976:2116] self: [6:7577789167510063977:0] peer: [3:7577789166523273243:0] socket: 395 qp: -1
2025-11-28T14:52:10.015904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789171805031282:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:10.015933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:10.016126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789167230278368:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:10.016862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789167230278359:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.016961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:10.017006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789167230278359:2108] [node 5] ICP22 created new session: [4:7577789171525245691:2108]
2025-11-28T14:52:10.017064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789171525245691:2108] [node 5] ICS09 handshake done sender: [4:7577789167230278368:2114] self: [4:7577789167230278367:0] peer: [5:7577789174605006801:0] socket: 372 qp: -1
2025-11-28T14:52:10.017099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789171525245691:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:10.017126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.017387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789171805031277:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.017517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789174605006803:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5198
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.017873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789167510063897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.017996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789171805031284:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:10.018251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789170310039416:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:10.018296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789167230278370:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:10.018296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789169827978384:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.018334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789169827978384:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.020220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789169827978384:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:10.020212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789167230278360:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.020313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.020359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789167230278360:2110] [node 6] ICP22 created new session: [4:7577789171525245694:2110]
2025-11-28T14:52:10.020307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789171805031274:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:10.020413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789171525245694:2110] [node 6] ICS09 handshake done sender: [4:7577789167230278370:2115] self: [4:7577789167230278369:0] peer: [6:7577789171805031279:0] socket: 422 qp: -1
2025-11-28T14:52:10.020456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789171525245694:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:10.020651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.020907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789169827978376:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.020946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789167510063963:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.021176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.020984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:10.021222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789169827978376:2109] [node 6] ICP22 created new session: [1:7577789174122945709:2109]
2025-11-28T14:52:10.021260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789174122945709:2109] [node 6] ICS09 handshake done sender: [1:7577789169827978384:2114] self: [1:7577789169827978383:0] peer: [6:7577789171805031286:0] socket: 427 qp: -1
2025-11-28T14:52:10.021222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789167510063963:2108] [node 4] ICP22 created new session: [6:7577789171805031289:2108]
2025-11-28T14:52:10.021289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789174122945709:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:10.021266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789171805031289:2108] [node 4] ICS09 handshake done sender: [6:7577789171805031274:2117] self: [6:7577789171805031279:0] peer: [4:7577789167230278369:0] socket: 423 qp: -1
2025-11-28T14:52:10.021328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.021293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789171805031289:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:10.021324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.021812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789174605006804:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5198
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.022094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789171525245693:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7964
2025-11-28T14:52:10.022366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789170310039414:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789170310039500:2119] held: no
2025-11-28T14:52:10.022544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789174605006799:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:10.022804Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789170310039477:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7199) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789174605006799:2120] held: no
2025-11-28T14:52:10.022889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789170310039477:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.022922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:10.022965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789170310039477:2108] [node 4] ICP22 created new session: [5:7577789174605006807:2108]
2025-11-28T14:52:10.023100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789174605006807:2108] [node 4] ICS09 handshake done sender: [5:7577789174605006799:2120] self: [5:7577789174605006801:0] peer: [4:7577789167230278367:0] socket: 425 qp: -1
2025-11-28T14:52:10.023247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789174605006807:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:10.023284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:10.024063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789170310039500:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:10.024479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789170310039414:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.024519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.024562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789170310039414:2048] [node 1] ICP22 created new session: [5:7577789174605006810:2048]
2025-11-28T14:52:10.024625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789174605006810:2048] [node 1] ICS09 handshake done sender: [5:7577789170310039500:2119] self: [5:7577789174605006800:0] peer: [1:7577789169827978381:0] socket: 367 qp: -1
2025-11-28T14:52:10.024676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789174605006810:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.024706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:10.025367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789167230278360:2110] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.025663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789171805031288:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:10.026134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789167510063897:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789171805031277:2119] held: no
2025-11-28T14:52:10.026363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789171805031277:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.028301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789170310039488:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.028336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789170310039488:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.026660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789167510063897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.026707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.026745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789167510063897:2048] [node 1] ICP22 created new session: [6:7577789171805031292:2048]
2025-11-28T14:52:10.026788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789171805031292:2048] [node 1] ICS09 handshake done sender: [6:7577789171805031277:2119] self: [6:7577789171805031286:0] peer: [1:7577789169827978383:0] socket: 428 qp: -1
2025-11-28T14:52:10.026814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789171805031292:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.026840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:10.027657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789171805031291:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:10.027981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789171805031291:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.028101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789167510063956:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.029075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789170310039488:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.029279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789171805031295:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22285
2025-11-28T14:52:10.029402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789170310039478:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.029467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.029515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789170310039478:2109] [node 6] ICP22 created new session: [5:7577789174605006813:2109]
2025-11-28T14:52:10.029555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789174605006813:2109] [node 6] ICS09 handshake done sender: [5:7577789170310039488:2114] self: [5:7577789170310039487:0] peer: [6:7577789171805031294:0] socket: 366 qp: -1
2025-11-28T14:52:10.029638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789174605006813:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:10.029681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.029773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789171805031291:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:10.030048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789167510063956:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.030089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:10.030129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789167510063956:2105] [node 5] ICP22 created new session: [6:7577789171805031296:2105]
2025-11-28T14:52:10.030172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789171805031296:2105] [node 5] ICS09 handshake done sender: [6:7577789171805031291:2123] self: [6:7577789171805031294:0] peer: [5:7577789170310039487:0] socket: 376 qp: -1
2025-11-28T14:52:10.030201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789171805031296:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:10.030225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:10.056463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:10.056733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789171292067524:2048] [node 1] ICS01 socket: 404 reason# EndOfStream
2025-11-28T14:52:10.056834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789166997100126:2048] [node 1] ICP30 unregister session Session# [2:7577789171292067524:2048] VirtualId# [2:7577789166997100224:0]
2025-11-28T14:52:10.056893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789171292067524:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.057017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.057091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.057505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:10.057706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789170818240563:2048] [node 1] ICS01 socket: 403 reason# EndOfStream
2025-11-28T14:52:10.057754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789166523273168:2048] [node 1] ICP30 unregister session Session# [3:7577789170818240563:2048] VirtualId# [3:7577789166523273262:0]
2025-11-28T14:52:10.057796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789170818240563:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.057867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.057914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.060473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:10.060716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789171525245688:2048] [node 1] ICS01 socket: 409 reason# EndOfStream
2025-11-28T14:52:10.060769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789167230278295:2048] [node 1] ICP30 unregister session Session# [4:7577789171525245688:2048] VirtualId# [4:7577789167230278383:0]
2025-11-28T14:52:10.060811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789171525245688:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.060893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.060942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.061243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:10.061449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789174605006810:2048] [node 1] ICS01 socket: 367 reason# EndOfStream
2025-11-28T14:52:10.061514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789170310039414:2048] [node 1] ICP30 unregister session Session# [5:7577789174605006810:2048] VirtualId# [5:7577789174605006800:0]
2025-11-28T14:52:10.061551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789174605006810:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.061635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.061688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.061912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:10.062042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789171805031292:2048] [node 1] ICS01 socket: 428 reason# EndOfStream
2025-11-28T14:52:10.062067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789167510063897:2048] [node 1] ICP30 unregister session Session# [6:7577789171805031292:2048] VirtualId# [6:7577789171805031286:0]
2025-11-28T14:52:10.062088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789171805031292:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.062150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.062196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.086032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:10.086264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789166523273255:2049] [node 2] ICS07 socket disconnect 381 reason# EndOfStream
2025-11-28T14:52:10.086299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789166523273255:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.086414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:10.086498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789166523273255:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:10.088640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:10.088882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789167230278381:2049] [node 2] ICS07 socket disconnect 384 reason# EndOfStream
2025-11-28T14:52:10.088914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789167230278381:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.089016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:10.089061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789167230278381:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:10.089738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789166523273169:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:10.089784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789166523273255:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:10.090131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:10.090299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789170310039498:2049] [node 2] ICS01 socket: 410 reason# EndOfStream
2025-11-28T14:52:10.090805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789166523273169:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.090324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789170310039415:2049] [node 2] ICP30 unregister session Session# [5:7577789170310039498:2049] VirtualId# [5:7577789170310039495:0]
2025-11-28T14:52:10.090345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789170310039498:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.090972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789166523273255:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.090412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:10.091100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789167230278296:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.090477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:10.091073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789166523273169:2049] [node 2] ICP30 unregister session Session# [3:7577789166523273255:2049] VirtualId# [3:7577789166523273253:0]
2025-11-28T14:52:10.091191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789167230278381:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.091120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:10.091158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789166523273169:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.091230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789167230278296:2049] [node 2] ICP30 unregister session Session# [4:7577789167230278381:2049] VirtualId# [4:7577789167230278377:0]
2025-11-28T14:52:10.091201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:10.091174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.091279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:10.091312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789167230278296:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.091325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.091402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789171805031280:2049] [node 2] ICS07 socket disconnect 418 reason# EndOfStream
2025-11-28T14:52:10.091427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789171805031280:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.091497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:10.091539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789171805031280:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:10.092241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789167510063898:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.092279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789171805031280:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.092300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789167510063898:2049] [node 2] ICP30 unregister session Session# [6:7577789171805031280:2049] VirtualId# [6:7577789171805031275:0]
2025-11-28T14:52:10.092330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:10.092351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789167510063898:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.092373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.101454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789167230278296:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:10.102447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789167510063898:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:10.115971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:10.116251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789167230278386:2050] [node 3] ICS01 socket: 387 reason# EndOfStream
2025-11-28T14:52:10.116314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789167230278297:2050] [node 3] ICP30 unregister session Session# [4:7577789167230278386:2050] VirtualId# [4:7577789167230278378:0]
2025-11-28T14:52:10.116357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789167230278386:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.116489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:10.116561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:10.116666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:10.116804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789174605006805:2050] [node 3] ICS07 socket disconnect 416 reason# EndOfStream
2025-11-28T14:52:10.116830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789174605006805:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.116896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:10.116942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789174605006805:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:10.117279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:10.117545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789171805031282:2050] [node 3] ICS07 socket disconnect 395 reason# EndOfStream
2025-11-28T14:52:10.117578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789171805031282:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.117659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:10.117724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789171805031282:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:10.117687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789170310039416:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25171) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.117747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789174605006805:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.117784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789170310039416:2050] [node 3] ICP30 unregister session Session# [5:7577789174605006805:2050] VirtualId# [5:7577789174605006798:0]
2025-11-28T14:52:10.117835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:10.117865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789170310039416:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.117882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.118431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789167510063899:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25171) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.118476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789171805031282:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.118506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789167510063899:2050] [node 3] ICP30 unregister session Session# [6:7577789171805031282:2050] VirtualId# [6:7577789167510063977:0]
2025-11-28T14:52:10.118540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:10.118570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789167510063899:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.118589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.128039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789167510063899:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:10.128031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789170310039416:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:10.143876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:10.144098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789174605006807:2108] [node 4] ICS01 socket: 425 reason# EndOfStream
2025-11-28T14:52:10.144141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789170310039477:2108] [node 4] ICP30 unregister session Session# [5:7577789174605006807:2108] VirtualId# [5:7577789174605006801:0]
2025-11-28T14:52:10.144186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789174605006807:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.144320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:10.144384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:10.144722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:10.144919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789171805031289:2108] [node 4] ICS01 socket: 423 reason# EndOfStream
2025-11-28T14:52:10.144965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789167510063963:2108] [node 4] ICP30 unregister session Session# [6:7577789171805031289:2108] VirtualId# [6:7577789171805031279:0]
2025-11-28T14:52:10.144995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789171805031289:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.145077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:10.145125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:10.164451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:10.164622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789171805031296:2105] [node 5] ICS01 socket: 376 reason# EndOfStream
2025-11-28T14:52:10.164651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789167510063956:2105] [node 5] ICP30 unregister session Session# [6:7577789171805031296:2105] VirtualId# [6:7577789171805031294:0]
2025-11-28T14:52:10.164674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789171805031296:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.164744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:10.164792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:10.219122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789173542989669:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:10.219200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789173542989668:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:10.237388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789173580887157:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:10.237424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789173580887158:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:10.256338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789172906564194:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:10.256444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789172906564195:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:10.278969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789174063733901:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:10.279000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789174063733902:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:10.279167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789174063733900:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:10.300904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789171803722235:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:10.300967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789171803722237:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:10.301017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789171803722236:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:10.323839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789172482697113:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:10.323875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789172482697115:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:10.323897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789172482697114:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:10.365186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789172482697174:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.401715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789172906564254:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25156
2025-11-28T14:52:10.408114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789173542989734:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:10.411593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789173542989735:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:10.411465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789172482697177:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:10.411639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789174063733965:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:10.411931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789172906564259:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:10.412060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789174063733966:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:10.412100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789173542989738:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:10.412253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789172906564260:2110] [node 5] ICP01 ready to work
2025-11-28T14:52:10.412258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789173542989668:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(50868503ns) {Configure:104(50862767ns) {ProcessConfigured:135(136395ns) {StartInitialHandshake:157(133280ns) {DropHandshakes:470(3438ns) {DropIncomingHandshake:442(961ns) DropOutgoingHandshake:457(481ns)} GenerateSessionVirtualId:988(1104ns) PrepareNewSessionHandshake:477(2794ns) {SwitchToState:179(1010ns)}}}}}
2025-11-28T14:52:10.412706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789172906564261:2111] [node 6] ICP01 ready to work
2025-11-28T14:52:10.412834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789172482697174:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47291440ns) {Configure:104(47288321ns) {ProcessConfigured:135(101059ns) {StartInitialHandshake:157(98396ns) {DropHandshakes:470(2342ns) {DropIncomingHandshake:442(814ns) DropOutgoingHandshake:457(429ns)} GenerateSessionVirtualId:988(641ns) PrepareNewSessionHandshake:477(1195ns) {SwitchToState:179(569ns)}}}}}
2025-11-28T14:52:10.414717Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789174063733900:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32695732ns) {Configure:104(32688192ns) {ProcessConfigured:135(122809ns) {StartInitialHandshake:157(120838ns) {DropHandshakes:470(1234ns) {DropIncomingHandshake:442(405ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(522ns) PrepareNewSessionHandshake:477(1214ns) {SwitchToState:179(722ns)}}}}}
2025-11-28T14:52:10.414736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789172906564194:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39492843ns) {Configure:104(39490328ns) {ProcessConfigured:135(34221ns) {StartInitialHandshake:157(32725ns) {DropHandshakes:470(4423ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(124ns)} GenerateSessionVirtualId:988(347ns) PrepareNewSessionHandshake:477(539ns) {SwitchToState:179(155ns)}}}}}
2025-11-28T14:52:10.415535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789173580887157:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33699108ns) {Configure:104(33687123ns) {ProcessConfigured:135(125479ns) {StartInitialHandshake:157(123569ns) {DropHandshakes:470(1131ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(839ns) {SwitchToState:179(407ns)}}}}}
2025-11-28T14:52:10.415558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789173580887158:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42162120ns) {Configure:104(42156096ns) {ProcessConfigured:135(153243ns) {StartInitialHandshake:157(151482ns) {DropHandshakes:470(1168ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(1077ns) {SwitchToState:179(578ns)}}}}}
2025-11-28T14:52:10.416020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789173580887225:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:10.416051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789173580887224:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:10.416305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789171803722236:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36244023ns) {Configure:104(36236304ns) {ProcessConfigured:135(165094ns) {StartInitialHandshake:157(117789ns) {DropHandshakes:470(1617ns) {DropIncomingHandshake:442(607ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(863ns) PrepareNewSessionHandshake:477(1775ns) {SwitchToState:179(1090ns)}}}}}
2025-11-28T14:52:10.416376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789171803722235:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36816438ns) {Configure:104(34169209ns) {ProcessConfigured:135(136871ns) {StartInitialHandshake:157(134931ns) {DropHandshakes:470(785ns) {DropIncomingHandshake:442(228ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(374ns) PrepareNewSessionHandshake:477(1703ns) {SwitchToState:179(739ns)}}}}}
2025-11-28T14:52:10.416489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789173580887226:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:10.416992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789171803722302:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:10.417176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789171803722303:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:10.425453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789172906564254:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.425753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789174063733980:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8742
2025-11-28T14:52:10.425684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789173542989747:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7242
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.426276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789173542989748:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7242
No RDMA context found for GID: 2025-11-28T14:52:10.426477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789173542989749:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7242
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.426516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789174063733980:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.426349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789172906564194:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.427946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989729:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.428426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789172906564275:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25156
2025-11-28T14:52:10.427981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789173542989729:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.428436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789174063733981:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8742
2025-11-28T14:52:10.429616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989747:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.430099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789173542989729:2107] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:10.431452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789173542989669:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.431371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789174063733900:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.431498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.431546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789173542989669:2049] [node 3] ICP22 created new session: [1:7577789173542989752:2049]
2025-11-28T14:52:10.431971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789172906564277:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25156
2025-11-28T14:52:10.431746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789173542989752:2049] [node 3] ICS09 handshake done sender: [1:7577789173542989729:2107] self: [1:7577789173542989728:0] peer: [3:7577789172906564276:0] socket: 372 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.432017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789171803722308:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19773
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.431767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789173542989752:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:10.431879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.431967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989749:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.432797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789171803722308:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.432720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789173580887231:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22004
2025-11-28T14:52:10.432810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789173580887232:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22004
2025-11-28T14:52:10.432448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989740:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.434171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789173580887233:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22004
2025-11-28T14:52:10.432475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789173542989740:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.434287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789173580887234:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22004
2025-11-28T14:52:10.434615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789173580887231:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.434753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789174063733981:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.434872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989748:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.434707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789172906564254:2107] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:10.434941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789174063733902:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.435062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789173580887233:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.445823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789173580887239:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22004
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.446367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789172906564275:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.446417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789173542989669:2049] [node 3] ICP18 handshake (actor [1:7577789173542989749:2117]) is too old
2025-11-28T14:52:10.446424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789174063733984:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8742
2025-11-28T14:52:10.446463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789173580887232:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.446500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789172906564194:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.446706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.446659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789174063733985:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8742
2025-11-28T14:52:10.446828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789172906564194:2048] [node 1] ICP22 created new session: [3:7577789172906564280:2048]
2025-11-28T14:52:10.446641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789173542989753:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7242
2025-11-28T14:52:10.447045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789172906564280:2048] [node 1] ICS09 handshake done sender: [3:7577789172906564254:2107] self: [3:7577789172906564276:0] peer: [1:7577789173542989728:0] socket: 373 qp: -1
2025-11-28T14:52:10.447195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789172906564280:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.447023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789173542989740:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:10.447321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.448181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789174063733980:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:10.448815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789173542989757:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7242
2025-11-28T14:52:10.448933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789173580887157:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.448825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789174063733900:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.449045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.448888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789173542989734:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.449074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789174063733900:2048] [node 1] ICP22 created new session: [4:7577789174063733986:2048]
2025-11-28T14:52:10.449151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789174063733986:2048] [node 1] ICS09 handshake done sender: [4:7577789174063733980:2117] self: [4:7577789174063733982:0] peer: [1:7577789173542989739:0] socket: 385 qp: -1
2025-11-28T14:52:10.449173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789174063733986:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.449102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:10.449159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789173542989734:2108] [node 4] ICP22 created new session: [1:7577789173542989759:2108]
2025-11-28T14:52:10.449279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:10.449209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789173542989759:2108] [node 4] ICS09 handshake done sender: [1:7577789173542989740:2112] self: [1:7577789173542989739:0] peer: [4:7577789174063733982:0] socket: 381 qp: -1
2025-11-28T14:52:10.449242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789173542989759:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:10.449277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:10.449795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989737:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.449817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789173542989737:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.450204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789172906564267:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.450238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789172906564267:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.452256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789172906564278:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25156
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.452449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789172906564279:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25156
2025-11-28T14:52:10.453260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789173542989737:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:10.453736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789173542989668:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.453847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:10.453884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789173542989668:2048] [node 2] ICP22 created new session: [1:7577789173542989763:2048]
2025-11-28T14:52:10.454101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789173542989763:2048] [node 2] ICS09 handshake done sender: [1:7577789173542989737:2111] self: [1:7577789173542989736:0] peer: [2:7577789173580887243:0] socket: 378 qp: -1
2025-11-28T14:52:10.454134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789173542989763:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:10.454212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:10.454240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697191:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.454477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789174063733989:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8742
2025-11-28T14:52:10.454527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789172906564267:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:10.454595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789172482697191:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.454700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789174063733981:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:10.454803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789172482697113:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.455077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789172906564259:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.455147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789174063733902:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.455193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:10.455224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789174063733902:2050] [node 3] ICP22 created new session: [4:7577789174063733990:2050]
2025-11-28T14:52:10.455231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989744:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.455250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789174063733990:2050] [node 3] ICS09 handshake done sender: [4:7577789174063733981:2118] self: [4:7577789174063733983:0] peer: [3:7577789172906564266:0] socket: 396 qp: -1
2025-11-28T14:52:10.455254Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789173542989744:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.455240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:10.455269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789174063733990:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:10.455303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789172906564259:2108] [node 4] ICP22 created new session: [3:7577789172906564285:2108]
2025-11-28T14:52:10.455355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:10.455388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789172906564285:2108] [node 4] ICS09 handshake done sender: [3:7577789172906564267:2114] self: [3:7577789172906564266:0] peer: [4:7577789174063733983:0] socket: 393 qp: -1
2025-11-28T14:52:10.455592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789172906564285:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:10.455642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:10.456234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789173542989744:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:10.456494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789173542989738:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.456642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.456731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789173542989738:2110] [node 6] ICP22 created new session: [1:7577789173542989766:2110]
No RDMA context found for GID: 2025-11-28T14:52:10.456888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789173542989766:2110] [node 6] ICS09 handshake done sender: [1:7577789173542989744:2114] self: [1:7577789173542989743:0] peer: [6:7577789172482697192:0] socket: 390 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:10.456938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789173542989766:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:10.456980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.457011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697193:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
2025-11-28T14:52:10.458624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789172482697113:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789172482697191:2115] held: no
2025-11-28T14:52:10.459272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789172482697191:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.459432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697194:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
2025-11-28T14:52:10.459457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789172482697113:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.459495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.459531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789172482697113:2048] [node 1] ICP22 created new session: [6:7577789172482697195:2048]
2025-11-28T14:52:10.459572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789172482697195:2048] [node 1] ICS09 handshake done sender: [6:7577789172482697191:2115] self: [6:7577789172482697192:0] peer: [1:7577789173542989743:0] socket: 420 qp: -1
2025-11-28T14:52:10.459610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789172482697195:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.459642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.460002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789172482697194:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.460015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789174063733993:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8742
2025-11-28T14:52:10.460117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697197:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
2025-11-28T14:52:10.460196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789172482697177:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.460532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789174063733979:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.460559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789174063733979:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.461375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789174063733979:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.461485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697200:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
2025-11-28T14:52:10.461575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789174063733966:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.461640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.461685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789174063733966:2110] [node 6] ICP22 created new session: [4:7577789174063733995:2110]
2025-11-28T14:52:10.461766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789174063733995:2110] [node 6] ICS09 handshake done sender: [4:7577789174063733979:2116] self: [4:7577789174063733978:0] peer: [6:7577789172482697199:0] socket: 384 qp: -1
2025-11-28T14:52:10.461994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789174063733995:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:10.462085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789172482697194:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:10.462056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.462304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789172482697177:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.462332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:10.462356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789172482697177:2109] [node 4] ICP22 created new session: [6:7577789172482697201:2109]
2025-11-28T14:52:10.462381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789172482697201:2109] [node 4] ICS09 handshake done sender: [6:7577789172482697194:2117] self: [6:7577789172482697199:0] peer: [4:7577789174063733978:0] socket: 424 qp: -1
2025-11-28T14:52:10.462401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789172482697201:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:10.462418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:10.463067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789172482697197:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.463209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789172482697115:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.463428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789172906564274:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.463461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789172906564274:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.463624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789172906564265:2113] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:10.471277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789172906564274:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.471539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789172906564261:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.471527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697204:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
2025-11-28T14:52:10.471628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.471693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789172906564261:2111] [node 6] ICP22 created new session: [3:7577789172906564289:2111]
2025-11-28T14:52:10.471739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789172906564289:2111] [node 6] ICS09 handshake done sender: [3:7577789172906564274:2116] self: [3:7577789172906564270:0] peer: [6:7577789172482697203:0] socket: 376 qp: -1
2025-11-28T14:52:10.471777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789172906564289:2111] [node 6] ICS10 traffic start
2025-11-28T14:52:10.471810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.472060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789172482697197:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:10.472563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789172482697115:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.472620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:10.472666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789172482697115:2050] [node 3] ICP22 created new session: [6:7577789172482697205:2050]
2025-11-28T14:52:10.472707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789172482697205:2050] [node 3] ICS09 handshake done sender: [6:7577789172482697197:2118] self: [6:7577789172482697203:0] peer: [3:7577789172906564270:0] socket: 386 qp: -1
2025-11-28T14:52:10.472742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789172482697205:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:10.472775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:10.485530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789171803722300:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:10.505661Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789171803722301:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.514092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989757:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.514183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789171803722315:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19773
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.514704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789171803722315:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.514782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789171803722316:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19773
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.514916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789171803722317:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19773
2025-11-28T14:52:10.515092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789171803722235:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.515296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789171803722316:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.515349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989742:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.515379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789171803722317:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.515382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789173542989742:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.515528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789171803722302:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.515612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789171803722237:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.515850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789174063733974:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.515881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789174063733974:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.516726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789173542989742:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:10.515907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789172906564269:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.515952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789172906564269:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:10.517280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789172906564269:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:10.517411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789173542989735:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.517563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789172906564260:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.518445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:10.518822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789174063733974:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.518510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789173542989735:2109] [node 5] ICP22 created new session: [1:7577789173542989770:2109]
2025-11-28T14:52:10.517867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:10.520933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789173542989770:2109] [node 5] ICS09 handshake done sender: [1:7577789173542989742:2113] self: [1:7577789173542989741:0] peer: [5:7577789171803722318:0] socket: 416 qp: -1
2025-11-28T14:52:10.518415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789172906564260:2110] [node 5] ICP22 created new session: [3:7577789172906564292:2110]
2025-11-28T14:52:10.519062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789174063733965:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.518571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789172906564292:2110] [node 5] ICS09 handshake done sender: [3:7577789172906564269:2115] self: [3:7577789172906564268:0] peer: [5:7577789171803722320:0] socket: 379 qp: -1
2025-11-28T14:52:10.522859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:10.518606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789172906564292:2110] [node 5] ICS10 traffic start
2025-11-28T14:52:10.518650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:10.523496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789171803722321:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19773
2025-11-28T14:52:10.523549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789174063733965:2108] [node 5] ICP22 created new session: [4:7577789174063733998:2108]
2025-11-28T14:52:10.523617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789174063733998:2108] [node 5] ICS09 handshake done sender: [4:7577789174063733974:2115] self: [4:7577789174063733973:0] peer: [5:7577789171803722319:0] socket: 382 qp: -1
2025-11-28T14:52:10.523668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789174063733998:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:10.522830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789173542989770:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:10.522897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:10.523940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:10.528027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789171803722235:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789171803722315:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.528318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789171803722322:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19773
2025-11-28T14:52:10.528692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789171803722315:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:10.528981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789171803722235:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.529038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.529081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789171803722235:2048] [node 1] ICP22 created new session: [5:7577789171803722324:2048]
2025-11-28T14:52:10.529132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789171803722324:2048] [node 1] ICS09 handshake done sender: [5:7577789171803722315:2116] self: [5:7577789171803722318:0] peer: [1:7577789173542989741:0] socket: 419 qp: -1
2025-11-28T14:52:10.529164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789171803722324:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.529201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.529621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789171803722323:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19773
2025-11-28T14:52:10.529988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789171803722317:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:10.531990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789171803722237:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.532042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:10.532087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789171803722237:2050] [node 3] ICP22 created new session: [5:7577789171803722327:2050]
2025-11-28T14:52:10.532127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789171803722327:2050] [node 3] ICS09 handshake done sender: [5:7577789171803722317:2118] self: [5:7577789171803722320:0] peer: [3:7577789172906564268:0] socket: 432 qp: -1
2025-11-28T14:52:10.532165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789171803722327:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:10.532200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:10.533106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789171803722316:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:10.533541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789171803722302:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.533586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.533626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789171803722302:2108] [node 4] ICP22 created new session: [5:7577789171803722329:2108]
2025-11-28T14:52:10.533672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789171803722329:2108] [node 4] ICS09 handshake done sender: [5:7577789171803722316:2117] self: [5:7577789171803722319:0] peer: [4:7577789174063733973:0] socket: 431 qp: -1
2025-11-28T14:52:10.533708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789171803722329:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:10.533740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:10.533695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789174063734000:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8742
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.535382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697207:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
2025-11-28T14:52:10.539160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789173580887222:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:10.552624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789173542989753:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.552848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789173542989668:2048] [node 2] ICP18 handshake (actor [1:7577789173542989753:2118]) is too old
2025-11-28T14:52:10.563323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789173580887228:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:10.576117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789174063733984:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.576158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789173580887234:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.576277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789173580887239:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.576354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789174063733901:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.577332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789173580887157:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789173580887232:2115] held: no
2025-11-28T14:52:10.577524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789173580887228:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.577899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789173580887244:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22004
2025-11-28T14:52:10.578837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789173580887232:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.579272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789173580887228:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:10.579229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789174063734002:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8742
2025-11-28T14:52:10.579964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789174063733984:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:10.579889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789173580887157:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.579946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:10.579998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789173580887157:2048] [node 1] ICP22 created new session: [2:7577789173580887249:2048]
2025-11-28T14:52:10.580155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789174063733901:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.580043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789173580887249:2048] [node 1] ICS09 handshake done sender: [2:7577789173580887232:2115] self: [2:7577789173580887243:0] peer: [1:7577789173542989736:0] socket: 392 qp: -1
2025-11-28T14:52:10.580081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789173580887249:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:10.580114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:10.580300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:10.580348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789174063733901:2049] [node 2] ICP22 created new session: [4:7577789174063734003:2049]
2025-11-28T14:52:10.580406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789174063734003:2049] [node 2] ICS09 handshake done sender: [4:7577789174063733984:2119] self: [4:7577789174063734001:0] peer: [2:7577789173580887227:0] socket: 414 qp: -1
2025-11-28T14:52:10.581099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789173580887225:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.580439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789174063734003:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:10.581157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:10.580490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:10.581201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789173580887225:2107] [node 4] ICP22 created new session: [2:7577789173580887251:2107]
2025-11-28T14:52:10.581258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789173580887251:2107] [node 4] ICS09 handshake done sender: [2:7577789173580887228:2112] self: [2:7577789173580887227:0] peer: [4:7577789174063734001:0] socket: 413 qp: -1
2025-11-28T14:52:10.581307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789173580887251:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:10.581344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.584408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697208:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
2025-11-28T14:52:10.594700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789172482697187:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:10.621719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789174063733993:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.621962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789174063733966:2110] [node 6] ICP18 handshake (actor [4:7577789174063733993:2122]) is too old
2025-11-28T14:52:10.639865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789171803722305:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:10.667063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789172906564279:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.667229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789172906564260:2110] [node 5] ICP18 handshake (actor [3:7577789172906564279:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.667860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789171803722332:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19773
2025-11-28T14:52:10.672568Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789171803722312:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:10.680714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789172482697207:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.680872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789172482697174:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.681291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789171803722312:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.681826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789171803722312:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:10.681932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697210:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
2025-11-28T14:52:10.682088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789171803722303:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.682149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.682189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789171803722303:2109] [node 6] ICP22 created new session: [5:7577789171803722334:2109]
2025-11-28T14:52:10.682223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789171803722334:2109] [node 6] ICS09 handshake done sender: [5:7577789171803722312:2115] self: [5:7577789171803722311:0] peer: [6:7577789172482697209:0] socket: 405 qp: -1
2025-11-28T14:52:10.682297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789171803722334:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:10.682322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.682261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789172482697207:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:10.682460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789172482697174:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.682501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:10.682598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789172482697174:2107] [node 5] ICP22 created new session: [6:7577789172482697211:2107]
2025-11-28T14:52:10.682731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789172482697211:2107] [node 5] ICS09 handshake done sender: [6:7577789172482697207:2121] self: [6:7577789172482697209:0] peer: [5:7577789171803722311:0] socket: 438 qp: -1
2025-11-28T14:52:10.682758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789172482697211:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:10.682822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:10.683055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789172482697174:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:10.685270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789173580887223:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:10.693253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789172906564278:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.693493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789172906564195:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.696643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789173580887237:2118] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:10.704959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789172482697208:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.705143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789172482697114:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.704874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789173580887223:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.705481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789173580887223:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:10.705568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789172906564295:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25156
2025-11-28T14:52:10.705963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789173580887237:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.706038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789173580887158:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.706026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789172906564278:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:10.706211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789172906564195:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.706108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:10.706349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:10.706313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789173580887158:2049] [node 3] ICP22 created new session: [2:7577789173580887255:2049]
2025-11-28T14:52:10.706399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789172906564195:2049] [node 2] ICP22 created new session: [3:7577789172906564296:2049]
2025-11-28T14:52:10.706436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789172906564296:2049] [node 2] ICS09 handshake done sender: [3:7577789172906564278:2119] self: [3:7577789172906564294:0] peer: [2:7577789173580887220:0] socket: 409 qp: -1
2025-11-28T14:52:10.707087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789173580887255:2049] [node 3] ICS09 handshake done sender: [2:7577789173580887223:2111] self: [2:7577789173580887220:0] peer: [3:7577789172906564294:0] socket: 406 qp: -1
2025-11-28T14:52:10.706471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789172906564296:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:10.707119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789173580887255:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:10.707302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:10.706503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:10.711390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789173580887230:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:10.712453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789173580887237:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:10.712560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789171803722332:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.712577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789172482697214:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20565
2025-11-28T14:52:10.712822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789173580887226:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.712833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789171803722236:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:10.712893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:10.712955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789173580887226:2109] [node 6] ICP22 created new session: [2:7577789173580887258:2109]
2025-11-28T14:52:10.712986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789172482697208:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:10.713008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789173580887258:2109] [node 6] ICS09 handshake done sender: [2:7577789173580887237:2118] self: [2:7577789173580887236:0] peer: [6:7577789172482697213:0] socket: 383 qp: -1
2025-11-28T14:52:10.713045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789173580887258:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:10.713077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:10.713291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789172482697114:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.713429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:10.713503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789172482697114:2049] [node 2] ICP22 created new session: [6:7577789172482697215:2049]
2025-11-28T14:52:10.713556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789172482697215:2049] [node 2] ICS09 handshake done sender: [6:7577789172482697208:2122] self: [6:7577789172482697213:0] peer: [2:7577789173580887236:0] socket: 395 qp: -1
2025-11-28T14:52:10.713594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789172482697215:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:10.713631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789173580887230:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:10.713634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:10.714164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789173580887230:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:10.714346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789173580887224:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:10.714338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789172482697114:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:10.714422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:10.714540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789173580887224:2108] [node 5] ICP22 created new session: [2:7577789173580887261:2108]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:10.714592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789173580887261:2108] [node 5] ICS09 handshake done sender: [2:7577789173580887230:2113] self: [2:7577789173580887229:0] peer: [5:7577789171803722336:0] socket: 440 qp: -1
2025-11-28T14:52:10.714622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789173580887261:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:10.714649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:10.714666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789171803722337:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19773
2025-11-28T14:52:10.715082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789171803722332:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:10.715333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789171803722236:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62549) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789171803722332:2122] held: no
2025-11-28T14:52:10.715433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789171803722236:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:10.715466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:10.715506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789171803722236:2049] [node 2] ICP22 created new session: [5:7577789171803722338:2049]
2025-11-28T14:52:10.715542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789171803722338:2049] [node 2] ICS09 handshake done sender: [5:7577789171803722332:2122] self: [5:7577789171803722336:0] peer: [2:7577789173580887229:0] socket: 425 qp: -1
2025-11-28T14:52:10.715573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789171803722338:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:10.715607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:10.746792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:10.747025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789173580887249:2048] [node 1] ICS01 socket: 392 reason# EndOfStream
2025-11-28T14:52:10.747071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789173580887157:2048] [node 1] ICP30 unregister session Session# [2:7577789173580887249:2048] VirtualId# [2:7577789173580887243:0]
2025-11-28T14:52:10.747110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789173580887249:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.747226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.747282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.748070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:10.748205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789172906564280:2048] [node 1] ICS07 socket disconnect 373 reason# EndOfStream
2025-11-28T14:52:10.748229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789172906564280:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.748302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.748343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789172906564280:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:10.750552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:10.750579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789172906564194:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18972) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:10.750624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789172906564280:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:10.750706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789174063733986:2048] [node 1] ICS01 socket: 385 reason# EndOfStream
2025-11-28T14:52:10.750944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789174063733900:2048] [node 1] ICP30 unregister session Session# [4:7577789174063733986:2048] VirtualId# [4:7577789174063733982:0]
2025-11-28T14:52:10.750964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789174063733986:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.751020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.751062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.751480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789172906564194:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18972) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.751574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789172906564280:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.751624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789172906564194:2048] [node 1] ICP30 unregister session Session# [3:7577789172906564280:2048] VirtualId# [3:7577789172906564276:0]
2025-11-28T14:52:10.751666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.751698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789172906564194:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18972) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.751715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18972) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.751882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:10.752090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789171803722324:2048] [node 1] ICS01 socket: 419 reason# EndOfStream
2025-11-28T14:52:10.752129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789171803722235:2048] [node 1] ICP30 unregister session Session# [5:7577789171803722324:2048] VirtualId# [5:7577789171803722318:0]
2025-11-28T14:52:10.752152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789171803722324:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.752216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.752266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.752698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:10.752852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789172482697195:2048] [node 1] ICS01 socket: 420 reason# EndOfStream
2025-11-28T14:52:10.752881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789172482697113:2048] [node 1] ICP30 unregister session Session# [6:7577789172482697195:2048] VirtualId# [6:7577789172482697192:0]
2025-11-28T14:52:10.752897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789172482697195:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.752950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:10.752994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:10.759939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:10.760153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789172906564296:2049] [node 2] ICS01 socket: 409 reason# EndOfStream
2025-11-28T14:52:10.760197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789172906564195:2049] [node 2] ICP30 unregister session Session# [3:7577789172906564296:2049] VirtualId# [3:7577789172906564294:0]
2025-11-28T14:52:10.760229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789172906564296:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.760313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:10.760380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:10.761884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789172906564194:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:10.762115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:10.762343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789174063734003:2049] [node 2] ICS01 socket: 414 reason# EndOfStream
2025-11-28T14:52:10.762390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789174063733901:2049] [node 2] ICP30 unregister session Session# [4:7577789174063734003:2049] VirtualId# [4:7577789174063734001:0]
2025-11-28T14:52:10.762417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789174063734003:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.762511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:10.762588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:10.763099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:10.763289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789171803722338:2049] [node 2] ICS01 socket: 425 reason# EndOfStream
2025-11-28T14:52:10.763334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789171803722236:2049] [node 2] ICP30 unregister session Session# [5:7577789171803722338:2049] VirtualId# [5:7577789171803722336:0]
2025-11-28T14:52:10.763360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789171803722338:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.763421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:10.763476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:10.763913Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:10.764042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789172482697215:2049] [node 2] ICS07 socket disconnect 395 reason# EndOfStream
2025-11-28T14:52:10.764070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789172482697215:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.764123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:10.764167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789172482697215:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:10.764800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789172482697114:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62549) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.764912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789172482697215:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.764961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789172482697114:2049] [node 2] ICP30 unregister session Session# [6:7577789172482697215:2049] VirtualId# [6:7577789172482697213:0]
2025-11-28T14:52:10.765005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:10.765038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789172482697114:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62549) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.765057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62549) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.775187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789172482697114:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:10.788207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:10.788442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789174063733990:2050] [node 3] ICS01 socket: 396 reason# EndOfStream
2025-11-28T14:52:10.788491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789174063733902:2050] [node 3] ICP30 unregister session Session# [4:7577789174063733990:2050] VirtualId# [4:7577789174063733983:0]
2025-11-28T14:52:10.788524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789174063733990:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.788618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:10.788660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:10.789254Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:10.789466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789171803722327:2050] [node 3] ICS07 socket disconnect 432 reason# EndOfStream
2025-11-28T14:52:10.789498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789171803722327:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.789567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:10.789629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789171803722327:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:10.790089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:10.790296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789172482697205:2050] [node 3] ICS07 socket disconnect 386 reason# EndOfStream
2025-11-28T14:52:10.790290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789171803722237:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17506) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.790372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789171803722327:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.790370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789172482697205:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.790456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789171803722237:2050] [node 3] ICP30 unregister session Session# [5:7577789171803722327:2050] VirtualId# [5:7577789171803722320:0]
2025-11-28T14:52:10.790449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:10.790511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:10.790498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789172482697205:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:10.790560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789171803722237:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17506) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.790574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17506) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.791197Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789172482697115:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17506) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.791232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789172482697205:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.791251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789172482697115:2050] [node 3] ICP30 unregister session Session# [6:7577789172482697205:2050] VirtualId# [6:7577789172482697203:0]
2025-11-28T14:52:10.791291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:10.791313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789172482697115:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17506) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.791328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17506) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.800808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789172482697115:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:10.800808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789171803722237:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:10.833066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:10.833293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789171803722329:2108] [node 4] ICS07 socket disconnect 431 reason# EndOfStream
2025-11-28T14:52:10.833335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789171803722329:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.833431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:10.833494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789171803722329:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:10.833732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:10.833874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789172482697201:2109] [node 4] ICS07 socket disconnect 424 reason# EndOfStream
2025-11-28T14:52:10.833895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789172482697201:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.833955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:10.833994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789172482697201:2109] [node 4] ICS15 start handshake
2025-11-28T14:52:10.834191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789171803722302:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9762) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.834323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789171803722329:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.834359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789171803722302:2108] [node 4] ICP30 unregister session Session# [5:7577789171803722329:2108] VirtualId# [5:7577789171803722319:0]
2025-11-28T14:52:10.834396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:10.834424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789171803722302:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.834433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.834489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789172482697177:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9762) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.834558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789172482697201:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.834605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789172482697177:2109] [node 4] ICP30 unregister session Session# [6:7577789172482697201:2109] VirtualId# [6:7577789172482697199:0]
2025-11-28T14:52:10.834647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:10.834677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789172482697177:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.834690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.844558Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:10.844805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789172482697177:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:52:10.844830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789172482697211:2107] [node 5] ICS07 socket disconnect 438 reason# EndOfStream
2025-11-28T14:52:10.844858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789172482697211:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:10.844969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:10.845018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789172482697211:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:10.845703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789172482697174:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15693) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:10.845777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789172482697211:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:10.845832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789172482697174:2107] [node 5] ICP30 unregister session Session# [6:7577789172482697211:2107] VirtualId# [6:7577789172482697209:0]
2025-11-28T14:52:10.845880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:10.845914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789172482697174:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:10.845931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:10.946639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789171616060331:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:10.946680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789171616060332:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:10.967112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789173701529864:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:10.967207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789173701529865:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:10.986899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789172833926426:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:10.986943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789172833926427:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:11.016281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789178189027052:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:11.016346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789178189027054:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:11.016345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789178189027053:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:11.032521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789175915553205:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:11.032530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789175915553206:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:11.032562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789175915553207:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:11.054208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789176357922854:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:11.054219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789176357922855:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:11.054252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789176357922856:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:11.120890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789175911027690:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:11.155782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789176357922913:2103] [node 5] ICP01 ready to work
2025-11-28T14:52:11.166592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789177128893786:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:11.173608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789175911027691:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:11.170974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789176357922917:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:11.173671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789178189027115:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:11.174407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789177128893787:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:11.174746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789175915553269:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:11.174708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789175911027692:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:11.175209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789177996497223:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:11.175646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789177128893788:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:11.175029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789175915553270:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:11.175742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789178189027116:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:11.175994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789177996497224:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:11.176440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789177996497225:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:11.180163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789173701529865:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47432460ns) {Configure:104(47424430ns) {ProcessConfigured:135(237364ns) {StartInitialHandshake:157(235247ns) {DropHandshakes:470(2538ns) {DropIncomingHandshake:442(641ns) DropOutgoingHandshake:457(464ns)} GenerateSessionVirtualId:988(719ns) PrepareNewSessionHandshake:477(909ns) {SwitchToState:179(332ns)}}}}}
2025-11-28T14:52:11.180169Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789173701529864:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(56426402ns) {Configure:104(56418617ns) {ProcessConfigured:135(201114ns) {StartInitialHandshake:157(198230ns) {DropHandshakes:470(1609ns) {DropIncomingHandshake:442(483ns) DropOutgoingHandshake:457(346ns)} GenerateSessionVirtualId:988(841ns) PrepareNewSessionHandshake:477(983ns) {SwitchToState:179(389ns)}}}}}
2025-11-28T14:52:11.180957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789178189027054:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35798861ns) {Configure:104(35788846ns) {ProcessConfigured:135(392566ns) {StartInitialHandshake:157(389718ns) {DropHandshakes:470(1909ns) {DropIncomingHandshake:442(569ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:477(128137ns) {SwitchToState:179(1210ns)}}}}}
2025-11-28T14:52:11.181091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789176357922854:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42923510ns) {Configure:104(42918146ns) {ProcessConfigured:135(523726ns) {StartInitialHandshake:157(520519ns) {DropHandshakes:470(164650ns) {DropIncomingHandshake:442(411ns) DropOutgoingHandshake:457(633ns)} GenerateSessionVirtualId:988(845ns) PrepareNewSessionHandshake:477(1311ns) {SwitchToState:179(521ns)}}}}}
2025-11-28T14:52:11.180963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789178189027053:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43545958ns) {Configure:104(43537441ns) {ProcessConfigured:135(526187ns) {StartInitialHandshake:157(523937ns) {DropHandshakes:470(2642ns) {DropIncomingHandshake:442(652ns) DropOutgoingHandshake:457(453ns)} GenerateSessionVirtualId:988(964ns) PrepareNewSessionHandshake:477(1162ns) {SwitchToState:179(426ns)}}}}}
2025-11-28T14:52:11.182938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789175915553206:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45128930ns) {Configure:104(45122659ns) {ProcessConfigured:135(645646ns) {StartInitialHandshake:157(644226ns) {DropHandshakes:470(1213ns) {DropIncomingHandshake:442(416ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(591ns) {SwitchToState:179(87ns)}}}}}
2025-11-28T14:52:11.185040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789171616060331:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23532802ns) {Configure:104(13313107ns) {ProcessConfigured:135(353932ns) {StartInitialHandshake:157(351678ns) {DropHandshakes:470(1114ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(567ns) PrepareNewSessionHandshake:477(2343ns) {SwitchToState:179(995ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.186170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789177996497239:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59168
2025-11-28T14:52:11.186529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789177996497240:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59168
2025-11-28T14:52:11.186467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789178189027128:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56883
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.190006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789177128893798:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21826
2025-11-28T14:52:11.189650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789175915553285:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64557
2025-11-28T14:52:11.190546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789178189027131:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56883
No RDMA context found for GID: No RDMA context found for GID: 2025-11-28T14:52:11.190545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789175915553284:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64557
::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.193070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789177128893799:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21826
2025-11-28T14:52:11.193794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789177996497242:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59168
2025-11-28T14:52:11.194884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789177128893804:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21826
2025-11-28T14:52:11.195211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789178189027128:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.194911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789175915553284:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.194601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789177996497241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59168
2025-11-28T14:52:11.194911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789177128893798:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.195527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789178189027053:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.195187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789177996497240:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.196042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789177128893799:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.195466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789177996497239:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.201076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789172833926427:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.201097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789177996497231:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.201136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789177996497231:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.205408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789175911027704:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53262
2025-11-28T14:52:11.205635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789177996497229:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.205579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789175911027701:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.205588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789175911027705:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53262
2025-11-28T14:52:11.205959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789177996497229:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:11.207021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789175911027706:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53262
2025-11-28T14:52:11.207745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789177996497231:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:11.207061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027705:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.208032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789177996497229:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:11.207181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027704:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.209035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789177996497223:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.209014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789178189027133:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56883
2025-11-28T14:52:11.209037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789173701529865:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.209082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:11.209257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789177128893807:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21826
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.209152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789173701529865:2049] [node 3] ICP22 created new session: [2:7577789177996497248:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.209211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:11.209218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789177996497248:2049] [node 3] ICS09 handshake done sender: [2:7577789177996497229:2111] self: [2:7577789177996497226:0] peer: [3:7577789177128893806:0] socket: 381 qp: -1
2025-11-28T14:52:11.209248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789177996497223:2107] [node 4] ICP22 created new session: [2:7577789177996497249:2107]
2025-11-28T14:52:11.209398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789177128893808:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21826
2025-11-28T14:52:11.209427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789178189027134:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56883
2025-11-28T14:52:11.209273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789177996497248:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:11.209313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:11.209331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789177996497249:2107] [node 4] ICS09 handshake done sender: [2:7577789177996497231:2112] self: [2:7577789177996497230:0] peer: [4:7577789178189027132:0] socket: 383 qp: -1
2025-11-28T14:52:11.209356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789177996497249:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:11.209405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:11.210198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789177128893799:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:11.210208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789178189027128:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.210982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789177996497245:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59168
2025-11-28T14:52:11.211495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789178189027053:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.211642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789172833926427:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.211718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:11.211759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:11.211805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789172833926427:2049] [node 2] ICP22 created new session: [3:7577789177128893809:2049]
2025-11-28T14:52:11.211783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789178189027053:2049] [node 2] ICP22 created new session: [4:7577789178189027135:2049]
2025-11-28T14:52:11.212070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789177128893809:2049] [node 2] ICS09 handshake done sender: [3:7577789177128893799:2115] self: [3:7577789177128893806:0] peer: [2:7577789177996497226:0] socket: 387 qp: -1
2025-11-28T14:52:11.211928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789178189027135:2049] [node 2] ICS09 handshake done sender: [4:7577789178189027128:2114] self: [4:7577789178189027132:0] peer: [2:7577789177996497230:0] socket: 391 qp: -1
2025-11-28T14:52:11.212173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789177128893809:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:11.211963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789178189027135:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:11.212967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:11.212050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.213773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922933:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.216085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789176357922933:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.216318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789176357922855:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.216584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789177996497238:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.216642Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789177996497238:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:11.217450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789177996497238:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.218389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789177996497225:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.218763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789175915553274:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.219154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:11.225070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922935:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.225133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789177128893804:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.219208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789177996497225:2109] [node 6] ICP22 created new session: [2:7577789177996497255:2109]
2025-11-28T14:52:11.225123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789177996497255:2109] [node 6] ICS09 handshake done sender: [2:7577789177996497238:2114] self: [2:7577789177996497233:0] peer: [6:7577789176357922934:0] socket: 380 qp: -1
2025-11-28T14:52:11.226435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789177996497255:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:11.226506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:11.226840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789176357922933:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:11.227072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789176357922855:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.227125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:11.227230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789176357922855:2049] [node 2] ICP22 created new session: [6:7577789176357922936:2049]
2025-11-28T14:52:11.227683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789176357922936:2049] [node 2] ICS09 handshake done sender: [6:7577789176357922933:2116] self: [6:7577789176357922934:0] peer: [2:7577789177996497233:0] socket: 425 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.227727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789176357922936:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:11.227776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:11.227941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789175915553287:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64557
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.228139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789175915553288:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64557
2025-11-28T14:52:11.228159Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789176357922855:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:11.228394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789175915553287:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.228509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789175915553288:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.228656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789175915553206:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.228719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789175915553269:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.228926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789177996497234:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.228993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789177996497234:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:11.228960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789178189027127:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.228991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789178189027127:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:11.230268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789178189027127:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:11.230304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789177996497234:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.230440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789178189027115:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.230694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789175915553291:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64557
2025-11-28T14:52:11.230497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:11.230668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789177996497224:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.230543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789178189027115:2108] [node 5] ICP22 created new session: [4:7577789178189027139:2108]
2025-11-28T14:52:11.230810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:11.230776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789178189027139:2108] [node 5] ICS09 handshake done sender: [4:7577789178189027127:2113] self: [4:7577789178189027126:0] peer: [5:7577789175915553290:0] socket: 424 qp: -1
2025-11-28T14:52:11.230858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789177996497224:2108] [node 5] ICP22 created new session: [2:7577789177996497258:2108]
2025-11-28T14:52:11.230819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789178189027139:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:11.230850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:11.230934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789177996497258:2108] [node 5] ICS09 handshake done sender: [2:7577789177996497234:2113] self: [2:7577789177996497232:0] peer: [5:7577789175915553289:0] socket: 386 qp: -1
2025-11-28T14:52:11.230982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789177996497258:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:11.231017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.231300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789175915553292:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64557
2025-11-28T14:52:11.231341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922939:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.231613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789175915553288:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:11.232456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789175915553276:2113] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.232756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789175915553287:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:11.245784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789175915553269:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.245735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789176357922939:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.245798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789175915553206:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.245875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:11.245876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:11.245936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789175915553206:2049] [node 2] ICP22 created new session: [5:7577789175915553293:2049]
2025-11-28T14:52:11.245890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789177996497242:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.245940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789175915553269:2108] [node 4] ICP22 created new session: [5:7577789175915553294:2108]
2025-11-28T14:52:11.245995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789175915553293:2049] [node 2] ICS09 handshake done sender: [5:7577789175915553287:2118] self: [5:7577789175915553289:0] peer: [2:7577789177996497232:0] socket: 428 qp: -1
2025-11-28T14:52:11.245970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789176357922917:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.246008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789175915553294:2108] [node 4] ICS09 handshake done sender: [5:7577789175915553288:2119] self: [5:7577789175915553290:0] peer: [4:7577789178189027126:0] socket: 429 qp: -1
2025-11-28T14:52:11.246050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789175915553293:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:11.246094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789177996497224:2108] [node 5] ICP18 handshake (actor [2:7577789177996497242:2118]) is too old
2025-11-28T14:52:11.246056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789175915553294:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:11.246093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:11.246103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:11.246325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789178189027130:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.246363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789178189027130:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:11.247776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789178189027130:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.247978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789178189027116:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.248024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:11.248107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789178189027116:2109] [node 6] ICP22 created new session: [4:7577789178189027143:2109]
2025-11-28T14:52:11.248144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789178189027143:2109] [node 6] ICS09 handshake done sender: [4:7577789178189027130:2115] self: [4:7577789178189027129:0] peer: [6:7577789176357922940:0] socket: 433 qp: -1
2025-11-28T14:52:11.248089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922941:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.248177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789178189027143:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:11.248208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:11.248423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789176357922939:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:11.248684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789176357922917:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.248754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:11.248806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789176357922917:2108] [node 4] ICP22 created new session: [6:7577789176357922942:2108]
2025-11-28T14:52:11.248847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789176357922942:2108] [node 4] ICS09 handshake done sender: [6:7577789176357922939:2118] self: [6:7577789176357922940:0] peer: [4:7577789178189027129:0] socket: 434 qp: -1
2025-11-28T14:52:11.248884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789176357922942:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:11.248998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:11.251587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789175915553275:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.260349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789178189027131:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.260593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789178189027115:2108] [node 5] ICP18 handshake (actor [4:7577789178189027131:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.261206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922944:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.264715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789177996497228:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.314013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789177128893794:2113] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.324233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789177128893811:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21826
2025-11-28T14:52:11.324825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789177128893811:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.343575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789177128893793:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.344534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789177996497241:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.344839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789173701529865:2049] [node 3] ICP18 handshake (actor [2:7577789177996497241:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.355955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789178189027145:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56883
2025-11-28T14:52:11.373927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789175911027695:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.375093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027701:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.374765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789177996497245:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.375060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789173701529864:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.375381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027706:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.375648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027695:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.375874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789175911027712:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53262
2025-11-28T14:52:11.376391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789175911027695:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.384260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789177996497261:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59168
2025-11-28T14:52:11.384805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789177996497245:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:11.384162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789171616060331:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.384262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:11.384968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789173701529864:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.384320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789171616060331:2048] [node 2] ICP22 created new session: [1:7577789175911027716:2048]
2025-11-28T14:52:11.384385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789175911027716:2048] [node 2] ICS09 handshake done sender: [1:7577789175911027695:2110] self: [1:7577789175911027693:0] peer: [2:7577789177996497260:0] socket: 413 qp: -1
2025-11-28T14:52:11.384432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789175911027716:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:11.384480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:11.385068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:11.386066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789173701529864:2048] [node 1] ICP22 created new session: [2:7577789177996497262:2048]
2025-11-28T14:52:11.386477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789177996497262:2048] [node 1] ICS09 handshake done sender: [2:7577789177996497245:2119] self: [2:7577789177996497260:0] peer: [1:7577789175911027693:0] socket: 416 qp: -1
2025-11-28T14:52:11.386922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789177996497262:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:11.388035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.393064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922945:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.394823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789175911027698:2112] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.404944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789178189027133:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.405235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789178189027052:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.405638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027698:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.406796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789175911027698:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.407044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789178189027147:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56883
2025-11-28T14:52:11.408819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789175911027697:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.413788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789177128893807:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.414097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789175911027690:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.425684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789172833926426:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.426088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789178189027133:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:11.445730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789178189027052:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.446351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:11.447273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789178189027052:2048] [node 1] ICP22 created new session: [4:7577789178189027148:2048]
2025-11-28T14:52:11.434418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:11.452798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789177128893791:2111] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.436569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789175911027690:2107] [node 4] ICP22 created new session: [1:7577789175911027720:2107]
2025-11-28T14:52:11.447334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789178189027148:2048] [node 1] ICS09 handshake done sender: [4:7577789178189027133:2117] self: [4:7577789178189027146:0] peer: [1:7577789175911027696:0] socket: 417 qp: -1
2025-11-28T14:52:11.446271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789175911027720:2107] [node 4] ICS09 handshake done sender: [1:7577789175911027698:2112] self: [1:7577789175911027696:0] peer: [4:7577789178189027146:0] socket: 415 qp: -1
2025-11-28T14:52:11.447368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789178189027148:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.446323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789175911027720:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.457280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789175915553285:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.467744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789175915553299:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64557
2025-11-28T14:52:11.467679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789177128893816:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21826
2025-11-28T14:52:11.467818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789175915553207:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.447422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:11.470985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789177128893791:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.471190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789177128893807:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:11.452254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027697:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.472726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789172833926426:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.452710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:11.472852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:11.472914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789172833926426:2048] [node 1] ICP22 created new session: [3:7577789177128893818:2048]
2025-11-28T14:52:11.473201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789177128893818:2048] [node 1] ICS09 handshake done sender: [3:7577789177128893807:2119] self: [3:7577789177128893815:0] peer: [1:7577789175911027694:0] socket: 419 qp: -1
2025-11-28T14:52:11.473242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789177128893818:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:11.473289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:11.454071Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789175911027690:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(39556566ns) {HandleHandshakeStatus:329(39551311ns) {DropIncomingHandshake:442(32891ns) DropOutgoingHandshake:457(665ns) SwitchToState:179(574ns) ProcessPendingSessionEvents:495(236877ns) {ForwardSessionEventToSession:612(108685ns) ForwardSessionEventToSession:612(14888ns) ForwardSessionEventToSession:612(11079ns) ForwardSessionEventToSession:612(4600ns) ForwardSessionEventToSession:612(5135ns) ForwardSessionEventToSession:612(23254ns) ForwardSessionEventToSession:612(3892ns) ForwardSessionEventToSession:612(14115ns) ForwardSessionEventToSession:612(7844ns) ForwardSessionEventToSession:612(12513ns)}}}
2025-11-28T14:52:11.455233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789175911027697:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:11.468315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789171616060332:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.472196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:11.472332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789171616060332:2049] [node 3] ICP22 created new session: [1:7577789175911027724:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.472428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789175911027724:2049] [node 3] ICS09 handshake done sender: [1:7577789175911027697:2111] self: [1:7577789175911027694:0] peer: [3:7577789177128893815:0] socket: 414 qp: -1
2025-11-28T14:52:11.473284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789175911027724:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:11.473362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:11.474107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789175915553301:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64557
2025-11-28T14:52:11.475049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789177128893791:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:11.475275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789177128893786:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.475409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:11.475457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789177128893786:2108] [node 5] ICP22 created new session: [3:7577789177128893820:2108]
2025-11-28T14:52:11.475684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789177128893820:2108] [node 5] ICS09 handshake done sender: [3:7577789177128893791:2111] self: [3:7577789177128893789:0] peer: [5:7577789175915553300:0] socket: 392 qp: -1
2025-11-28T14:52:11.475854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789177128893820:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:11.475926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:11.476037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789175915553285:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:11.476567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789175915553207:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.476650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:11.476702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789175915553207:2050] [node 3] ICP22 created new session: [5:7577789175915553302:2050]
2025-11-28T14:52:11.476760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789175915553302:2050] [node 3] ICS09 handshake done sender: [5:7577789175915553285:2117] self: [5:7577789175915553300:0] peer: [3:7577789177128893789:0] socket: 400 qp: -1
2025-11-28T14:52:11.476792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789175915553302:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:11.476850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.477646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922946:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.478018Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789175915553207:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:11.480450Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789175915553282:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.507757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789175915553282:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.506033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789176357922944:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.506249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789176357922913:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.508422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789175915553282:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.507793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789177128893801:2116] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.513053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789175915553270:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.512811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789178189027145:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.513053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789178189027054:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.513156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:11.513216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789175915553270:2109] [node 6] ICP22 created new session: [5:7577789175915553305:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.513272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789175915553305:2109] [node 6] ICS09 handshake done sender: [5:7577789175915553282:2114] self: [5:7577789175915553281:0] peer: [6:7577789176357922947:0] socket: 402 qp: -1
2025-11-28T14:52:11.519675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789177128893801:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.513319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789175915553305:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:11.512852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922948:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.513364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:11.521347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789176357922944:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:11.521749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789178189027151:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56883
2025-11-28T14:52:11.520519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789177128893801:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:11.521175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789177128893787:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.522689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789178189027054:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16981) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789178189027145:2119] held: no
2025-11-28T14:52:11.521952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789176357922913:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.521248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:11.523231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:11.524784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789178189027145:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:11.521299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789177128893787:2107] [node 4] ICP22 created new session: [3:7577789177128893824:2107]
2025-11-28T14:52:11.525419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789178189027054:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.525538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:11.525656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789178189027054:2050] [node 3] ICP22 created new session: [4:7577789178189027152:2050]
2025-11-28T14:52:11.525804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789178189027152:2050] [node 3] ICS09 handshake done sender: [4:7577789178189027145:2119] self: [4:7577789178189027150:0] peer: [3:7577789177128893800:0] socket: 437 qp: -1
2025-11-28T14:52:11.523797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789176357922913:2103] [node 5] ICP22 created new session: [6:7577789176357922949:2103]
2025-11-28T14:52:11.521357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789177128893824:2107] [node 4] ICS09 handshake done sender: [3:7577789177128893801:2116] self: [3:7577789177128893800:0] peer: [4:7577789178189027150:0] socket: 399 qp: -1
2025-11-28T14:52:11.525845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789178189027152:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:11.524024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789176357922949:2103] [node 5] ICS09 handshake done sender: [6:7577789176357922944:2120] self: [6:7577789176357922947:0] peer: [5:7577789175915553281:0] socket: 436 qp: -1
2025-11-28T14:52:11.521399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789177128893824:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:11.524249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789176357922949:2103] [node 5] ICS10 traffic start
2025-11-28T14:52:11.521838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:11.525081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:11.527287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:11.533860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789175915553283:2115] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.541940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027712:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.544701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789175911027700:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.553188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789176357922945:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.553373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789176357922854:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.553617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027700:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.554058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789175911027700:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.554209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922952:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.554254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789175911027692:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.554342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:11.554388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789175911027692:2109] [node 6] ICP22 created new session: [1:7577789175911027728:2109]
2025-11-28T14:52:11.554429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789175911027728:2109] [node 6] ICS09 handshake done sender: [1:7577789175911027700:2113] self: [1:7577789175911027699:0] peer: [6:7577789176357922951:0] socket: 379 qp: -1
2025-11-28T14:52:11.554458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789175911027728:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:11.554453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789176357922945:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:11.554487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:11.554665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789176357922854:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.554826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:11.554909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789176357922854:2048] [node 1] ICP22 created new session: [6:7577789176357922953:2048]
2025-11-28T14:52:11.554980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789176357922953:2048] [node 1] ICS09 handshake done sender: [6:7577789176357922945:2121] self: [6:7577789176357922951:0] peer: [1:7577789175911027699:0] socket: 407 qp: -1
2025-11-28T14:52:11.555057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789176357922953:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:11.555117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:11.556576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789175911027703:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.564682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789175915553299:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.565000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789175915553205:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.565331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789175911027703:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.565868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789175911027703:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:11.566045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789175915553308:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64557
2025-11-28T14:52:11.566045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789175911027691:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.566122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:11.566195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789175911027691:2108] [node 5] ICP22 created new session: [1:7577789175911027731:2108]
2025-11-28T14:52:11.566231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789175911027731:2108] [node 5] ICS09 handshake done sender: [1:7577789175911027703:2115] self: [1:7577789175911027702:0] peer: [5:7577789175915553307:0] socket: 442 qp: -1
2025-11-28T14:52:11.566270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789175911027731:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:11.566349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789175915553299:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:11.566294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:11.566681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789175915553205:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.566730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:11.566777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789175915553205:2048] [node 1] ICP22 created new session: [5:7577789175915553309:2048]
2025-11-28T14:52:11.566915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789175915553309:2048] [node 1] ICS09 handshake done sender: [5:7577789175915553299:2122] self: [5:7577789175915553307:0] peer: [1:7577789175911027702:0] socket: 444 qp: -1
2025-11-28T14:52:11.566953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789175915553309:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:11.566998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:11.567233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789175915553205:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:11.568130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789177128893803:2117] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:11.568874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789176357922946:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:11.569035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789176357922856:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:11.569175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789177128893803:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:11.569681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789177128893803:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:11.569788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789176357922956:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30534
2025-11-28T14:52:11.569895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789177128893788:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:11.569974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789176357922946:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:11.569978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:11.570019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789177128893788:2110] [node 6] ICP22 created new session: [3:7577789177128893827:2110]
2025-11-28T14:52:11.570052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789177128893827:2110] [node 6] ICS09 handshake done sender: [3:7577789177128893803:2117] self: [3:7577789177128893802:0] peer: [6:7577789176357922955:0] socket: 393 qp: -1
2025-11-28T14:52:11.570195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789176357922856:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:11.570102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789177128893827:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:11.570313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:11.570316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:11.570392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789176357922856:2050] [node 3] ICP22 created new session: [6:7577789176357922957:2050]
2025-11-28T14:52:11.570510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789176357922957:2050] [node 3] ICS09 handshake done sender: [6:7577789176357922946:2122] self: [6:7577789176357922955:0] peer: [3:7577789177128893802:0] socket: 406 qp: -1
2025-11-28T14:52:11.571042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789176357922957:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:11.571120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:11.574565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:11.574770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789177996497262:2048] [node 1] ICS07 socket disconnect 416 reason# EndOfStream
2025-11-28T14:52:11.574804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789177996497262:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.574889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:11.574931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789177996497262:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:11.575856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:11.576005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789177128893818:2048] [node 1] ICS01 socket: 419 reason# EndOfStream
2025-11-28T14:52:11.576039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789172833926426:2048] [node 1] ICP30 unregister session Session# [3:7577789177128893818:2048] VirtualId# [3:7577789177128893815:0]
2025-11-28T14:52:11.576064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789177128893818:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.576119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:11.576159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:11.577310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789173701529864:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:11.577359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789177996497262:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:11.578751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:11.578910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789178189027148:2048] [node 1] ICS07 socket disconnect 417 reason# EndOfStream
2025-11-28T14:52:11.578938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789178189027148:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.578988Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789173701529864:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.578994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:11.579075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789177996497262:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.579046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789178189027148:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:11.579105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789173701529864:2048] [node 1] ICP30 unregister session Session# [2:7577789177996497262:2048] VirtualId# [2:7577789177996497260:0]
2025-11-28T14:52:11.579145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:11.579175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789173701529864:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.579191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.579596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789178189027052:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.579635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789178189027148:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.579689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789178189027052:2048] [node 1] ICP30 unregister session Session# [4:7577789178189027148:2048] VirtualId# [4:7577789178189027146:0]
2025-11-28T14:52:11.579726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:11.579748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789178189027052:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.579756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.579859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:11.580101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789175915553309:2048] [node 1] ICS07 socket disconnect 444 reason# EndOfStream
2025-11-28T14:52:11.580135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789175915553309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.580196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:11.580237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789175915553309:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:11.580856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789175915553205:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.580891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789175915553309:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.580925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:11.581004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789175915553205:2048] [node 1] ICP30 unregister session Session# [5:7577789175915553309:2048] VirtualId# [5:7577789175915553307:0]
2025-11-28T14:52:11.581057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:11.581091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789175915553205:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.581109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.581104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789176357922953:2048] [node 1] ICS07 socket disconnect 407 reason# EndOfStream
2025-11-28T14:52:11.581128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789176357922953:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.581174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:11.581200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789176357922953:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:11.581728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789176357922854:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.581815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789176357922953:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.581842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789176357922854:2048] [node 1] ICP30 unregister session Session# [6:7577789176357922953:2048] VirtualId# [6:7577789176357922951:0]
2025-11-28T14:52:11.581873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:11.581891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789176357922854:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.581898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.589278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789178189027052:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:11.591302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789176357922854:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:11.591311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789175915553205:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:11.614393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:11.614634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789177128893809:2049] [node 2] ICS07 socket disconnect 387 reason# EndOfStream
2025-11-28T14:52:11.614678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789177128893809:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.614792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:11.614850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789177128893809:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:11.617341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:11.617479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789172833926427:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:11.617522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789177128893809:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:11.617553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789178189027135:2049] [node 2] ICS07 socket disconnect 391 reason# EndOfStream
2025-11-28T14:52:11.617589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789178189027135:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.617665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:11.617704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789178189027135:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:11.618588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789172833926427:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.618623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789177128893809:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.618623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789178189027053:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.618720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789172833926427:2049] [node 2] ICP30 unregister session Session# [3:7577789177128893809:2049] VirtualId# [3:7577789177128893806:0]
2025-11-28T14:52:11.618848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789178189027135:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.618753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:11.618875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789178189027053:2049] [node 2] ICP30 unregister session Session# [4:7577789178189027135:2049] VirtualId# [4:7577789178189027132:0]
2025-11-28T14:52:11.618862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789172833926427:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.618923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:11.618880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.618942Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789178189027053:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.618902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:11.618956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.619119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789175915553293:2049] [node 2] ICS07 socket disconnect 428 reason# EndOfStream
2025-11-28T14:52:11.619158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789175915553293:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.619233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:11.619279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789175915553293:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:11.619672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:11.619797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789175915553206:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.619863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789176357922936:2049] [node 2] ICS07 socket disconnect 425 reason# EndOfStream
2025-11-28T14:52:11.619876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789175915553293:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.619916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789175915553206:2049] [node 2] ICP30 unregister session Session# [5:7577789175915553293:2049] VirtualId# [5:7577789175915553289:0]
2025-11-28T14:52:11.619894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789176357922936:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.619946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:11.619972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:11.619990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789176357922936:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:11.620021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789175915553206:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.620037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.620571Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789176357922855:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.620636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789176357922936:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.620669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789176357922855:2049] [node 2] ICP30 unregister session Session# [6:7577789176357922936:2049] VirtualId# [6:7577789176357922934:0]
2025-11-28T14:52:11.620728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:11.620755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789176357922855:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.620767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.628490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:11.628644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789178189027152:2050] [node 3] ICS01 socket: 437 reason# EndOfStream
2025-11-28T14:52:11.628674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789178189027054:2050] [node 3] ICP30 unregister session Session# [4:7577789178189027152:2050] VirtualId# [4:7577789178189027150:0]
2025-11-28T14:52:11.628695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789178189027152:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.628756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:11.628806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:11.629177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789178189027053:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:11.629570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:11.629773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789175915553302:2050] [node 3] ICS07 socket disconnect 400 reason# EndOfStream
2025-11-28T14:52:11.629805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789175915553302:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.629873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:11.629917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789175915553302:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:11.630174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789175915553206:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:11.630215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789176357922855:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:11.630475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:11.630514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789175915553207:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16981) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.630644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789176357922957:2050] [node 3] ICS07 socket disconnect 406 reason# EndOfStream
2025-11-28T14:52:11.630561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789175915553302:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.630666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789176357922957:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.630676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789175915553207:2050] [node 3] ICP30 unregister session Session# [5:7577789175915553302:2050] VirtualId# [5:7577789175915553300:0]
2025-11-28T14:52:11.630713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:11.630722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:11.630733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789175915553207:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16981) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.630756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789176357922957:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:11.630747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16981) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.631294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789176357922856:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16981) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.631395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789176357922957:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.631425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789176357922856:2050] [node 3] ICP30 unregister session Session# [6:7577789176357922957:2050] VirtualId# [6:7577789176357922955:0]
2025-11-28T14:52:11.631470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:11.631506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789176357922856:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16981) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.631522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16981) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.640506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789175915553207:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:11.641497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789176357922856:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:11.684245Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:11.684489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789175915553294:2108] [node 4] ICS07 socket disconnect 429 reason# EndOfStream
2025-11-28T14:52:11.684524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789175915553294:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.684645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:11.684701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789175915553294:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:11.685332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:11.685547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789176357922942:2108] [node 4] ICS07 socket disconnect 434 reason# EndOfStream
2025-11-28T14:52:11.685426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789175915553269:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13278) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.685579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789176357922942:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.685505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789175915553294:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.685645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:11.685568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789175915553269:2108] [node 4] ICP30 unregister session Session# [5:7577789175915553294:2108] VirtualId# [5:7577789175915553290:0]
2025-11-28T14:52:11.685696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789176357922942:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:11.685633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:11.685661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789175915553269:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.685671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.686409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789176357922917:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13278) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.686504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789176357922942:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.686569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789176357922917:2108] [node 4] ICP30 unregister session Session# [6:7577789176357922942:2108] VirtualId# [6:7577789176357922940:0]
2025-11-28T14:52:11.686618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:11.686644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789176357922917:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.686657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.696830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789176357922917:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:11.697396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:11.697616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789176357922949:2103] [node 5] ICS07 socket disconnect 436 reason# EndOfStream
2025-11-28T14:52:11.697653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789176357922949:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:11.697735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:11.697783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789176357922949:2103] [node 5] ICS15 start handshake
2025-11-28T14:52:11.698523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789176357922913:2103] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11772) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:11.698645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789176357922949:2103] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:11.698715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789176357922913:2103] [node 5] ICP30 unregister session Session# [6:7577789176357922949:2103] VirtualId# [6:7577789176357922947:0]
2025-11-28T14:52:11.698765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:11.698795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789176357922913:2103] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11772) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:11.698809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11772) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:11.772764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789175381135557:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:11.772792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789175381135558:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:11.793198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789176328905943:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:11.793209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789176328905942:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:11.812952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789175008251520:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:11.812996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789175008251521:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:11.843697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789175690763571:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:11.843725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789175690763573:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:11.843729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789175690763572:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:11.871990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789178740536508:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:11.872023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789178740536510:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:11.872150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789178740536509:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:11.901681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789174872039197:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:11.901735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789174872039199:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:11.901762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789174872039198:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:11.959781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789174872039258:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:11.981125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789175381135621:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:12.001431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789183035503868:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:12.004678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789179985730930:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:12.001270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789175381135627:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:12.007094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789179167006557:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:12.004723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789175381135628:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:12.007091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789179303218898:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:12.007336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789176328906032:2131] [node 6] ICP01 ready to work
2025-11-28T14:52:12.008162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789179985730931:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:12.007312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789183035503869:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:12.007458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789180623873329:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:12.008311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789180623873330:2110] [node 5] ICP01 ready to work
2025-11-28T14:52:12.008105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789179303218900:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:12.008106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789179303218899:2124] [node 6] ICP01 ready to work
2025-11-28T14:52:12.009571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789175381135557:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(50157472ns) {Configure:104(50151968ns) {ProcessConfigured:135(594786ns) {StartInitialHandshake:157(592425ns) {DropHandshakes:470(3265ns) {DropIncomingHandshake:442(844ns) DropOutgoingHandshake:457(581ns)} GenerateSessionVirtualId:988(621ns) PrepareNewSessionHandshake:477(16071ns) {SwitchToState:179(1137ns)}}}}}
2025-11-28T14:52:12.009618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789174872039258:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49295264ns) {Configure:104(49283598ns) {ProcessConfigured:135(547744ns) {StartInitialHandshake:157(544679ns) {DropHandshakes:470(1629ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(1258ns) PrepareNewSessionHandshake:477(2160ns) {SwitchToState:179(1258ns)}}}}}
2025-11-28T14:52:12.011361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789174872039198:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39034800ns) {Configure:104(39030583ns) {ProcessConfigured:135(386650ns) {StartInitialHandshake:157(383480ns) {DropHandshakes:470(2798ns) {DropIncomingHandshake:442(907ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:477(1121ns) {SwitchToState:179(401ns)}}}}}
2025-11-28T14:52:12.013144Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789176328905943:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47954492ns) {Configure:104(47945642ns) {ProcessConfigured:135(736837ns) {StartInitialHandshake:157(734489ns) {DropHandshakes:470(1624ns) {DropIncomingHandshake:442(717ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(362ns) PrepareNewSessionHandshake:477(1811ns) {SwitchToState:179(977ns)}}}}}
2025-11-28T14:52:12.014440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789176328905942:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48837949ns) {Configure:104(48832698ns) {ProcessConfigured:135(553048ns) {StartInitialHandshake:157(550554ns) {DropHandshakes:470(1112ns) {DropIncomingHandshake:442(392ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(964ns) PrepareNewSessionHandshake:477(2379ns) {SwitchToState:179(1228ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.032395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789183035503869:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16459240ns) {Configure:104(16455339ns) {ProcessConfigured:135(79892ns) {StartInitialHandshake:157(75990ns) {DropHandshakes:470(3058ns) {DropIncomingHandshake:442(669ns) DropOutgoingHandshake:457(415ns)} GenerateSessionVirtualId:988(1105ns) PrepareNewSessionHandshake:477(1424ns) {SwitchToState:179(575ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.033695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789180623873339:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5753
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.033611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789183035503879:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30997
2025-11-28T14:52:12.035719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789183035503878:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30997
2025-11-28T14:52:12.035731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789179676102938:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49757
2025-11-28T14:52:12.034962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789180623873341:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5753
2025-11-28T14:52:12.037175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789183035503878:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.039200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789180623873330:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22476123ns) {Configure:104(22471986ns) {ProcessConfigured:135(1948716ns) {StartInitialHandshake:157(1945517ns) {DropHandshakes:470(3032ns) {DropIncomingHandshake:442(717ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(967ns) PrepareNewSessionHandshake:477(6052ns) {SwitchToState:179(1024ns)}}}}}
2025-11-28T14:52:12.036727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789179303218909:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5978
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.039770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789179303218910:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5978
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.039424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789175690763572:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34920102ns) {Configure:104(34914232ns) {ProcessConfigured:135(361407ns) {StartInitialHandshake:157(359375ns) {DropHandshakes:470(1982ns) {DropIncomingHandshake:442(718ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(441ns) PrepareNewSessionHandshake:477(1874ns) {SwitchToState:179(897ns)}}}}}
2025-11-28T14:52:12.039441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789180623873341:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.037347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789178740536508:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.043818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789179303218899:2124] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24243336ns) {Configure:104(24239008ns) {ProcessConfigured:135(2806264ns) {StartInitialHandshake:157(2800480ns) {DropHandshakes:470(1137ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:477(1850ns) {SwitchToState:179(572ns)}}}}}
2025-11-28T14:52:12.039462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789180623873339:2141] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.039720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789175690763573:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29892363ns) {Configure:104(29885411ns) {ProcessConfigured:135(704963ns) {StartInitialHandshake:157(702402ns) {DropHandshakes:470(123317ns) {DropIncomingHandshake:442(691ns) DropOutgoingHandshake:457(343ns)} GenerateSessionVirtualId:988(682ns) PrepareNewSessionHandshake:477(1391ns) {SwitchToState:179(468ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.039270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789183035503879:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.044210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006571:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
2025-11-28T14:52:12.046340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789179676102928:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.042733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789175690763571:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35672911ns) {Configure:104(35669992ns) {ProcessConfigured:135(458135ns) {StartInitialHandshake:157(455604ns) {DropHandshakes:470(1161ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(441ns) PrepareNewSessionHandshake:477(250914ns) {SwitchToState:179(1673ns)}}}}}
2025-11-28T14:52:12.052833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789179303218909:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.043298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789180623873344:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5753
2025-11-28T14:52:12.046399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789179676102928:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:12.045776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789179985730930:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(34785232ns) {Configure:104(34781405ns) {ProcessConfigured:135(6320119ns) {StartInitialHandshake:157(6317746ns) {DropHandshakes:470(1468ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:477(1616ns) {SwitchToState:179(666ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.053337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789179303218910:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.067119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789179167006571:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.046573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789176328905942:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.066843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789179303218911:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5978
2025-11-28T14:52:12.071285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789175008251520:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.064237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789179676102926:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.064211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789180623873348:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5753
2025-11-28T14:52:12.064468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789180623873344:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.071551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789179303218912:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5978
2025-11-28T14:52:12.064284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789179676102926:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:12.067793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789179676102928:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:12.072030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789174872039258:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.071652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789179676102939:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49757
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.072206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789179985730939:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51265
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.072310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789179985730941:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51265
2025-11-28T14:52:12.072263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789183035503887:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30997
2025-11-28T14:52:12.072329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789179676102926:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:12.073072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789180623873351:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5753
2025-11-28T14:52:12.072520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789179676102930:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.073159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789183035503873:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.073211Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789183035503873:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:12.076337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789179303218912:2136] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.072552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789179676102930:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:12.076210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789183035503878:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:12.077533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789179303218920:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5978
2025-11-28T14:52:12.077539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789178740536508:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.075213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789175381135627:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.077796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:12.079602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789179985730931:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36146316ns) {Configure:104(36139628ns) {ProcessConfigured:135(502503ns) {StartInitialHandshake:157(500371ns) {DropHandshakes:470(1146ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(258ns)} GenerateSessionVirtualId:988(350ns) PrepareNewSessionHandshake:477(1084ns) {SwitchToState:179(425ns)}}}}}
2025-11-28T14:52:12.075566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789175381135557:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.078002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789183035503873:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.080726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789174872039258:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5497) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789179167006571:2115] held: no
2025-11-28T14:52:12.078417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789178740536508:2048] [node 1] ICP22 created new session: [5:7577789183035503889:2048]
2025-11-28T14:52:12.075606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:12.079163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789183035503889:2048] [node 1] ICS09 handshake done sender: [5:7577789183035503878:2115] self: [5:7577789183035503885:0] peer: [1:7577789179676102927:0] socket: 390 qp: -1
2025-11-28T14:52:12.079208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789183035503889:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:12.076617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789175381135557:2048] [node 2] ICP22 created new session: [1:7577789179676102943:2048]
2025-11-28T14:52:12.079209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789183035503868:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.077752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:12.079519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:12.082086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789180623873339:2141] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:12.077972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789175381135627:2109] [node 5] ICP22 created new session: [1:7577789179676102944:2109]
2025-11-28T14:52:12.079541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:12.082647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006573:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.082728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789179985730947:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51265
2025-11-28T14:52:12.077980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789179676102943:2048] [node 2] ICS09 handshake done sender: [1:7577789179676102926:2116] self: [1:7577789179676102925:0] peer: [2:7577789180623873349:0] socket: 385 qp: -1
2025-11-28T14:52:12.083485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789176328905942:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.086455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789179303218922:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5978
2025-11-28T14:52:12.078023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789179676102943:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:12.080193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789183035503868:2109] [node 6] ICP22 created new session: [5:7577789183035503891:2109]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:12.084447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:12.086790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789179167006571:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:12.080236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789183035503891:2109] [node 6] ICS09 handshake done sender: [5:7577789183035503873:2112] self: [5:7577789183035503871:0] peer: [6:7577789179167006572:0] socket: 403 qp: -1
2025-11-28T14:52:12.078078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.078449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789179676102944:2109] [node 5] ICS09 handshake done sender: [1:7577789179676102928:2117] self: [1:7577789179676102927:0] peer: [5:7577789183035503885:0] socket: 388 qp: -1
2025-11-28T14:52:12.080720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789183035503891:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:12.086773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789176328905942:2048] [node 1] ICP22 created new session: [2:7577789180623873353:2048]
2025-11-28T14:52:12.093333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789179303218910:2134] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:12.090425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789179985730947:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.087859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789174872039258:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.080949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:12.078504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789179676102944:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:12.087131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789180623873353:2048] [node 1] ICS09 handshake done sender: [2:7577789180623873339:2141] self: [2:7577789180623873349:0] peer: [1:7577789179676102925:0] socket: 395 qp: -1
2025-11-28T14:52:12.078897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:12.087914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:12.090623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789175690763571:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.087164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789180623873353:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:12.081999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789179676102930:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:12.094246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789179985730948:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51265
2025-11-28T14:52:12.087524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.087962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789174872039258:2106] [node 5] ICP22 created new session: [6:7577789179167006576:2106]
2025-11-28T14:52:12.082060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789179676102942:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49757
2025-11-28T14:52:12.088009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789179167006576:2106] [node 5] ICS09 handshake done sender: [6:7577789179167006571:2115] self: [6:7577789179167006572:0] peer: [5:7577789183035503871:0] socket: 404 qp: -1
2025-11-28T14:52:12.088045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789179167006576:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:12.083735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789175381135558:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.088081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:12.086789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:12.093518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006574:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
2025-11-28T14:52:12.087089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789175381135558:2049] [node 3] ICP22 created new session: [1:7577789179676102951:2049]
2025-11-28T14:52:12.099455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789179167006574:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.099665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789174872039197:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.088627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789179676102951:2049] [node 3] ICS09 handshake done sender: [1:7577789179676102930:2118] self: [1:7577789179676102929:0] peer: [3:7577789179303218919:0] socket: 389 qp: -1
2025-11-28T14:52:12.099210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789175008251520:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.099306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:12.099469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789175008251520:2048] [node 1] ICP22 created new session: [3:7577789179303218923:2048]
2025-11-28T14:52:12.099520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789179303218923:2048] [node 1] ICS09 handshake done sender: [3:7577789179303218910:2134] self: [3:7577789179303218919:0] peer: [1:7577789179676102929:0] socket: 396 qp: -1
2025-11-28T14:52:12.099561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789179303218923:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:12.099607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.103620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789176328905942:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16169894ns) {HandleHandshakeStatus:329(16167889ns) {DropIncomingHandshake:442(1444ns) DropOutgoingHandshake:457(4514ns) SwitchToState:179(522ns) ProcessPendingSessionEvents:495(2730847ns) {ForwardSessionEventToSession:612(1044057ns) ForwardSessionEventToSession:612(1264774ns) ForwardSessionEventToSession:612(3893ns) ForwardSessionEventToSession:612(254225ns) ForwardSessionEventToSession:612(5047ns) ForwardSessionEventToSession:612(2934ns) ForwardSessionEventToSession:612(124371ns) ForwardSessionEventToSession:612(3982ns) ForwardSessionEventToSession:612(4662ns) ForwardSessionEventToSession:612(5717ns)}}}
2025-11-28T14:52:12.088670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789179676102951:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:12.103786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789180623873352:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5753
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.106376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789179985730950:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51265
2025-11-28T14:52:12.094467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:12.096462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789179676102933:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.096503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789179676102933:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:12.099605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789179676102950:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49757
2025-11-28T14:52:12.100155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789179676102934:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.100181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789179676102934:2120] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:52:12.100235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789179676102933:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:12.107950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789179985730947:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:12.100426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789179676102950:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.103729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789175381135621:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.104649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
::ffff:127.1.0.0
2025-11-28T14:52:12.104729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789175381135621:2107] [node 4] ICP22 created new session: [1:7577789179676102955:2107]
2025-11-28T14:52:12.108492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006579:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
2025-11-28T14:52:12.104782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789179676102955:2107] [node 4] ICS09 handshake done sender: [1:7577789179676102933:2119] self: [1:7577789179676102931:0] peer: [4:7577789179985730949:0] socket: 432 qp: -1
2025-11-28T14:52:12.104818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789179676102955:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:12.104858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:12.106837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789179676102934:2120] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:12.108510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789175381135628:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.108591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:12.108666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789175381135628:2110] [node 6] ICP22 created new session: [1:7577789179676102958:2110]
2025-11-28T14:52:12.108708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789179676102958:2110] [node 6] ICS09 handshake done sender: [1:7577789179676102934:2120] self: [1:7577789179676102932:0] peer: [6:7577789179167006578:0] socket: 436 qp: -1
2025-11-28T14:52:12.108663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789175690763571:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.108747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789179676102958:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:12.108740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:12.108796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:12.108802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789175690763571:2048] [node 1] ICP22 created new session: [4:7577789179985730951:2048]
2025-11-28T14:52:12.108860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789179985730951:2048] [node 1] ICS09 handshake done sender: [4:7577789179985730947:2117] self: [4:7577789179985730949:0] peer: [1:7577789179676102931:0] socket: 433 qp: -1
2025-11-28T14:52:12.108964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789179985730951:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:12.109006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:12.109396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789179167006574:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:12.113551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789180623873335:2138] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.112706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789174872039197:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.112766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:12.115489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789179676102938:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.113001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789174872039197:2048] [node 1] ICP22 created new session: [6:7577789179167006580:2048]
2025-11-28T14:52:12.115733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789175381135557:2048] [node 2] ICP18 handshake (actor [1:7577789179676102938:2121]) is too old
2025-11-28T14:52:12.114006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789179167006580:2048] [node 1] ICS09 handshake done sender: [6:7577789179167006574:2117] self: [6:7577789179167006578:0] peer: [1:7577789179676102932:0] socket: 438 qp: -1
2025-11-28T14:52:12.115403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789179167006580:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:12.115665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.132535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006582:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
2025-11-28T14:52:12.132832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789174872039197:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19989159ns) {HandleHandshakeStatus:329(19985576ns) {DropIncomingHandshake:442(1420ns) DropOutgoingHandshake:457(5478ns) SwitchToState:179(476ns) ProcessPendingSessionEvents:495(1560502ns) {ForwardSessionEventToSession:612(611250ns) ForwardSessionEventToSession:612(11262ns) ForwardSessionEventToSession:612(2470ns) ForwardSessionEventToSession:612(89660ns) ForwardSessionEventToSession:612(4150ns) ForwardSessionEventToSession:612(3362ns) ForwardSessionEventToSession:612(805867ns) ForwardSessionEventToSession:612(5615ns) ForwardSessionEventToSession:612(3545ns) ForwardSessionEventToSession:612(9558ns)}}}
2025-11-28T14:52:12.132932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789174872039197:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.133441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789179985730953:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51265
2025-11-28T14:52:12.137282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789179303218911:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.136553Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789180623873332:2137] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.137434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789175008251521:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.138788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789180623873332:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.139225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789180623873332:2137] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:12.147929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789176328905943:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.147993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:12.155376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789176328905943:2049] [node 3] ICP22 created new session: [2:7577789180623873357:2049]
No RDMA context found for GID: 2025-11-28T14:52:12.163218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789180623873357:2049] [node 3] ICS09 handshake done sender: [2:7577789180623873332:2137] self: [2:7577789180623873331:0] peer: [3:7577789179303218925:0] socket: 398 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:12.163287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789180623873357:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:12.163387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:12.163450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789179303218926:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5978
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.164125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789183035503894:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30997
2025-11-28T14:52:12.164471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789179303218911:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:12.164924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789175008251521:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.165058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:12.165348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789175008251521:2049] [node 2] ICP22 created new session: [3:7577789179303218927:2049]
2025-11-28T14:52:12.165437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789179303218927:2049] [node 2] ICS09 handshake done sender: [3:7577789179303218911:2135] self: [3:7577789179303218925:0] peer: [2:7577789180623873331:0] socket: 402 qp: -1
2025-11-28T14:52:12.165483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789179303218927:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:12.165524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:12.171433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789180623873337:2139] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.180994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789179985730939:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.181202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789175690763572:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.181717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789180623873337:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.182173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789180623873337:2139] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:12.182457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789179985730955:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51265
2025-11-28T14:52:12.182710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789180623873329:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.182781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:12.182826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789180623873329:2107] [node 4] ICP22 created new session: [2:7577789180623873361:2107]
2025-11-28T14:52:12.182950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789180623873361:2107] [node 4] ICS09 handshake done sender: [2:7577789180623873337:2139] self: [2:7577789180623873334:0] peer: [4:7577789179985730954:0] socket: 405 qp: -1
2025-11-28T14:52:12.182999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789180623873361:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:12.183056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:12.183078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789179985730939:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:12.183358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789175690763572:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.183405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:12.183455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789175690763572:2049] [node 2] ICP22 created new session: [4:7577789179985730956:2049]
2025-11-28T14:52:12.183441Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789179303218906:2130] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.183555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789179985730956:2049] [node 2] ICS09 handshake done sender: [4:7577789179985730939:2113] self: [4:7577789179985730954:0] peer: [2:7577789180623873334:0] socket: 415 qp: -1
2025-11-28T14:52:12.186620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789179985730956:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:12.186684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:12.200862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789175690763572:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17169303ns) {HandleHandshakeStatus:329(17166875ns) {DropIncomingHandshake:442(499ns) DropOutgoingHandshake:457(4967ns) SwitchToState:179(308ns) ProcessPendingSessionEvents:495(13661379ns) {ForwardSessionEventToSession:612(8971250ns) ForwardSessionEventToSession:612(277469ns) ForwardSessionEventToSession:612(3291ns) ForwardSessionEventToSession:612(3221ns) ForwardSessionEventToSession:612(4356978ns) ForwardSessionEventToSession:612(4568ns) ForwardSessionEventToSession:612(10846ns) ForwardSessionEventToSession:612(2942ns) ForwardSessionEventToSession:612(3666ns) ForwardSessionEventToSession:612(5108ns)}}}
2025-11-28T14:52:12.201148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789179985730941:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.201918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789175690763573:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.202297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789179303218906:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.202831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789179303218906:2130] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.203039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789179303218898:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.204455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:12.205626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789179985730959:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51265
2025-11-28T14:52:12.206078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789179303218898:2107] [node 4] ICP22 created new session: [3:7577789179303218930:2107]
2025-11-28T14:52:12.206158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789179303218930:2107] [node 4] ICS09 handshake done sender: [3:7577789179303218906:2130] self: [3:7577789179303218904:0] peer: [4:7577789179985730958:0] socket: 414 qp: -1
2025-11-28T14:52:12.206204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789179303218930:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:12.206254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:12.206633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789179985730941:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:12.206916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789175690763573:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.206967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:12.207010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789175690763573:2050] [node 3] ICP22 created new session: [4:7577789179985730960:2050]
2025-11-28T14:52:12.207082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789179985730960:2050] [node 3] ICS09 handshake done sender: [4:7577789179985730941:2115] self: [4:7577789179985730958:0] peer: [3:7577789179303218904:0] socket: 416 qp: -1
2025-11-28T14:52:12.207117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789179985730960:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:12.207151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.208833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789183035503895:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30997
2025-11-28T14:52:12.225137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789179303218905:2129] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.263270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789179676102939:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.263537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789175381135558:2049] [node 3] ICP18 handshake (actor [1:7577789179676102939:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.265123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006584:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
2025-11-28T14:52:12.274124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789179985730938:2112] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.304532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789179303218920:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.304832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789179303218898:2107] [node 4] ICP18 handshake (actor [3:7577789179303218920:2138]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.314492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789183035503896:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30997
2025-11-28T14:52:12.330518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789179303218917:2137] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.331502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789179167006584:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.331661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789174872039199:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.332120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789179303218917:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.344273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006586:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
2025-11-28T14:52:12.344175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789179303218917:2137] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:12.355051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789179303218899:2124] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.355171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:12.355229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789179303218899:2124] [node 6] ICP22 created new session: [3:7577789179303218935:2124]
2025-11-28T14:52:12.369992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789179303218935:2124] [node 6] ICS09 handshake done sender: [3:7577789179303218917:2137] self: [3:7577789179303218915:0] peer: [6:7577789179167006585:0] socket: 411 qp: -1
2025-11-28T14:52:12.370064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789179303218935:2124] [node 6] ICS10 traffic start
2025-11-28T14:52:12.370133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:12.355127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789179167006584:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:12.370430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789174872039199:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.375678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:12.376278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789174872039199:2050] [node 3] ICP22 created new session: [6:7577789179167006587:2050]
2025-11-28T14:52:12.383047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789179167006587:2050] [node 3] ICS09 handshake done sender: [6:7577789179167006584:2120] self: [6:7577789179167006585:0] peer: [3:7577789179303218915:0] socket: 412 qp: -1
2025-11-28T14:52:12.383119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789179167006587:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:12.383180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:12.383581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789174872039199:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:12.384639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789179303218899:2124] [node 6] ICP35 event processing took too much time PendingConnection:279(39457544ns) {HandleHandshakeStatus:329(29540046ns) {DropIncomingHandshake:442(25780ns) DropOutgoingHandshake:457(635ns) SwitchToState:179(927ns) ProcessPendingSessionEvents:495(8146424ns) {ForwardSessionEventToSession:612(8080904ns) ForwardSessionEventToSession:612(12567ns) ForwardSessionEventToSession:612(3003ns) ForwardSessionEventToSession:612(3072ns) ForwardSessionEventToSession:612(5196ns) ForwardSessionEventToSession:612(2443ns) ForwardSessionEventToSession:612(3070ns) ForwardSessionEventToSession:612(8722ns) ForwardSessionEventToSession:612(2394ns) ForwardSessionEventToSession:612(14160ns)}}}
2025-11-28T14:52:12.386925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789179985730936:2111] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.400269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789179676102942:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.404655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789175381135621:2107] [node 4] ICP18 handshake (actor [1:7577789179676102942:2123]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.408695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789183035503883:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.421463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006589:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
2025-11-28T14:52:12.446581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789179985730948:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.455199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789179985730933:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.456426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789180623873352:2147] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.456659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789180623873329:2107] [node 4] ICP18 handshake (actor [2:7577789180623873352:2147]) is too old
2025-11-28T14:52:12.498780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789180623873338:2140] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.530234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789179167006582:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.530487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789174872039198:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.535590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789180623873338:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.546578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789180623873338:2140] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.546215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789179167006570:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.559400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006591:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
2025-11-28T14:52:12.558960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789176328906032:2131] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.559363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:12.562542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789179985730953:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.560647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789176328906032:2131] [node 6] ICP22 created new session: [2:7577789180623873364:2131]
2025-11-28T14:52:12.566097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789179167006582:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:12.565844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789183035503894:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.563569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789180623873364:2131] [node 6] ICS09 handshake done sender: [2:7577789180623873338:2140] self: [2:7577789180623873336:0] peer: [6:7577789179167006590:0] socket: 439 qp: -1
2025-11-28T14:52:12.566458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789174872039198:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30998) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789179167006582:2119] held: no
2025-11-28T14:52:12.575702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789174872039198:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.595114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:12.597416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789174872039198:2049] [node 2] ICP22 created new session: [6:7577789179167006592:2049]
2025-11-28T14:52:12.566878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789178740536509:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.598021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789183035503898:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30997
2025-11-28T14:52:12.598333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789183035503894:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:12.565003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789180623873346:2144] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.597930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789179167006592:2049] [node 2] ICS09 handshake done sender: [6:7577789179167006582:2119] self: [6:7577789179167006590:0] peer: [2:7577789180623873336:0] socket: 401 qp: -1
2025-11-28T14:52:12.566207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789180623873364:2131] [node 6] ICS10 traffic start
2025-11-28T14:52:12.597973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789179167006592:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:12.601055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789178740536509:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.566265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:12.566453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789180623873346:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.588732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789180623873346:2144] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:12.597974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789176328906032:2131] [node 6] ICP35 event processing took too much time PendingConnection:279(38786775ns) {HandleHandshakeStatus:329(38782853ns) {DropIncomingHandshake:442(24252ns) DropOutgoingHandshake:457(475ns) SwitchToState:179(511ns) ProcessPendingSessionEvents:495(13012509ns) {ForwardSessionEventToSession:612(9814766ns) ForwardSessionEventToSession:612(545143ns) ForwardSessionEventToSession:612(3522ns) ForwardSessionEventToSession:612(2599729ns) ForwardSessionEventToSession:612(4676ns) ForwardSessionEventToSession:612(3723ns) ForwardSessionEventToSession:612(8527ns) ForwardSessionEventToSession:612(2601ns) ForwardSessionEventToSession:612(3093ns) ForwardSessionEventToSession:612(9270ns)}}}
2025-11-28T14:52:12.598551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789180623873330:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.598623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:12.598667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789180623873330:2110] [node 5] ICP22 created new session: [2:7577789180623873367:2110]
2025-11-28T14:52:12.598719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789180623873367:2110] [node 5] ICS09 handshake done sender: [2:7577789180623873346:2144] self: [2:7577789180623873345:0] peer: [5:7577789183035503897:0] socket: 446 qp: -1
2025-11-28T14:52:12.599901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789180623873367:2110] [node 5] ICS10 traffic start
2025-11-28T14:52:12.598057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:12.601164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:12.600753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:12.601447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789178740536509:2049] [node 2] ICP22 created new session: [5:7577789183035503899:2049]
2025-11-28T14:52:12.601527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789183035503899:2049] [node 2] ICS09 handshake done sender: [5:7577789183035503894:2119] self: [5:7577789183035503897:0] peer: [2:7577789180623873345:0] socket: 447 qp: -1
2025-11-28T14:52:12.604216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789183035503899:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:12.604282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:12.606895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789178740536509:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:12.613283Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789179303218907:2131] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.630902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789179303218908:2132] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.635876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789180623873348:2145] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.636046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789176328905943:2049] [node 3] ICP18 handshake (actor [2:7577789180623873348:2145]) is too old
2025-11-28T14:52:12.598286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789174872039198:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(22530921ns) {HandleHandshakeStatus:329(22527936ns) {DropIncomingHandshake:442(653ns) DropOutgoingHandshake:457(287ns) SwitchToState:179(494ns) ProcessPendingSessionEvents:495(1694ns)}}
2025-11-28T14:52:12.639101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789183035503895:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.639325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789178740536510:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.640642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789179303218907:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.641267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789179303218907:2131] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.648929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789179303218900:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.659645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:12.659702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789183035503902:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30997
2025-11-28T14:52:12.659742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789179303218900:2108] [node 5] ICP22 created new session: [3:7577789179303218938:2108]
2025-11-28T14:52:12.659819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789179303218938:2108] [node 5] ICS09 handshake done sender: [3:7577789179303218907:2131] self: [3:7577789179303218903:0] peer: [5:7577789183035503901:0] socket: 452 qp: -1
2025-11-28T14:52:12.659859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789179303218938:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:12.667696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:12.668176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789183035503895:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:12.670581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789179303218900:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19258140ns) {HandleHandshakeStatus:329(19254062ns) {DropIncomingHandshake:442(26629ns) DropOutgoingHandshake:457(543ns) SwitchToState:179(918ns) ProcessPendingSessionEvents:495(181882ns) {ForwardSessionEventToSession:612(94595ns) ForwardSessionEventToSession:612(19575ns) ForwardSessionEventToSession:612(2935ns) ForwardSessionEventToSession:612(12689ns) ForwardSessionEventToSession:612(3111ns) ForwardSessionEventToSession:612(3211ns) ForwardSessionEventToSession:612(13102ns) ForwardSessionEventToSession:612(2949ns) ForwardSessionEventToSession:612(3078ns) ForwardSessionEventToSession:612(17503ns)}}}
2025-11-28T14:52:12.671165Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789178740536510:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23063) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789183035503895:2120] held: no
2025-11-28T14:52:12.672192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789178740536510:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.672262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:12.672315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789178740536510:2050] [node 3] ICP22 created new session: [5:7577789183035503903:2050]
2025-11-28T14:52:12.672350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789183035503903:2050] [node 3] ICS09 handshake done sender: [5:7577789183035503895:2120] self: [5:7577789183035503901:0] peer: [3:7577789179303218903:0] socket: 453 qp: -1
2025-11-28T14:52:12.672404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789183035503903:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:12.672444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:12.677900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789179985730940:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.691823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789183035503896:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.692905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789179985730940:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.692912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789183035503869:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.694389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789179167006589:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:12.694583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789179167006557:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:12.693421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789179985730946:2116] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:12.693484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789179985730940:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.695365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789179985730946:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:12.697050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789179167006595:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45085
2025-11-28T14:52:12.698658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789179167006589:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:12.695578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789183035503906:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30997
2025-11-28T14:52:12.695601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789179985730930:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.703546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789179167006557:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.703611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:12.703657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789179167006557:2109] [node 4] ICP22 created new session: [6:7577789179167006596:2109]
2025-11-28T14:52:12.703708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789179167006596:2109] [node 4] ICS09 handshake done sender: [6:7577789179167006589:2122] self: [6:7577789179167006594:0] peer: [4:7577789179985730945:0] socket: 394 qp: -1
2025-11-28T14:52:12.703745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789179167006596:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:12.703787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:12.695722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:12.695765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789179985730930:2108] [node 5] ICP22 created new session: [4:7577789179985730968:2108]
2025-11-28T14:52:12.695833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789179985730968:2108] [node 5] ICS09 handshake done sender: [4:7577789179985730940:2114] self: [4:7577789179985730937:0] peer: [5:7577789183035503905:0] socket: 454 qp: -1
2025-11-28T14:52:12.695872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789179985730968:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:12.696046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:12.696661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789179985730946:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:12.699122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789179985730931:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:12.699190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:12.699240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789179985730931:2109] [node 6] ICP22 created new session: [4:7577789179985730970:2109]
2025-11-28T14:52:12.699283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789179985730970:2109] [node 6] ICS09 handshake done sender: [4:7577789179985730946:2116] self: [4:7577789179985730945:0] peer: [6:7577789179167006594:0] socket: 384 qp: -1
2025-11-28T14:52:12.699320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789179985730970:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:12.699353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:12.698530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789183035503896:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:12.698597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789183035503869:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789183035503896:2121] held: no
2025-11-28T14:52:12.700675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789183035503869:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:12.703674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:12.703730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789183035503869:2108] [node 4] ICP22 created new session: [5:7577789183035503907:2108]
2025-11-28T14:52:12.703766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789183035503907:2108] [node 4] ICS09 handshake done sender: [5:7577789183035503896:2121] self: [5:7577789183035503905:0] peer: [4:7577789179985730937:0] socket: 424 qp: -1
2025-11-28T14:52:12.703791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789183035503907:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:12.703827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:12.705430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789179167006557:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:12.749950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:12.750129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789180623873353:2048] [node 1] ICS07 socket disconnect 395 reason# EndOfStream
2025-11-28T14:52:12.750158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789180623873353:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.750237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:12.750274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789180623873353:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:12.751243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:12.751455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789179303218923:2048] [node 1] ICS07 socket disconnect 396 reason# EndOfStream
2025-11-28T14:52:12.751491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789179303218923:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.751568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:12.751627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789179303218923:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:12.752709Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789176328905942:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:12.752757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789180623873353:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:12.754350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789175008251520:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:12.754407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789179303218923:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:12.755261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:12.755624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789179985730951:2048] [node 1] ICS07 socket disconnect 433 reason# EndOfStream
2025-11-28T14:52:12.755662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789179985730951:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.755752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:12.755848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789179985730951:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:12.755791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789176328905942:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:12.755985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789180623873353:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:12.756027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789176328905942:2048] [node 1] ICP30 unregister session Session# [2:7577789180623873353:2048] VirtualId# [2:7577789180623873349:0]
2025-11-28T14:52:12.756097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:12.756144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789176328905942:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.756125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789175008251520:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:12.756159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.756237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789179303218923:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:12.756276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789175008251520:2048] [node 1] ICP30 unregister session Session# [3:7577789179303218923:2048] VirtualId# [3:7577789179303218919:0]
2025-11-28T14:52:12.756330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:12.756357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789175008251520:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.756858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:12.756388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.756831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789175690763571:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:12.757084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789183035503889:2048] [node 1] ICS01 socket: 390 reason# EndOfStream
2025-11-28T14:52:12.756887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789179985730951:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:12.757141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789178740536508:2048] [node 1] ICP30 unregister session Session# [5:7577789183035503889:2048] VirtualId# [5:7577789183035503885:0]
2025-11-28T14:52:12.757174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789183035503889:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.756948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789175690763571:2048] [node 1] ICP30 unregister session Session# [4:7577789179985730951:2048] VirtualId# [4:7577789179985730949:0]
2025-11-28T14:52:12.756992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:12.757273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:12.757035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789175690763571:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.757333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:12.757050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.758202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:12.758425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789179167006580:2048] [node 1] ICS07 socket disconnect 438 reason# EndOfStream
2025-11-28T14:52:12.758458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789179167006580:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.758534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:12.758596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789179167006580:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:12.759264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789174872039197:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:12.759331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789179167006580:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:12.759369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789174872039197:2048] [node 1] ICP30 unregister session Session# [6:7577789179167006580:2048] VirtualId# [6:7577789179167006578:0]
2025-11-28T14:52:12.759447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:12.759485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789174872039197:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.759500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.765756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:12.765828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789175008251520:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:12.765939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789179303218927:2049] [node 2] ICS07 socket disconnect 402 reason# EndOfStream
2025-11-28T14:52:12.765967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789179303218927:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.766061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:12.766114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789179303218927:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:12.766873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789175690763571:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:12.767771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:12.767834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789175008251521:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30998) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:12.767906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789179985730956:2049] [node 2] ICS07 socket disconnect 415 reason# EndOfStream
2025-11-28T14:52:12.767958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789179985730956:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.767874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789179303218927:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:12.768043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:12.768086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789179985730956:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:12.768837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789175008251521:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30998) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:12.768913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789174872039197:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:12.768924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789179303218927:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:12.768958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789175008251521:2049] [node 2] ICP30 unregister session Session# [3:7577789179303218927:2049] VirtualId# [3:7577789179303218925:0]
2025-11-28T14:52:12.769018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:12.769051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789175008251521:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30998) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.769066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30998) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.769225Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:12.769112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789175690763572:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30998) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:12.769157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789179985730956:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:12.769229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789175690763572:2049] [node 2] ICP30 unregister session Session# [4:7577789179985730956:2049] VirtualId# [4:7577789179985730954:0]
2025-11-28T14:52:12.769438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789183035503899:2049] [node 2] ICS01 socket: 447 reason# EndOfStream
2025-11-28T14:52:12.769267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:12.769491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789178740536509:2049] [node 2] ICP30 unregister session Session# [5:7577789183035503899:2049] VirtualId# [5:7577789183035503897:0]
2025-11-28T14:52:12.769287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789175690763572:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30998) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.769539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789183035503899:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.769295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30998) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.769627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:12.769711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:12.770325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:12.770451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789179167006592:2049] [node 2] ICS01 socket: 401 reason# EndOfStream
2025-11-28T14:52:12.770482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789174872039198:2049] [node 2] ICP30 unregister session Session# [6:7577789179167006592:2049] VirtualId# [6:7577789179167006590:0]
2025-11-28T14:52:12.770504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789179167006592:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.770559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:12.770596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:12.778761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:12.778911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789179985730960:2050] [node 3] ICS07 socket disconnect 416 reason# EndOfStream
2025-11-28T14:52:12.778934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789179985730960:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.779002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:12.779044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789179985730960:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:12.779128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789175690763572:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:12.779643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:12.779773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789175690763573:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23063) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:12.779785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789183035503903:2050] [node 3] ICS01 socket: 453 reason# EndOfStream
2025-11-28T14:52:12.779847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789178740536510:2050] [node 3] ICP30 unregister session Session# [5:7577789183035503903:2050] VirtualId# [5:7577789183035503901:0]
2025-11-28T14:52:12.779855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789179985730960:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:12.779870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789183035503903:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.779926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789175690763573:2050] [node 3] ICP30 unregister session Session# [4:7577789179985730960:2050] VirtualId# [4:7577789179985730958:0]
2025-11-28T14:52:12.779919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:12.779987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:12.779951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:12.780023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789175690763573:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23063) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.780039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23063) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.780574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:12.780715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789179167006587:2050] [node 3] ICS01 socket: 412 reason# EndOfStream
2025-11-28T14:52:12.780745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789174872039199:2050] [node 3] ICP30 unregister session Session# [6:7577789179167006587:2050] VirtualId# [6:7577789179167006585:0]
2025-11-28T14:52:12.780764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789179167006587:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.780802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:12.780840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:12.788458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:12.788622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789183035503907:2108] [node 4] ICS01 socket: 424 reason# EndOfStream
2025-11-28T14:52:12.788655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789183035503869:2108] [node 4] ICP30 unregister session Session# [5:7577789183035503907:2108] VirtualId# [5:7577789183035503905:0]
2025-11-28T14:52:12.788681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789183035503907:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.788740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:12.788775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:12.789409Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:12.789522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789179167006596:2109] [node 4] ICS07 socket disconnect 394 reason# EndOfStream
2025-11-28T14:52:12.789543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789179167006596:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.789585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:12.789615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789179167006596:2109] [node 4] ICS15 start handshake
2025-11-28T14:52:12.790259Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789179167006557:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16840) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:12.790297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789179167006596:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:12.790360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789179167006557:2109] [node 4] ICP30 unregister session Session# [6:7577789179167006596:2109] VirtualId# [6:7577789179167006594:0]
2025-11-28T14:52:12.790396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:12.790420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789179167006557:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16840) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.790444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16840) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:12.800681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789179167006557:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:52:12.835484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:12.835744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789179167006576:2106] [node 5] ICS01 socket: 404 reason# EndOfStream
2025-11-28T14:52:12.835792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789174872039258:2106] [node 5] ICP30 unregister session Session# [6:7577789179167006576:2106] VirtualId# [6:7577789179167006572:0]
2025-11-28T14:52:12.835839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789179167006576:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:12.835934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:12.835979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:12.937723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789181720415503:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:12.938060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789181720415504:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:12.965655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789179615083383:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:12.965709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789179615083384:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:12.996174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789179419444667:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:12.996196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789179419444666:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:13.029680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789184318148029:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:13.029737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789184318148031:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:13.029774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789184318148030:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:13.052893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789184009069166:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:13.052956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789184009069168:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:13.053242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789184009069167:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:13.078841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789185192890150:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:13.079192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789185192890148:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:13.079627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789185192890149:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:13.162737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789186015382863:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:13.169757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789183910050743:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:13.216113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789185192890205:2103] [node 5] ICP01 ready to work
2025-11-28T14:52:13.216241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789183714412043:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:13.220776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789186015382866:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:13.225010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789183910050752:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:13.226371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789183714412044:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:13.228241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789184318148092:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:13.228306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789186015382867:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:13.228201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789183714412045:2126] [node 6] ICP01 ready to work
2025-11-28T14:52:13.228639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789185192890218:2104] [node 4] ICP01 ready to work
2025-11-28T14:52:13.236898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789184009069230:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:13.226312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789183910050753:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.244833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789179615083383:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21130750ns) {Configure:104(21122820ns) {ProcessConfigured:135(66801ns) {StartInitialHandshake:157(62587ns) {DropHandshakes:470(3607ns) {DropIncomingHandshake:442(702ns) DropOutgoingHandshake:457(877ns)} GenerateSessionVirtualId:988(652ns) PrepareNewSessionHandshake:477(1514ns) {SwitchToState:179(492ns)}}}}}
2025-11-28T14:52:13.245112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789183910050754:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3637
2025-11-28T14:52:13.231848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789184318148093:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:13.235054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789184009069231:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.248876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789184318148031:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47792350ns) {Configure:104(47786940ns) {ProcessConfigured:135(5453764ns) {StartInitialHandshake:157(5450384ns) {DropHandshakes:470(6077ns) {DropIncomingHandshake:442(879ns) DropOutgoingHandshake:457(417ns)} GenerateSessionVirtualId:988(849ns) PrepareNewSessionHandshake:477(4180ns) {SwitchToState:179(1557ns)}}}}}
2025-11-28T14:52:13.259610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789184009069168:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(68206945ns) {Configure:104(68201023ns) {ProcessConfigured:135(69904ns) {StartInitialHandshake:157(66305ns) {DropHandshakes:470(2462ns) {DropIncomingHandshake:442(538ns) DropOutgoingHandshake:457(466ns)} GenerateSessionVirtualId:988(850ns) PrepareNewSessionHandshake:477(1496ns) {SwitchToState:179(662ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.266701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789186015382875:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21292
2025-11-28T14:52:13.260103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789184009069234:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18281
2025-11-28T14:52:13.268297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789183910050754:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.267147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789186015382866:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34786339ns) {Configure:104(34486222ns) {ProcessConfigured:135(49412ns) {StartInitialHandshake:157(46795ns) {DropHandshakes:470(2855ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(491ns)} GenerateSessionVirtualId:988(1132ns) PrepareNewSessionHandshake:477(1650ns) {SwitchToState:179(671ns)}}}}}
2025-11-28T14:52:13.268348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789184318148030:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(63986992ns) {Configure:104(63974967ns) {ProcessConfigured:135(7101496ns) {StartInitialHandshake:157(7098224ns) {DropHandshakes:470(3293ns) {DropIncomingHandshake:442(796ns) DropOutgoingHandshake:457(472ns)} GenerateSessionVirtualId:988(1076ns) PrepareNewSessionHandshake:477(3414ns) {SwitchToState:179(1626ns)}}}}}
2025-11-28T14:52:13.267392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789186015382863:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(85378280ns) {Configure:104(85373640ns) {ProcessConfigured:135(7089556ns) {StartInitialHandshake:157(7086670ns) {DropHandshakes:470(3781ns) {DropIncomingHandshake:442(856ns) DropOutgoingHandshake:457(698ns)} GenerateSessionVirtualId:988(1192ns) PrepareNewSessionHandshake:477(7514ns) {SwitchToState:179(801ns)}}}}}
2025-11-28T14:52:13.271079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789179615083384:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43763357ns) {Configure:104(43759871ns) {ProcessConfigured:135(108327ns) {StartInitialHandshake:157(106767ns) {DropHandshakes:470(2410ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(697ns)} GenerateSessionVirtualId:988(929ns) PrepareNewSessionHandshake:477(2031ns) {SwitchToState:179(667ns)}}}}}
2025-11-28T14:52:13.270983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789186015382875:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.272149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789179615083383:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.275124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789184318148092:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(39964446ns) {Configure:104(39960216ns) {ProcessConfigured:135(1347245ns) {StartInitialHandshake:157(1344331ns) {DropHandshakes:470(834ns) {DropIncomingHandshake:442(183ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(400ns) PrepareNewSessionHandshake:477(1607ns) {SwitchToState:179(305ns)}}}}}
2025-11-28T14:52:13.275555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789185192890205:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49446670ns) {Configure:104(49438751ns) {ProcessConfigured:135(878152ns) {StartInitialHandshake:157(876428ns) {DropHandshakes:470(1430ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(442ns)} GenerateSessionVirtualId:988(545ns) PrepareNewSessionHandshake:477(2089ns) {SwitchToState:179(1342ns)}}}}}
2025-11-28T14:52:13.276064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789184009069167:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47027407ns) {Configure:104(47022289ns) {ProcessConfigured:135(577665ns) {StartInitialHandshake:157(576561ns) {DropHandshakes:470(829ns) {DropIncomingHandshake:442(182ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:477(1720ns) {SwitchToState:179(353ns)}}}}}
2025-11-28T14:52:13.273302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789183714412045:2126] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(38733550ns) {Configure:104(38728276ns) {ProcessConfigured:135(3498443ns) {StartInitialHandshake:157(1682076ns) {DropHandshakes:470(4671ns) {DropIncomingHandshake:442(661ns) DropOutgoingHandshake:457(500ns)} GenerateSessionVirtualId:988(2318ns) PrepareNewSessionHandshake:477(3664ns) {SwitchToState:179(1848ns)}}}}}
2025-11-28T14:52:13.276074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789186015382860:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.278944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789184009069166:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(58790750ns) {Configure:104(58784485ns) {ProcessConfigured:135(3509839ns) {StartInitialHandshake:157(3507513ns) {DropHandshakes:470(1325ns) {DropIncomingHandshake:442(374ns) DropOutgoingHandshake:457(363ns)} GenerateSessionVirtualId:988(545ns) PrepareNewSessionHandshake:477(2641ns) {SwitchToState:179(1210ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.273358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789179419444666:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(101763022ns) {Configure:104(101758495ns) {ProcessConfigured:135(799101ns) {StartInitialHandshake:157(797167ns) {DropHandshakes:470(624ns) {DropIncomingHandshake:442(129ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(383ns) PrepareNewSessionHandshake:477(1496ns) {SwitchToState:179(593ns)}}}}}
2025-11-28T14:52:13.278023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789185192890149:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49670342ns) {Configure:104(49663790ns) {ProcessConfigured:135(1090027ns) {StartInitialHandshake:157(1083337ns) {DropHandshakes:470(1105ns) {DropIncomingHandshake:442(332ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(1190ns) {SwitchToState:179(524ns)}}}}}
2025-11-28T14:52:13.276123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789186015382860:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:13.282885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789184318148029:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(54432444ns) {Configure:104(51746758ns) {ProcessConfigured:135(1784425ns) {StartInitialHandshake:157(1782210ns) {DropHandshakes:470(1075ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(1290ns) {SwitchToState:179(250ns)}}}}}
2025-11-28T14:52:13.275578Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789179419444667:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(71171516ns) {Configure:104(71154024ns) {ProcessConfigured:135(573494ns) {StartInitialHandshake:157(570619ns) {DropHandshakes:470(1728ns) {DropIncomingHandshake:442(445ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:477(1146ns) {SwitchToState:179(439ns)}}}}}
2025-11-28T14:52:13.281796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789184009069234:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.295553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789183910050765:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3637
2025-11-28T14:52:13.291591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789185192890150:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(53964350ns) {Configure:104(53958023ns) {ProcessConfigured:135(5272189ns) {StartInitialHandshake:157(5269112ns) {DropHandshakes:470(1189ns) {DropIncomingHandshake:442(310ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(978ns) PrepareNewSessionHandshake:477(3020ns) {SwitchToState:179(487ns)}}}}}
2025-11-28T14:52:13.295588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789183910050743:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(57258851ns) {Configure:104(57249096ns) {ProcessConfigured:135(6592546ns) {StartInitialHandshake:157(6590646ns) {DropHandshakes:470(1202ns) {DropIncomingHandshake:442(371ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(911ns) PrepareNewSessionHandshake:477(2698ns) {SwitchToState:179(1486ns)}}}}}
2025-11-28T14:52:13.280941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789181720415504:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(107574322ns) {Configure:104(107564057ns) {ProcessConfigured:135(1226917ns) {StartInitialHandshake:157(1225471ns) {DropHandshakes:470(1145ns) {DropIncomingHandshake:442(333ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(1375ns) {SwitchToState:179(86ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.291609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789185192890148:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(93844161ns) {Configure:104(93838024ns) {ProcessConfigured:135(3265699ns) {StartInitialHandshake:157(3263829ns) {DropHandshakes:470(1072ns) {DropIncomingHandshake:442(261ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:477(1832ns) {SwitchToState:179(840ns)}}}}}
2025-11-28T14:52:13.295726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789184318148093:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39677189ns) {Configure:104(39659564ns) {ProcessConfigured:135(7581076ns) {StartInitialHandshake:157(7578220ns) {DropHandshakes:470(1264ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(546ns) PrepareNewSessionHandshake:477(3573ns) {SwitchToState:179(1522ns)}}}}}
2025-11-28T14:52:13.295086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789183714412044:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49020435ns) {Configure:104(49011639ns) {ProcessConfigured:135(8705612ns) {StartInitialHandshake:157(8702620ns) {DropHandshakes:470(1453ns) {DropIncomingHandshake:442(523ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:477(3269ns) {SwitchToState:179(1879ns)}}}}}
2025-11-28T14:52:13.296558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789184009069231:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53945756ns) {Configure:104(53940960ns) {ProcessConfigured:135(5159652ns) {StartInitialHandshake:157(5156834ns) {DropHandshakes:470(3663ns) {DropIncomingHandshake:442(735ns) DropOutgoingHandshake:457(894ns)} GenerateSessionVirtualId:988(1069ns) PrepareNewSessionHandshake:477(2085ns) {SwitchToState:179(1171ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.297572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789185192890218:2104] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(46135425ns) {Configure:104(46125342ns) {ProcessConfigured:135(7862712ns) {StartInitialHandshake:157(7861013ns) {DropHandshakes:470(609ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(2198ns) {SwitchToState:179(495ns)}}}}}
2025-11-28T14:52:13.299299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789183714412043:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48065509ns) {Configure:104(48055592ns) {ProcessConfigured:135(2009417ns) {StartInitialHandshake:157(2006928ns) {DropHandshakes:470(1581ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(683ns)} GenerateSessionVirtualId:988(407ns) PrepareNewSessionHandshake:477(2354ns) {SwitchToState:179(1056ns)}}}}}
2025-11-28T14:52:13.290823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789186015382860:2107] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:13.297462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789184009069166:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.295617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789181720415503:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.301116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789183910050753:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55003404ns) {Configure:104(54997862ns) {ProcessConfigured:135(12784856ns) {StartInitialHandshake:157(12782478ns) {DropHandshakes:470(2661ns) {DropIncomingHandshake:442(591ns) DropOutgoingHandshake:457(493ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:477(10125ns) {SwitchToState:179(933ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.300791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789184318148104:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9079
2025-11-28T14:52:13.305799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789183910050766:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3637
2025-11-28T14:52:13.299333Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789184009069230:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(41360230ns) {Configure:104(41353305ns) {ProcessConfigured:135(2926875ns) {StartInitialHandshake:157(2923686ns) {DropHandshakes:470(2110ns) {DropIncomingHandshake:442(421ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(560ns) PrepareNewSessionHandshake:477(1975ns) {SwitchToState:179(853ns)}}}}}
2025-11-28T14:52:13.296923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:13.307196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789183714412053:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24933
2025-11-28T14:52:13.299390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789181720415503:2048] [node 2] ICP22 created new session: [1:7577789186015382882:2048]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:13.307534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789184318148104:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.302350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789186015382869:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.306985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789183910050752:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(64351962ns) {Configure:104(64343457ns) {ProcessConfigured:135(5120332ns) {StartInitialHandshake:157(5116133ns) {DropHandshakes:470(2319ns) {DropIncomingHandshake:442(457ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(700ns) PrepareNewSessionHandshake:477(2489ns) {SwitchToState:179(1168ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 2025-11-28T14:52:13.307256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789183714412055:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24933
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.307304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789179615083383:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789183910050754:2119] held: no
2025-11-28T14:52:13.302388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789186015382869:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:13.302759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789186015382882:2048] [node 2] ICS09 handshake done sender: [1:7577789186015382860:2107] self: [1:7577789186015382859:0] peer: [2:7577789183910050761:0] socket: 390 qp: -1
2025-11-28T14:52:13.309717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789184009069243:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18281
2025-11-28T14:52:13.309949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890233:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.307502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789183714412061:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24933
2025-11-28T14:52:13.308934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789184318148029:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.303926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789186015382882:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:13.310214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890232:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.305406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:13.310460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789183714412055:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.309726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789184318148108:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9079
2025-11-28T14:52:13.310836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890235:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.308383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789183910050754:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:13.308237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789186015382869:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.309027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789179615083383:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.330868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789185192890233:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.309253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789186015382867:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.310864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789183714412053:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.309069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:13.330681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789184009069248:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18281
2025-11-28T14:52:13.341497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789185192890148:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:13.310170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789186015382885:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21292
2025-11-28T14:52:13.327533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789183714412062:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24933
2025-11-28T14:52:13.309125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789179615083383:2048] [node 1] ICP22 created new session: [2:7577789183910050773:2048]
2025-11-28T14:52:13.310889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.343398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789183714412061:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.311856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789186015382867:2109] [node 5] ICP22 created new session: [1:7577789186015382886:2109]
2025-11-28T14:52:13.309442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789183910050766:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.343640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789184009069249:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18281
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.311994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789186015382872:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.310169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789183910050773:2048] [node 1] ICS09 handshake done sender: [2:7577789183910050754:2119] self: [2:7577789183910050761:0] peer: [1:7577789186015382859:0] socket: 391 qp: -1
2025-11-28T14:52:13.344798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890238:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.345046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789179419444667:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.312035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789186015382872:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:13.343959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789184009069234:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.346111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789185192890238:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.319925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789186015382886:2109] [node 5] ICS09 handshake done sender: [1:7577789186015382869:2111] self: [1:7577789186015382868:0] peer: [5:7577789184009069246:0] socket: 393 qp: -1
2025-11-28T14:52:13.327495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789186015382886:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:13.310983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789183910050773:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:13.345806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789183714412065:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24933
2025-11-28T14:52:13.346365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789185192890149:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.311701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:13.345084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789184009069166:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.346160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789184318148110:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9079
2025-11-28T14:52:13.339251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789183910050772:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3637
2025-11-28T14:52:13.330742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:13.342116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789183910050771:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3637
2025-11-28T14:52:13.342211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789181720415503:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19454823ns) {HandleHandshakeStatus:329(19452089ns) {DropIncomingHandshake:442(36423ns) DropOutgoingHandshake:457(865ns) SwitchToState:179(1172ns) ProcessPendingSessionEvents:495(1895812ns) {ForwardSessionEventToSession:612(237176ns) ForwardSessionEventToSession:612(230934ns) ForwardSessionEventToSession:612(2736ns) ForwardSessionEventToSession:612(251552ns) ForwardSessionEventToSession:612(3193ns) ForwardSessionEventToSession:612(2234ns) ForwardSessionEventToSession:612(730510ns) ForwardSessionEventToSession:612(6659ns) ForwardSessionEventToSession:612(3370ns) ForwardSessionEventToSession:612(412364ns)}}}
2025-11-28T14:52:13.345136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.345512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789184009069166:2048] [node 1] ICP22 created new session: [5:7577789184009069250:2048]
2025-11-28T14:52:13.345784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789184009069250:2048] [node 1] ICS09 handshake done sender: [5:7577789184009069234:2112] self: [5:7577789184009069246:0] peer: [1:7577789186015382868:0] socket: 398 qp: -1
2025-11-28T14:52:13.342556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789179615083383:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18623691ns) {HandleHandshakeStatus:329(18621701ns) {DropIncomingHandshake:442(501ns) DropOutgoingHandshake:457(461ns) SwitchToState:179(442ns) ProcessPendingSessionEvents:495(1358ns)}}
2025-11-28T14:52:13.342275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789186015382872:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:13.348449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789184318148104:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:13.346178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789184009069250:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:13.348454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890240:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.343135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789186015382874:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.343497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789183910050774:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3637
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.349273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789184318148029:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.343155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789186015382874:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:13.349332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:13.346226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:13.343580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789186015382863:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.344754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789183910050772:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.349371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789184318148029:2048] [node 1] ICP22 created new session: [4:7577789184318148111:2048]
2025-11-28T14:52:13.345906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789183910050759:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.345498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789186015382874:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:13.349530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789184318148111:2048] [node 1] ICS09 handshake done sender: [4:7577789184318148104:2114] self: [4:7577789184318148109:0] peer: [1:7577789186015382871:0] socket: 402 qp: -1
2025-11-28T14:52:13.345505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:13.345929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789183910050759:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:13.345694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789186015382863:2108] [node 4] ICP22 created new session: [1:7577789186015382891:2108]
2025-11-28T14:52:13.349571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789184318148111:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:13.345771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789186015382891:2108] [node 4] ICS09 handshake done sender: [1:7577789186015382872:2112] self: [1:7577789186015382871:0] peer: [4:7577789184318148109:0] socket: 401 qp: -1
2025-11-28T14:52:13.347719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789183910050770:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.349757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:13.345808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789186015382867:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(35880778ns) {HandleHandshakeStatus:329(35878246ns) {DropIncomingHandshake:442(383ns) DropOutgoingHandshake:457(466ns) SwitchToState:179(783ns) ProcessPendingSessionEvents:495(2496054ns) {ForwardSessionEventToSession:612(966639ns) ForwardSessionEventToSession:612(168692ns) ForwardSessionEventToSession:612(3469ns) ForwardSessionEventToSession:612(171653ns) ForwardSessionEventToSession:612(2686ns) ForwardSessionEventToSession:612(2406ns) ForwardSessionEventToSession:612(745595ns) ForwardSessionEventToSession:612(2572ns) ForwardSessionEventToSession:612(152249ns) ForwardSessionEventToSession:612(267625ns)}}}
2025-11-28T14:52:13.347747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789183910050770:2126] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:13.350208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789183714412067:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24933
2025-11-28T14:52:13.346174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789186015382891:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:13.346220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:13.348312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789183910050759:2121] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.346241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789186015382866:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.346745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:13.351268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789185192890233:2125] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:13.348818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789179615083384:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.347534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789186015382866:2110] [node 6] ICP22 created new session: [1:7577789186015382893:2110]
2025-11-28T14:52:13.356534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789183714412061:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:13.349189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:13.347733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789186015382893:2110] [node 6] ICS09 handshake done sender: [1:7577789186015382874:2113] self: [1:7577789186015382873:0] peer: [6:7577789185192890239:0] socket: 416 qp: -1
2025-11-28T14:52:13.359570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789185192890148:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.359907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:13.359945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789179419444667:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.347763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789186015382893:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:13.349604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789179615083384:2049] [node 3] ICP22 created new session: [2:7577789183910050780:2049]
2025-11-28T14:52:13.356489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789184318148113:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9079
2025-11-28T14:52:13.348119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:13.359866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789184009069252:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18281
2025-11-28T14:52:13.360518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:13.360514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789185192890148:2048] [node 1] ICP22 created new session: [6:7577789185192890243:2048]
2025-11-28T14:52:13.349788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789183910050780:2049] [node 3] ICS09 handshake done sender: [2:7577789183910050759:2121] self: [2:7577789183910050758:0] peer: [3:7577789183714412066:0] socket: 415 qp: -1
2025-11-28T14:52:13.361008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789179419444667:2049] [node 2] ICP22 created new session: [3:7577789183714412068:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.360848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789184318148113:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.361026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789185192890243:2048] [node 1] ICS09 handshake done sender: [6:7577789185192890233:2125] self: [6:7577789185192890239:0] peer: [1:7577789186015382873:0] socket: 396 qp: -1
2025-11-28T14:52:13.349915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789183910050780:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:13.361256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789185192890243:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:13.361041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789183714412068:2049] [node 2] ICS09 handshake done sender: [3:7577789183714412061:2135] self: [3:7577789183714412066:0] peer: [2:7577789183910050758:0] socket: 419 qp: -1
2025-11-28T14:52:13.350089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:13.361291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890242:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.350089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789183910050770:2126] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:13.361063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789183714412068:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:13.361308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:13.361260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789184318148030:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.350943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789183910050752:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.361148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:13.350988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:13.351284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789183910050752:2109] [node 6] ICP22 created new session: [2:7577789183910050784:2109]
2025-11-28T14:52:13.351709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789183910050784:2109] [node 6] ICS09 handshake done sender: [2:7577789183910050770:2126] self: [2:7577789183910050769:0] peer: [6:7577789185192890241:0] socket: 436 qp: -1
2025-11-28T14:52:13.351755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789183910050784:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:13.351802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:13.361976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789183910050768:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.362012Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789183910050768:2125] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:13.362476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789185192890238:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:13.362753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789185192890149:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.362814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:13.362848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789185192890149:2049] [node 2] ICP22 created new session: [6:7577789185192890245:2049]
2025-11-28T14:52:13.362880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789185192890245:2049] [node 2] ICS09 handshake done sender: [6:7577789185192890238:2127] self: [6:7577789185192890241:0] peer: [2:7577789183910050769:0] socket: 437 qp: -1
2025-11-28T14:52:13.362902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789185192890245:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:13.362920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:13.363350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789183910050768:2125] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.363971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789183910050753:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.364058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:13.364103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789184318148116:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9079
2025-11-28T14:52:13.364269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789183910050753:2107] [node 4] ICP22 created new session: [2:7577789183910050787:2107]
2025-11-28T14:52:13.364338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789183910050787:2107] [node 4] ICS09 handshake done sender: [2:7577789183910050768:2125] self: [2:7577789183910050764:0] peer: [4:7577789184318148115:0] socket: 446 qp: -1
2025-11-28T14:52:13.364263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789184318148117:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9079
2025-11-28T14:52:13.364564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789183910050787:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:13.364841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789184318148030:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13582) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789184318148113:2118] held: no
2025-11-28T14:52:13.364602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:13.365976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789184318148113:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:13.366136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789184318148030:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.366171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:13.366292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789184318148030:2049] [node 2] ICP22 created new session: [4:7577789184318148118:2049]
2025-11-28T14:52:13.366334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789184318148118:2049] [node 2] ICS09 handshake done sender: [4:7577789184318148113:2118] self: [4:7577789184318148115:0] peer: [2:7577789183910050764:0] socket: 448 qp: -1
2025-11-28T14:52:13.366350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789184318148118:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:13.366418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:13.370763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789184318148100:2112] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.379435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789185192890232:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.379577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789185192890218:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.379827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789184318148100:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.380235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789184318148100:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.380457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789184318148092:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.380401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890249:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.380574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:13.380643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789184318148092:2109] [node 6] ICP22 created new session: [4:7577789184318148121:2109]
2025-11-28T14:52:13.380690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789184318148121:2109] [node 6] ICS09 handshake done sender: [4:7577789184318148100:2112] self: [4:7577789184318148099:0] peer: [6:7577789185192890248:0] socket: 410 qp: -1
2025-11-28T14:52:13.380725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789184318148121:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:13.380831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:13.380870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789185192890232:2124] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:13.381055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789185192890218:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.381087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:13.381116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789185192890218:2104] [node 4] ICP22 created new session: [6:7577789185192890250:2104]
2025-11-28T14:52:13.381170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789185192890250:2104] [node 4] ICS09 handshake done sender: [6:7577789185192890232:2124] self: [6:7577789185192890248:0] peer: [4:7577789184318148099:0] socket: 413 qp: -1
2025-11-28T14:52:13.381206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789185192890250:2104] [node 4] ICS10 traffic start
2025-11-28T14:52:13.381244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:13.381289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789183910050762:2122] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.381956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789184009069243:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.382127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789184009069167:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.382366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789183910050762:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.384737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789183910050762:2122] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.390386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789183910050743:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.390497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:13.390490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789184009069254:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18281
2025-11-28T14:52:13.390560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789183910050743:2108] [node 5] ICP22 created new session: [2:7577789183910050790:2108]
2025-11-28T14:52:13.390629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789183910050790:2108] [node 5] ICS09 handshake done sender: [2:7577789183910050762:2122] self: [2:7577789183910050760:0] peer: [5:7577789184009069253:0] socket: 407 qp: -1
2025-11-28T14:52:13.390669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789183910050790:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:13.390722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:13.390770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789184009069243:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:13.391180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789184009069167:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13582) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789184009069243:2116] held: no
2025-11-28T14:52:13.391445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789184009069167:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.391511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:13.391551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789184009069167:2049] [node 2] ICP22 created new session: [5:7577789184009069255:2049]
2025-11-28T14:52:13.391592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789184009069255:2049] [node 2] ICS09 handshake done sender: [5:7577789184009069243:2116] self: [5:7577789184009069253:0] peer: [2:7577789183910050760:0] socket: 411 qp: -1
2025-11-28T14:52:13.391631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789184009069255:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:13.391670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.393703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890252:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.396721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789184009069239:2114] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.405032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789186015382885:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.405292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789186015382867:2109] [node 5] ICP18 handshake (actor [1:7577789186015382885:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.405622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789184318148123:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9079
2025-11-28T14:52:13.406955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789183714412056:2133] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.419330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789184318148108:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.419532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789184318148031:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.419796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789183714412056:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.420215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789183714412056:2133] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:13.424719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789183714412043:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.424855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.424899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789183714412043:2107] [node 4] ICP22 created new session: [3:7577789183714412071:2107]
2025-11-28T14:52:13.425096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789183714412071:2107] [node 4] ICS09 handshake done sender: [3:7577789183714412056:2133] self: [3:7577789183714412052:0] peer: [4:7577789184318148124:0] socket: 421 qp: -1
2025-11-28T14:52:13.425312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789184318148125:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9079
2025-11-28T14:52:13.425139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789183714412071:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:13.425210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:13.425615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789184318148031:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25953) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789184318148108:2116] held: no
2025-11-28T14:52:13.426454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789184318148108:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.426744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789184318148031:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.426783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:13.426897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789184009069259:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18281
2025-11-28T14:52:13.426861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789184318148031:2050] [node 3] ICP22 created new session: [4:7577789184318148126:2050]
2025-11-28T14:52:13.426899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789184318148126:2050] [node 3] ICS09 handshake done sender: [4:7577789184318148108:2116] self: [4:7577789184318148124:0] peer: [3:7577789183714412052:0] socket: 422 qp: -1
2025-11-28T14:52:13.426941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789184318148126:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:13.427018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:13.448709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789183714412048:2128] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.469943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789185192890235:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.470118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789185192890150:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.470435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789183714412048:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.470949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789183714412048:2128] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:13.471144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789183714412045:2126] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.471232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:13.471265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789183714412045:2126] [node 6] ICP22 created new session: [3:7577789183714412075:2126]
2025-11-28T14:52:13.471364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890254:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.471405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789183714412075:2126] [node 6] ICS09 handshake done sender: [3:7577789183714412048:2128] self: [3:7577789183714412046:0] peer: [6:7577789185192890253:0] socket: 417 qp: -1
2025-11-28T14:52:13.471450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789183714412075:2126] [node 6] ICS10 traffic start
2025-11-28T14:52:13.471478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.472048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789185192890235:2126] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:13.472194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789186015382895:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21292
2025-11-28T14:52:13.472290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789185192890150:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.472349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:13.472416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789185192890150:2050] [node 3] ICP22 created new session: [6:7577789185192890255:2050]
2025-11-28T14:52:13.472463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789185192890255:2050] [node 3] ICS09 handshake done sender: [6:7577789185192890235:2126] self: [6:7577789185192890253:0] peer: [3:7577789183714412046:0] socket: 420 qp: -1
2025-11-28T14:52:13.472494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789185192890255:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:13.472532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:13.478776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789183714412049:2129] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.505440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789183910050771:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.505695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789179615083384:2049] [node 3] ICP18 handshake (actor [2:7577789183910050771:2127]) is too old
2025-11-28T14:52:13.526713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789185192890230:2122] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.553494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789183714412062:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.553927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789183714412045:2126] [node 6] ICP18 handshake (actor [3:7577789183714412062:2136]) is too old
2025-11-28T14:52:13.559158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789185192890225:2120] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.579964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789183910050774:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.580234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789183910050752:2109] [node 6] ICP18 handshake (actor [2:7577789183910050774:2129]) is too old
2025-11-28T14:52:13.586314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789185192890224:2119] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.587342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789184009069249:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.625403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789186015382879:2115] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.631017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789183714412065:2137] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.631239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789179419444666:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.631551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789186015382879:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.632237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789186015382879:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:13.641169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789181720415504:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.661064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [4:7577789184318148105:2115] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.677222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789183714412079:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24933
2025-11-28T14:52:13.669120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789184009069252:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.676767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789184009069231:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.677438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:13.678636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789185192890231:2123] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.677533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789181720415504:2049] [node 3] ICP22 created new session: [1:7577789186015382897:2049]
2025-11-28T14:52:13.679653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789184318148105:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.679469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789183714412065:2137] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.695731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789184009069262:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18281
2025-11-28T14:52:13.677586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789186015382897:2049] [node 3] ICS09 handshake done sender: [1:7577789186015382879:2115] self: [1:7577789186015382877:0] peer: [3:7577789183714412078:0] socket: 438 qp: -1
2025-11-28T14:52:13.693008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789179419444666:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.684942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789184318148116:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.677625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789186015382897:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:13.677688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:13.695073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789181720415504:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(38299934ns) {HandleHandshakeStatus:329(38296221ns) {DropIncomingHandshake:442(620ns) DropOutgoingHandshake:457(596ns) SwitchToState:179(5009ns) ProcessPendingSessionEvents:495(882536ns) {ForwardSessionEventToSession:612(829050ns) ForwardSessionEventToSession:612(3899ns) ForwardSessionEventToSession:612(9610ns) ForwardSessionEventToSession:612(2841ns) ForwardSessionEventToSession:612(5737ns) ForwardSessionEventToSession:612(5848ns) ForwardSessionEventToSession:612(2652ns) ForwardSessionEventToSession:612(3165ns) ForwardSessionEventToSession:612(5304ns) ForwardSessionEventToSession:612(2455ns)}}}
2025-11-28T14:52:13.698737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789184009069252:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:13.693552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:13.695246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789179419444666:2048] [node 1] ICP22 created new session: [3:7577789183714412080:2048]
2025-11-28T14:52:13.696403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789183714412080:2048] [node 1] ICS09 handshake done sender: [3:7577789183714412065:2137] self: [3:7577789183714412078:0] peer: [1:7577789186015382877:0] socket: 439 qp: -1
2025-11-28T14:52:13.697046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789183714412080:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:13.698008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:13.692727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789184318148105:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:13.696025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789184318148092:2109] [node 6] ICP18 handshake (actor [4:7577789184318148116:2119]) is too old
2025-11-28T14:52:13.698151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789184318148093:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.698671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:13.700487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789184318148093:2108] [node 5] ICP22 created new session: [4:7577789184318148129:2108]
2025-11-28T14:52:13.700806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789184318148129:2108] [node 5] ICS09 handshake done sender: [4:7577789184318148105:2115] self: [4:7577789184318148101:0] peer: [5:7577789184009069261:0] socket: 450 qp: -1
2025-11-28T14:52:13.700845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789184318148129:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:13.700886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:13.702026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789184009069231:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.702093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:13.702142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789184009069231:2108] [node 4] ICP22 created new session: [5:7577789184009069263:2108]
2025-11-28T14:52:13.702184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789184009069263:2108] [node 4] ICS09 handshake done sender: [5:7577789184009069252:2120] self: [5:7577789184009069261:0] peer: [4:7577789184318148101:0] socket: 451 qp: -1
2025-11-28T14:52:13.702212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789184009069263:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:13.702248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:13.706633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789184009069241:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.741433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789185192890252:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.742801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789185192890205:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.742447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789184009069244:2117] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.745335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789184318148123:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.743215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789184009069241:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.764478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789184318148093:2108] [node 5] ICP18 handshake (actor [4:7577789184318148123:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.750097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789184009069241:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:13.788736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789185192890258:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57631
2025-11-28T14:52:13.776484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789184009069230:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.780581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:13.784672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789184009069230:2109] [node 6] ICP22 created new session: [5:7577789184009069266:2109]
2025-11-28T14:52:13.804861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789185192890252:2131] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:13.800330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789184009069266:2109] [node 6] ICS09 handshake done sender: [5:7577789184009069241:2115] self: [5:7577789184009069240:0] peer: [6:7577789185192890257:0] socket: 418 qp: -1
2025-11-28T14:52:13.804519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789184009069266:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:13.804890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789185192890205:2103] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26951) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789185192890252:2131] held: no
2025-11-28T14:52:13.804754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:13.805418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789184009069230:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(36449263ns) {HandleHandshakeStatus:329(36446087ns) {DropIncomingHandshake:442(40996ns) DropOutgoingHandshake:457(1133ns) SwitchToState:179(543ns) ProcessPendingSessionEvents:495(205021ns) {ForwardSessionEventToSession:612(108971ns) ForwardSessionEventToSession:612(32487ns) ForwardSessionEventToSession:612(4468ns) ForwardSessionEventToSession:612(3060ns) ForwardSessionEventToSession:612(9248ns) ForwardSessionEventToSession:612(3627ns) ForwardSessionEventToSession:612(9330ns) ForwardSessionEventToSession:612(3601ns) ForwardSessionEventToSession:612(3150ns) ForwardSessionEventToSession:612(17902ns)}}}
2025-11-28T14:52:13.813057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789183714412057:2134] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:13.812517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789185192890205:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.812834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:13.812889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789185192890205:2103] [node 5] ICP22 created new session: [6:7577789185192890259:2103]
2025-11-28T14:52:13.812943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789185192890259:2103] [node 5] ICS09 handshake done sender: [6:7577789185192890252:2131] self: [6:7577789185192890257:0] peer: [5:7577789184009069240:0] socket: 426 qp: -1
2025-11-28T14:52:13.812980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789185192890259:2103] [node 5] ICS10 traffic start
2025-11-28T14:52:13.813044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:13.844295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789183714412057:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.843666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789184009069259:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.844856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789183714412057:2134] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:13.843914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789184009069168:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:13.851064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789183714412051:2130] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:13.852831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789183714412044:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:13.853493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789184009069269:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18281
2025-11-28T14:52:13.851943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789186015382895:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:13.853496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:13.852125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789181720415504:2049] [node 3] ICP18 handshake (actor [1:7577789186015382895:2117]) is too old
2025-11-28T14:52:13.853575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789183714412044:2108] [node 5] ICP22 created new session: [3:7577789183714412083:2108]
2025-11-28T14:52:13.853533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789184009069168:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25953) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789184009069259:2122] held: no
2025-11-28T14:52:13.853836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789183714412083:2108] [node 5] ICS09 handshake done sender: [3:7577789183714412057:2134] self: [3:7577789183714412054:0] peer: [5:7577789184009069268:0] socket: 403 qp: -1
2025-11-28T14:52:13.853885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789183714412083:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:13.853942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:13.855192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789184009069259:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:13.855421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789184009069168:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:13.855680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:13.855747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789184009069168:2050] [node 3] ICP22 created new session: [5:7577789184009069270:2050]
2025-11-28T14:52:13.855799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789184009069270:2050] [node 3] ICS09 handshake done sender: [5:7577789184009069259:2122] self: [5:7577789184009069268:0] peer: [3:7577789183714412054:0] socket: 460 qp: -1
2025-11-28T14:52:13.855833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789184009069270:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:13.855876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:13.899090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:13.899333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789183714412080:2048] [node 1] ICS01 socket: 439 reason# EndOfStream
2025-11-28T14:52:13.899382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789179419444666:2048] [node 1] ICP30 unregister session Session# [3:7577789183714412080:2048] VirtualId# [3:7577789183714412078:0]
2025-11-28T14:52:13.899424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789183714412080:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.899512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:13.899558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:13.900048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:13.900267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789183910050773:2048] [node 1] ICS01 socket: 391 reason# EndOfStream
2025-11-28T14:52:13.900318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789179615083383:2048] [node 1] ICP30 unregister session Session# [2:7577789183910050773:2048] VirtualId# [2:7577789183910050761:0]
2025-11-28T14:52:13.900355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789183910050773:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.900481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:13.900571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:13.902080Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:13.902315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789184318148111:2048] [node 1] ICS07 socket disconnect 402 reason# EndOfStream
2025-11-28T14:52:13.902342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789184318148111:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.902570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:13.902627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789184318148111:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:13.905185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789184318148029:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:13.904506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:13.905700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789184318148111:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:13.904733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789184009069250:2048] [node 1] ICS07 socket disconnect 398 reason# EndOfStream
2025-11-28T14:52:13.904761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789184009069250:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.905700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:13.904837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:13.905870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789184318148029:2048] [node 1] ICP30 unregister session Session# [4:7577789184318148111:2048] VirtualId# [4:7577789184318148109:0]
2025-11-28T14:52:13.904877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789184009069250:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:13.905943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:13.906003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789184318148029:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.906024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.906157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789185192890243:2048] [node 1] ICS07 socket disconnect 396 reason# EndOfStream
2025-11-28T14:52:13.906193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789185192890243:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.906322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:13.906684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789184009069166:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:13.906776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789184009069250:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:13.906393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789185192890243:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:13.907919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789184009069166:2048] [node 1] ICP30 unregister session Session# [5:7577789184009069250:2048] VirtualId# [5:7577789184009069246:0]
2025-11-28T14:52:13.907976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:13.908006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789184009069166:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.908019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.907671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789185192890148:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:13.907720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789185192890243:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:13.907763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789185192890148:2048] [node 1] ICP30 unregister session Session# [6:7577789185192890243:2048] VirtualId# [6:7577789185192890239:0]
2025-11-28T14:52:13.907809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:13.907846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789185192890148:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.907869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.915837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789184318148029:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:13.917884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789185192890148:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:13.917884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789184009069166:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:13.927729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:13.928000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789183714412068:2049] [node 2] ICS07 socket disconnect 419 reason# EndOfStream
2025-11-28T14:52:13.928033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789183714412068:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.928153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:13.928198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789183714412068:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:13.930088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789179419444667:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13582) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:13.930617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789183714412068:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:13.930735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789179419444667:2049] [node 2] ICP30 unregister session Session# [3:7577789183714412068:2049] VirtualId# [3:7577789183714412066:0]
2025-11-28T14:52:13.930793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:13.930834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789179419444667:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13582) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.930849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13582) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.930800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:13.931006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789184318148118:2049] [node 2] ICS01 socket: 448 reason# EndOfStream
2025-11-28T14:52:13.931053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789184318148030:2049] [node 2] ICP30 unregister session Session# [4:7577789184318148118:2049] VirtualId# [4:7577789184318148115:0]
2025-11-28T14:52:13.931085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789184318148118:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.931152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:13.931198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:13.931917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:13.932059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789184009069255:2049] [node 2] ICS01 socket: 411 reason# EndOfStream
2025-11-28T14:52:13.932099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789184009069167:2049] [node 2] ICP30 unregister session Session# [5:7577789184009069255:2049] VirtualId# [5:7577789184009069253:0]
2025-11-28T14:52:13.932122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789184009069255:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.932169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:13.932192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:13.933028Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:13.933260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789185192890245:2049] [node 2] ICS07 socket disconnect 437 reason# EndOfStream
2025-11-28T14:52:13.933307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789185192890245:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.933403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:13.933451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789185192890245:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:13.934429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789185192890149:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13582) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:13.934480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789185192890245:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:13.934513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789185192890149:2049] [node 2] ICP30 unregister session Session# [6:7577789185192890245:2049] VirtualId# [6:7577789185192890241:0]
2025-11-28T14:52:13.934550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:13.934576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789185192890149:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13582) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.934589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13582) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.945364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789185192890149:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:13.946173Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:13.946388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789184318148126:2050] [node 3] ICS01 socket: 422 reason# EndOfStream
2025-11-28T14:52:13.946437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789184318148031:2050] [node 3] ICP30 unregister session Session# [4:7577789184318148126:2050] VirtualId# [4:7577789184318148124:0]
2025-11-28T14:52:13.946476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789184318148126:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.946552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:13.946590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:13.947692Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:13.947997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789184009069270:2050] [node 3] ICS01 socket: 460 reason# EndOfStream
2025-11-28T14:52:13.948056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789184009069168:2050] [node 3] ICP30 unregister session Session# [5:7577789184009069270:2050] VirtualId# [5:7577789184009069268:0]
2025-11-28T14:52:13.948083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789184009069270:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.948145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:13.948180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:13.948582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:13.948807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789185192890255:2050] [node 3] ICS07 socket disconnect 420 reason# EndOfStream
2025-11-28T14:52:13.948842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789185192890255:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:13.948957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:13.949009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789185192890255:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:13.950088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789185192890150:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25953) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:13.950140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789185192890255:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:13.950202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789185192890150:2050] [node 3] ICP30 unregister session Session# [6:7577789185192890255:2050] VirtualId# [6:7577789185192890253:0]
2025-11-28T14:52:13.950293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:13.950367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789185192890150:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25953) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.950394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25953) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:13.959829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789185192890150:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:14.006633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:14.006910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789185192890250:2104] [node 4] ICS07 socket disconnect 413 reason# EndOfStream
2025-11-28T14:52:14.006954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789185192890250:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.007082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:14.007138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789185192890250:2104] [node 4] ICS15 start handshake
2025-11-28T14:52:14.008139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789185192890218:2104] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30499) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:14.008255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789185192890250:2104] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:14.008307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789185192890218:2104] [node 4] ICP30 unregister session Session# [6:7577789185192890250:2104] VirtualId# [6:7577789185192890248:0]
2025-11-28T14:52:14.008392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:14.008445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789185192890218:2104] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30499) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.008480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30499) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.018228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789185192890218:2104] [node 4] ICP33 wake up from error state
2025-11-28T14:52:14.038186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:14.038495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789185192890259:2103] [node 5] ICS01 socket: 426 reason# EndOfStream
2025-11-28T14:52:14.038550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789185192890205:2103] [node 5] ICP30 unregister session Session# [6:7577789185192890259:2103] VirtualId# [6:7577789185192890257:0]
2025-11-28T14:52:14.038584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789185192890259:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.038693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:14.038747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:14.086545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789190908519173:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:14.086602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789190908519174:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:14.113753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789190066571648:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.113818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789190066571649:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:14.139527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789188898699401:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.139573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789188898699402:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:14.168846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789190973835617:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.168909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789190973835618:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:14.168958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789190973835619:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:14.196938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789189816318761:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.196978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789189816318762:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:14.197052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789189816318763:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:14.218932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789188980851754:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.218992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789188980851755:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:14.219042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789188980851756:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:14.272629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789188980851814:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:14.342303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789190908519252:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:14.346516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789190066571712:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:14.346928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789188980851814:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(74006477ns) {Configure:104(74003051ns) {ProcessConfigured:135(679313ns) {StartInitialHandshake:157(675948ns) {DropHandshakes:470(3262ns) {DropIncomingHandshake:442(837ns) DropOutgoingHandshake:457(653ns)} GenerateSessionVirtualId:988(1133ns) PrepareNewSessionHandshake:477(1555ns) {SwitchToState:179(603ns)}}}}}
2025-11-28T14:52:14.347085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789190066571713:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:14.346898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789190973835698:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:14.347530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789188980851829:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:14.347508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789190973835699:2117] [node 6] ICP01 ready to work
2025-11-28T14:52:14.347545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789190066571714:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:14.349013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789189816318761:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53712500ns) {Configure:104(53707159ns) {ProcessConfigured:135(94787ns) {StartInitialHandshake:157(91671ns) {DropHandshakes:470(2314ns) {DropIncomingHandshake:442(568ns) DropOutgoingHandshake:457(289ns)} GenerateSessionVirtualId:988(1087ns) PrepareNewSessionHandshake:477(2626ns) {SwitchToState:179(1000ns)}}}}}
2025-11-28T14:52:14.349625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789190908519255:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:14.350107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789190908519256:2119] [node 6] ICP01 ready to work
2025-11-28T14:52:14.349700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789189816318827:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:14.350093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789189816318828:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:14.349419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789188898699493:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:14.349859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789188898699494:2114] [node 5] ICP01 ready to work
2025-11-28T14:52:14.350339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789188898699495:2125] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.370865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789190066571725:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.371107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789188898699506:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26239
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.372424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789190066571729:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-28T14:52:14.372712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789190066571730:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-28T14:52:14.372463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789189816318837:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16674
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.374296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789190908519265:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1305
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.374481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789190908519266:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1305
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.374681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789190066571731:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:14.374911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789190973835713:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15152
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.375089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789190908519269:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1305
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.375085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789190973835714:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15152
2025-11-28T14:52:14.375318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789190908519270:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1305
2025-11-28T14:52:14.375274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789190973835715:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15152
2025-11-28T14:52:14.375692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519265:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.375815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519266:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.375920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789189816318837:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.377222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789190066571729:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.377308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789190066571725:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.378740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789188898699506:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.379385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789190973835713:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.379880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789190973835619:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.379967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789188898699508:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26239
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.380147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789188898699509:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26239
2025-11-28T14:52:14.380204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789190908519272:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1305
2025-11-28T14:52:14.380348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789188898699499:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.380687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519269:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.380386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789188898699499:2139] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.380908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789188898699510:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26239
2025-11-28T14:52:14.381165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789188898699499:2139] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.381490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789188898699493:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.381479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789190973835717:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15152
2025-11-28T14:52:14.381546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:14.381586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789188898699493:2107] [node 4] ICP22 created new session: [3:7577789188898699513:2107]
2025-11-28T14:52:14.381638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789188898699513:2107] [node 4] ICS09 handshake done sender: [3:7577789188898699499:2139] self: [3:7577789188898699498:0] peer: [4:7577789190973835716:0] socket: 413 qp: -1
2025-11-28T14:52:14.381683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789188898699513:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:14.381717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:14.382038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789190973835713:2133] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:14.382291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789190973835619:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.382344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:14.382384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789190973835619:2050] [node 3] ICP22 created new session: [4:7577789190973835718:2050]
2025-11-28T14:52:14.382451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789190973835718:2050] [node 3] ICS09 handshake done sender: [4:7577789190973835713:2133] self: [4:7577789190973835716:0] peer: [3:7577789188898699498:0] socket: 422 qp: -1
2025-11-28T14:52:14.382481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789190973835718:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:14.382509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.387935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789190066571732:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-28T14:52:14.388342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789190066571730:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.388940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789188898699508:2143] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.389068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789188898699402:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.389155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789190066571732:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.389272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789188898699509:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.389512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789190066571718:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.389533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789190066571718:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.389669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789189816318841:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16674
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.389856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789189816318842:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16674
2025-11-28T14:52:14.390030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789189816318841:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.390139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789189816318763:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.390258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789190066571718:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:14.390558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789188898699501:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.390624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789190066571649:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.390611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789188898699501:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.390747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:14.390803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789190066571649:2049] [node 3] ICP22 created new session: [2:7577789190066571738:2049]
2025-11-28T14:52:14.390852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789190066571738:2049] [node 3] ICS09 handshake done sender: [2:7577789190066571718:2112] self: [2:7577789190066571717:0] peer: [3:7577789188898699516:0] socket: 416 qp: -1
2025-11-28T14:52:14.390882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789190066571738:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:14.390928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.391232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789188898699518:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26239
2025-11-28T14:52:14.393043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789190973835714:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.393139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789188898699508:2143] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.393343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789190973835618:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.393349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789189816318845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16674
2025-11-28T14:52:14.393440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789189816318846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16674
2025-11-28T14:52:14.393458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789188898699402:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7255) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789188898699508:2143] held: no
2025-11-28T14:52:14.393607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789188898699402:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.393659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:14.393682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519272:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.393717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789188898699402:2049] [node 2] ICP22 created new session: [3:7577789188898699520:2049]
2025-11-28T14:52:14.393774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789188898699520:2049] [node 2] ICS09 handshake done sender: [3:7577789188898699508:2143] self: [3:7577789188898699516:0] peer: [2:7577789190066571717:0] socket: 427 qp: -1
2025-11-28T14:52:14.393815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789188898699520:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:14.393846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:14.393650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789189816318842:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.393924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789190973835715:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.393924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789190066571720:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.393961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789190066571720:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.393778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789189816318827:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.394045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789190973835707:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.394041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519270:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.394070Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789190973835707:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.394942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789190973835617:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.395023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789190066571731:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.395067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789189816318846:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.395102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789188898699501:2140] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:14.395185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519258:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.395218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789190908519258:2127] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.395203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789190973835707:2131] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:14.395765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789190066571648:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.396010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789190066571720:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:14.396145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789190908519258:2127] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.396330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789189816318848:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16674
2025-11-28T14:52:14.396207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789190973835698:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.396251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:14.396297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789190973835698:2108] [node 5] ICP22 created new session: [4:7577789190973835726:2108]
2025-11-28T14:52:14.396490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789188898699494:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.396601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789190066571712:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.396349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789190973835726:2108] [node 5] ICS09 handshake done sender: [4:7577789190973835707:2131] self: [4:7577789190973835706:0] peer: [5:7577789189816318847:0] socket: 403 qp: -1
2025-11-28T14:52:14.396814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519254:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.396845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789190908519254:2126] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.396821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:14.396638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:14.396451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789190973835726:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:14.396869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789190066571712:2107] [node 4] ICP22 created new session: [2:7577789190066571743:2107]
2025-11-28T14:52:14.397073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789189816318841:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:14.396834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789188898699494:2114] [node 5] ICP22 created new session: [3:7577789188898699522:2114]
2025-11-28T14:52:14.396489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:14.396953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789188898699522:2114] [node 5] ICS09 handshake done sender: [3:7577789188898699501:2140] self: [3:7577789188898699500:0] peer: [5:7577789189816318844:0] socket: 409 qp: -1
2025-11-28T14:52:14.397057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789190066571743:2107] [node 4] ICS09 handshake done sender: [2:7577789190066571720:2113] self: [2:7577789190066571719:0] peer: [4:7577789190973835721:0] socket: 418 qp: -1
2025-11-28T14:52:14.396997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789188898699522:2114] [node 5] ICS10 traffic start
2025-11-28T14:52:14.397099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:14.397095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789190066571743:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:14.397120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:14.397824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789189816318762:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.397879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789189816318763:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.397978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.398022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789189816318763:2050] [node 3] ICP22 created new session: [5:7577789189816318850:2050]
2025-11-28T14:52:14.398058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789189816318850:2050] [node 3] ICS09 handshake done sender: [5:7577789189816318841:2117] self: [5:7577789189816318844:0] peer: [3:7577789188898699500:0] socket: 440 qp: -1
2025-11-28T14:52:14.398087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789189816318850:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:14.398137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:14.397870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789190908519252:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.397960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:14.398095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789190973835724:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15152
2025-11-28T14:52:14.398020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789190908519252:2107] [node 4] ICP22 created new session: [1:7577789190908519279:2107]
2025-11-28T14:52:14.398090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789190908519279:2107] [node 4] ICS09 handshake done sender: [1:7577789190908519258:2127] self: [1:7577789190908519257:0] peer: [4:7577789190973835723:0] socket: 421 qp: -1
2025-11-28T14:52:14.398120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789190908519279:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:14.398303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.398758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789190973835725:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15152
2025-11-28T14:52:14.398846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789188898699510:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.399581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789188898699401:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.400699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789189816318842:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:14.402979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789190066571722:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.403044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789190973835618:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7255) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789190973835714:2134] held: no
2025-11-28T14:52:14.403010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789190066571722:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.403107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519260:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.403150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789190908519260:2128] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.403311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789189816318827:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.403249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851841:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.404024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851842:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.404061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:14.404268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789189816318827:2108] [node 4] ICP22 created new session: [5:7577789189816318852:2108]
2025-11-28T14:52:14.404454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789189816318852:2108] [node 4] ICS09 handshake done sender: [5:7577789189816318842:2118] self: [5:7577789189816318847:0] peer: [4:7577789190973835706:0] socket: 441 qp: -1
2025-11-28T14:52:14.404473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789190973835714:2134] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:14.404485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789189816318852:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:14.404527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:14.404859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789190908519254:2126] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.405007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789188898699526:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26239
2025-11-28T14:52:14.405001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789190066571746:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-28T14:52:14.405552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789190908519173:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.405653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789190066571722:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:14.405672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:14.405736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789190908519173:2048] [node 2] ICP22 created new session: [1:7577789190908519284:2048]
2025-11-28T14:52:14.405788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789190908519284:2048] [node 2] ICS09 handshake done sender: [1:7577789190908519254:2126] self: [1:7577789190908519253:0] peer: [2:7577789190066571742:0] socket: 420 qp: -1
2025-11-28T14:52:14.405828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789190908519284:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:14.405874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:14.405954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789190066571713:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.406068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:14.404970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789190973835715:2135] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:14.406116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789190066571713:2108] [node 5] ICP22 created new session: [2:7577789190066571748:2108]
2025-11-28T14:52:14.404666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789188980851842:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.406164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789190066571748:2108] [node 5] ICS09 handshake done sender: [2:7577789190066571722:2114] self: [2:7577789190066571721:0] peer: [5:7577789189816318849:0] socket: 444 qp: -1
2025-11-28T14:52:14.406204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789190066571748:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:14.405359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789190973835617:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6405) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789190973835715:2135] held: no
2025-11-28T14:52:14.406239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:14.405454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789190973835617:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.406201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789188898699503:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.405534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:14.404970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789188980851841:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.405594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789190973835617:2048] [node 1] ICP22 created new session: [4:7577789190973835729:2048]
2025-11-28T14:52:14.406230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789188898699503:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.405646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789190973835729:2048] [node 1] ICS09 handshake done sender: [4:7577789190973835715:2135] self: [4:7577789190973835723:0] peer: [1:7577789190908519257:0] socket: 425 qp: -1
2025-11-28T14:52:14.405682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789190973835729:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:14.406680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789190908519260:2128] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:14.405246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851843:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.405712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:14.405619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789188980851756:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.406233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789190973835618:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.405686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789188980851755:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.406260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:14.406301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789190973835618:2049] [node 2] ICP22 created new session: [4:7577789190973835731:2049]
2025-11-28T14:52:14.406376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789190973835731:2049] [node 2] ICS09 handshake done sender: [4:7577789190973835714:2134] self: [4:7577789190973835721:0] peer: [2:7577789190066571719:0] socket: 424 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:14.407039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789190066571731:2119] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:14.406398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789190973835731:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:14.406583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.407344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789190066571724:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.407183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789189816318854:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16674
2025-11-28T14:52:14.407369Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789190066571724:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.407669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789188898699510:2145] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:14.407302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789189816318855:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16674
2025-11-28T14:52:14.408338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789188898699401:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.408418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:14.408420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789190066571648:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.408467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789188898699401:2048] [node 1] ICP22 created new session: [3:7577789188898699528:2048]
2025-11-28T14:52:14.408527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:14.408547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789188898699528:2048] [node 1] ICS09 handshake done sender: [3:7577789188898699510:2145] self: [3:7577789188898699525:0] peer: [1:7577789190908519259:0] socket: 437 qp: -1
2025-11-28T14:52:14.408589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789190066571648:2048] [node 1] ICP22 created new session: [2:7577789190066571750:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.408586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789188898699528:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:14.408631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:14.408648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789190066571750:2048] [node 1] ICS09 handshake done sender: [2:7577789190066571731:2119] self: [2:7577789190066571742:0] peer: [1:7577789190908519253:0] socket: 426 qp: -1
2025-11-28T14:52:14.408680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789190066571750:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:14.408683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851846:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.408745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: 2025-11-28T14:52:14.408472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789190908519174:2049] [node 3] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:14.408595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:14.408644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789190908519174:2049] [node 3] ICP22 created new session: [1:7577789190908519286:2049]
2025-11-28T14:52:14.408696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789190908519286:2049] [node 3] ICS09 handshake done sender: [1:7577789190908519260:2128] self: [1:7577789190908519259:0] peer: [3:7577789188898699525:0] socket: 430 qp: -1
2025-11-28T14:52:14.408723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789190908519286:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:14.408751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:14.409037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851847:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.409187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789188898699503:2141] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:14.409884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789189816318846:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:14.411629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789188980851842:2125] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:14.411693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789189816318762:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7255) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789189816318846:2120] held: no
2025-11-28T14:52:14.411804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789189816318762:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.411868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:14.412025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789189816318762:2049] [node 2] ICP22 created new session: [5:7577789189816318857:2049]
2025-11-28T14:52:14.412029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789188898699495:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.412102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:14.412079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789189816318857:2049] [node 2] ICS09 handshake done sender: [5:7577789189816318846:2120] self: [5:7577789189816318849:0] peer: [2:7577789190066571721:0] socket: 446 qp: -1
2025-11-28T14:52:14.412043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789188980851756:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.412157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789188898699495:2125] [node 6] ICP22 created new session: [3:7577789188898699530:2125]
2025-11-28T14:52:14.412154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789189816318857:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:14.412213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:14.412175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:14.412259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789188980851756:2050] [node 3] ICP22 created new session: [6:7577789188980851848:2050]
2025-11-28T14:52:14.412243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789188898699530:2125] [node 6] ICS09 handshake done sender: [3:7577789188898699503:2141] self: [3:7577789188898699502:0] peer: [6:7577789188980851844:0] socket: 436 qp: -1
2025-11-28T14:52:14.412292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789188898699530:2125] [node 6] ICS10 traffic start
2025-11-28T14:52:14.412299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789188980851848:2050] [node 3] ICS09 handshake done sender: [6:7577789188980851842:2125] self: [6:7577789188980851844:0] peer: [3:7577789188898699502:0] socket: 453 qp: -1
2025-11-28T14:52:14.412319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:14.412335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789188980851848:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:14.412417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789190066571648:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:14.412727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:14.413510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789188980851843:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.414319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789188980851829:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.414445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851850:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.414680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789190973835709:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.414714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789190973835709:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.415510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789190066571724:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.415815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789190066571714:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.415898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:14.415938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789190066571714:2109] [node 6] ICP22 created new session: [2:7577789190066571753:2109]
2025-11-28T14:52:14.415865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789190973835733:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15152
2025-11-28T14:52:14.415998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789190066571753:2109] [node 6] ICS09 handshake done sender: [2:7577789190066571724:2115] self: [2:7577789190066571723:0] peer: [6:7577789188980851845:0] socket: 410 qp: -1
2025-11-28T14:52:14.416126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789190066571753:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:14.416091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789190973835709:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:14.416166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:14.416585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789190973835699:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.416633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:14.416694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789190973835699:2117] [node 6] ICP22 created new session: [4:7577789190973835735:2117]
2025-11-28T14:52:14.416740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789190973835735:2117] [node 6] ICS09 handshake done sender: [4:7577789190973835709:2132] self: [4:7577789190973835708:0] peer: [6:7577789188980851851:0] socket: 405 qp: -1
2025-11-28T14:52:14.416352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789189816318854:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.416778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789190973835735:2117] [node 6] ICS10 traffic start
2025-11-28T14:52:14.416809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:14.416555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789189816318761:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.417263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789188980851847:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.417764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519262:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.417797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789190908519262:2129] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.418496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789188980851841:2124] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:14.418524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789190908519262:2129] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.418849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789190908519255:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.418930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:14.418973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789190908519255:2109] [node 5] ICP22 created new session: [1:7577789190908519290:2109]
2025-11-28T14:52:14.418961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789188980851755:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7255) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789188980851841:2124] held: no
2025-11-28T14:52:14.419006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789190908519290:2109] [node 5] ICS09 handshake done sender: [1:7577789190908519262:2129] self: [1:7577789190908519261:0] peer: [5:7577789189816318859:0] socket: 454 qp: -1
2025-11-28T14:52:14.419039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789190908519290:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:14.418985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789189816318860:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16674
2025-11-28T14:52:14.419071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:14.419066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789190973835733:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.419229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789190973835699:2117] [node 6] ICP18 handshake (actor [4:7577789190973835733:2139]) is too old
2025-11-28T14:52:14.419081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789188980851755:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.419146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:14.419188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789188980851755:2049] [node 2] ICP22 created new session: [6:7577789188980851855:2049]
2025-11-28T14:52:14.419236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789188980851855:2049] [node 2] ICS09 handshake done sender: [6:7577789188980851841:2124] self: [6:7577789188980851845:0] peer: [2:7577789190066571723:0] socket: 452 qp: -1
2025-11-28T14:52:14.419461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789189816318854:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:14.419264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789188980851855:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.419293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:14.420092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851853:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.420332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789189816318761:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.420503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:14.420559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789189816318761:2048] [node 1] ICP22 created new session: [5:7577789189816318861:2048]
2025-11-28T14:52:14.420634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789189816318861:2048] [node 1] ICS09 handshake done sender: [5:7577789189816318854:2122] self: [5:7577789189816318859:0] peer: [1:7577789190908519261:0] socket: 458 qp: -1
2025-11-28T14:52:14.420667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789189816318861:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:14.420694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.427182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851854:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.427468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789188980851814:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.428383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789188980851829:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789188980851843:2126] held: no
2025-11-28T14:52:14.428705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789188980851854:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.428770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789188980851843:2126] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:14.428815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789189816318836:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.428862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789189816318836:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.429103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789188980851829:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.429178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:14.429251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789188980851829:2109] [node 4] ICP22 created new session: [6:7577789188980851858:2109]
2025-11-28T14:52:14.429302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789188980851858:2109] [node 4] ICS09 handshake done sender: [6:7577789188980851843:2126] self: [6:7577789188980851851:0] peer: [4:7577789190973835708:0] socket: 429 qp: -1
2025-11-28T14:52:14.429342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789188980851858:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:14.429409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:14.429804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789189816318836:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:14.430142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789189816318828:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.430210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:14.430163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789188980851754:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:14.430258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789189816318828:2109] [node 6] ICP22 created new session: [5:7577789189816318864:2109]
2025-11-28T14:52:14.430337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789189816318864:2109] [node 6] ICS09 handshake done sender: [5:7577789189816318836:2115] self: [5:7577789189816318835:0] peer: [6:7577789188980851857:0] socket: 461 qp: -1
2025-11-28T14:52:14.430376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789189816318864:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:14.430445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.430606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851860:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.432510Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789188980851814:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8769) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789188980851847:2128] held: no
2025-11-28T14:52:14.432632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789190908519264:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:14.432671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789190908519264:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:14.432670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789188980851847:2128] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:14.433380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789188980851814:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.433448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:14.433505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789188980851814:2105] [node 5] ICP22 created new session: [6:7577789188980851862:2105]
2025-11-28T14:52:14.433575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789190908519264:2130] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:14.433555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789188980851862:2105] [node 5] ICS09 handshake done sender: [6:7577789188980851847:2128] self: [6:7577789188980851857:0] peer: [5:7577789189816318835:0] socket: 431 qp: -1
2025-11-28T14:52:14.433591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789188980851862:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:14.433637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:14.434225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789190908519256:2119] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:14.434318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:14.434366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789190908519256:2119] [node 6] ICP22 created new session: [1:7577789190908519293:2119]
2025-11-28T14:52:14.434409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789190908519293:2119] [node 6] ICS09 handshake done sender: [1:7577789190908519264:2130] self: [1:7577789190908519263:0] peer: [6:7577789188980851861:0] socket: 463 qp: -1
2025-11-28T14:52:14.434437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789190908519293:2119] [node 6] ICS10 traffic start
2025-11-28T14:52:14.434471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:14.434817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789188980851864:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18225
2025-11-28T14:52:14.435234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789188980851854:2131] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:14.435417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789188980851754:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:14.435479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:14.435557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789188980851754:2048] [node 1] ICP22 created new session: [6:7577789188980851865:2048]
2025-11-28T14:52:14.435717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789188980851865:2048] [node 1] ICS09 handshake done sender: [6:7577789188980851854:2131] self: [6:7577789188980851861:0] peer: [1:7577789190908519263:0] socket: 464 qp: -1
2025-11-28T14:52:14.435763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789188980851865:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:14.435792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:14.458807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:14.459068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789190066571750:2048] [node 1] ICS01 socket: 426 reason# EndOfStream
2025-11-28T14:52:14.459113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789190066571648:2048] [node 1] ICP30 unregister session Session# [2:7577789190066571750:2048] VirtualId# [2:7577789190066571742:0]
2025-11-28T14:52:14.459152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789190066571750:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.459241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:14.459295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:14.460074Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:14.460286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789188898699528:2048] [node 1] ICS01 socket: 437 reason# EndOfStream
2025-11-28T14:52:14.460326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789188898699401:2048] [node 1] ICP30 unregister session Session# [3:7577789188898699528:2048] VirtualId# [3:7577789188898699525:0]
2025-11-28T14:52:14.460359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789188898699528:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.460438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:14.460491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:14.465196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:14.465423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789190973835729:2048] [node 1] ICS01 socket: 425 reason# EndOfStream
2025-11-28T14:52:14.465469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789190973835617:2048] [node 1] ICP30 unregister session Session# [4:7577789190973835729:2048] VirtualId# [4:7577789190973835723:0]
2025-11-28T14:52:14.465508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789190973835729:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.465588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:14.465657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:14.466295Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:14.467103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:14.466655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789189816318861:2048] [node 1] ICS07 socket disconnect 458 reason# EndOfStream
2025-11-28T14:52:14.466691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789189816318861:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.466779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:14.467308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789188980851865:2048] [node 1] ICS07 socket disconnect 464 reason# EndOfStream
2025-11-28T14:52:14.466815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789189816318861:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:14.467340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789188980851865:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.467403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:14.467468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789188980851865:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:14.467708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789189816318761:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6405) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:14.467770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789189816318861:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:14.467813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789189816318761:2048] [node 1] ICP30 unregister session Session# [5:7577789189816318861:2048] VirtualId# [5:7577789189816318859:0]
2025-11-28T14:52:14.467860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:14.467893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789189816318761:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6405) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.467928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6405) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.468420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789188980851754:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6405) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:14.468471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789188980851865:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:14.468502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789188980851754:2048] [node 1] ICP30 unregister session Session# [6:7577789188980851865:2048] VirtualId# [6:7577789188980851861:0]
2025-11-28T14:52:14.468540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:14.468573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789188980851754:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6405) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.468613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6405) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.478229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789189816318761:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:14.479973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789188980851754:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:14.492634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:14.492885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789188898699520:2049] [node 2] ICS01 socket: 427 reason# EndOfStream
2025-11-28T14:52:14.492940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789188898699402:2049] [node 2] ICP30 unregister session Session# [3:7577789188898699520:2049] VirtualId# [3:7577789188898699516:0]
2025-11-28T14:52:14.492978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789188898699520:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.493066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:14.493132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:14.496060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:14.496303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789190973835731:2049] [node 2] ICS01 socket: 424 reason# EndOfStream
2025-11-28T14:52:14.496352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789190973835618:2049] [node 2] ICP30 unregister session Session# [4:7577789190973835731:2049] VirtualId# [4:7577789190973835721:0]
2025-11-28T14:52:14.496401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789190973835731:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.496474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:14.496513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:14.497211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:14.497445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789189816318857:2049] [node 2] ICS01 socket: 446 reason# EndOfStream
2025-11-28T14:52:14.497503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789189816318762:2049] [node 2] ICP30 unregister session Session# [5:7577789189816318857:2049] VirtualId# [5:7577789189816318849:0]
2025-11-28T14:52:14.497537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789189816318857:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.497645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:14.497690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:14.498021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:14.498152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789188980851855:2049] [node 2] ICS01 socket: 452 reason# EndOfStream
2025-11-28T14:52:14.498179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789188980851755:2049] [node 2] ICP30 unregister session Session# [6:7577789188980851855:2049] VirtualId# [6:7577789188980851845:0]
2025-11-28T14:52:14.498197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789188980851855:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.498249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:14.498286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:14.524699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:14.524940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789190973835718:2050] [node 3] ICS07 socket disconnect 422 reason# EndOfStream
2025-11-28T14:52:14.524975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789190973835718:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.525055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:14.525101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789190973835718:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:14.525860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:14.525971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789190973835619:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32614) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:14.526023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789190973835718:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:14.526026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789189816318850:2050] [node 3] ICS01 socket: 440 reason# EndOfStream
2025-11-28T14:52:14.526052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789190973835619:2050] [node 3] ICP30 unregister session Session# [4:7577789190973835718:2050] VirtualId# [4:7577789190973835716:0]
2025-11-28T14:52:14.526066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789189816318763:2050] [node 3] ICP30 unregister session Session# [5:7577789189816318850:2050] VirtualId# [5:7577789189816318844:0]
2025-11-28T14:52:14.526097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:14.526088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789189816318850:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.526134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789190973835619:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.526147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.526185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:14.526225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:14.526902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:14.527068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789188980851848:2050] [node 3] ICS07 socket disconnect 453 reason# EndOfStream
2025-11-28T14:52:14.527091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789188980851848:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.527148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:14.527177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789188980851848:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:14.527850Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789188980851756:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32614) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:14.527899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789188980851848:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:14.527922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789188980851756:2050] [node 3] ICP30 unregister session Session# [6:7577789188980851848:2050] VirtualId# [6:7577789188980851844:0]
2025-11-28T14:52:14.527954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:14.527971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789188980851756:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.527986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:14.537400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789188980851756:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:14.555512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:14.555665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789189816318852:2108] [node 4] ICS01 socket: 441 reason# EndOfStream
2025-11-28T14:52:14.555692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789189816318827:2108] [node 4] ICP30 unregister session Session# [5:7577789189816318852:2108] VirtualId# [5:7577789189816318847:0]
2025-11-28T14:52:14.555717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789189816318852:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.555809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:14.555850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:14.556271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:14.556428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789188980851858:2109] [node 4] ICS01 socket: 429 reason# EndOfStream
2025-11-28T14:52:14.556458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789188980851829:2109] [node 4] ICP30 unregister session Session# [6:7577789188980851858:2109] VirtualId# [6:7577789188980851851:0]
2025-11-28T14:52:14.556482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789188980851858:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.556572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:14.556615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:14.584712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:14.584961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789188980851862:2105] [node 5] ICS01 socket: 431 reason# EndOfStream
2025-11-28T14:52:14.585011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789188980851814:2105] [node 5] ICP30 unregister session Session# [6:7577789188980851862:2105] VirtualId# [6:7577789188980851857:0]
2025-11-28T14:52:14.585054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789188980851862:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:14.585153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:14.585213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:14.665637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789188540267877:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:14.665708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789188540267878:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:14.720588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789190650087251:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.720683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789190650087252:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:14.743338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789188832237660:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.743388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789188832237661:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:14.764408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789189743669042:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.764468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789189743669044:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:14.764494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789189743669043:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:14.793561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789190358228779:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.793613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789190358228780:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:14.793648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789190358228781:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:14.825897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789189599466915:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:14.825939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789189599466916:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:14.825988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789189599466917:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:15.020339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789192835235245:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:15.021061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789193127205019:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:15.021436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789193894434276:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:15.020968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789192835235246:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:15.021678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789194653196148:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:15.021696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789194945054612:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:15.021922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789194653196147:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:15.022446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789192835235247:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:15.021746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789194945054613:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:15.024793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789193127205020:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:15.028178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789193127205021:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:15.027531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789193894434277:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:15.024423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789194945054614:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:15.029424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789188832237661:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(80918143ns) {Configure:104(80911959ns) {ProcessConfigured:135(548302ns) {StartInitialHandshake:157(546172ns) {DropHandshakes:470(39682ns) {DropIncomingHandshake:442(664ns) DropOutgoingHandshake:457(436ns)} GenerateSessionVirtualId:988(1007ns) PrepareNewSessionHandshake:477(1430ns) {SwitchToState:179(580ns)}}}}}
2025-11-28T14:52:15.029593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789189599466915:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(60078685ns) {Configure:104(60073388ns) {ProcessConfigured:135(457583ns) {StartInitialHandshake:157(454635ns) {DropHandshakes:470(2471ns) {DropIncomingHandshake:442(649ns) DropOutgoingHandshake:457(456ns)} GenerateSessionVirtualId:988(807ns) PrepareNewSessionHandshake:477(1833ns) {SwitchToState:179(780ns)}}}}}
2025-11-28T14:52:15.029924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789189743669042:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(76050926ns) {Configure:104(76045885ns) {ProcessConfigured:135(62339ns) {StartInitialHandshake:157(60138ns) {DropHandshakes:470(4172ns) {DropIncomingHandshake:442(640ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(1079ns) PrepareNewSessionHandshake:477(935ns) {SwitchToState:179(312ns)}}}}}
2025-11-28T14:52:15.030331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789189743669043:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(75759606ns) {Configure:104(75755891ns) {ProcessConfigured:135(128481ns) {StartInitialHandshake:157(126440ns) {DropHandshakes:470(1110ns) {DropIncomingHandshake:442(340ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(3764ns) {SwitchToState:179(880ns)}}}}}
2025-11-28T14:52:15.030890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789194038636423:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:15.031437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789194038636424:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:15.044321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789192835235262:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-28T14:52:15.044318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789194945054626:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21525
::ffff:127.1.0.0
2025-11-28T14:52:15.044510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789192835235261:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.044523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789194945054627:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21525
2025-11-28T14:52:15.044509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789194038636430:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48943
2025-11-28T14:52:15.044951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789194038636429:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48943
2025-11-28T14:52:15.046980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789194945054628:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21525
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.046994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789193127205033:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63046
2025-11-28T14:52:15.047088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789193127205032:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63046
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.049417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789192835235263:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.063381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789193127205036:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63046
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.069682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789194653196162:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51549
2025-11-28T14:52:15.073166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789194038636436:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48943
2025-11-28T14:52:15.070032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235263:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.074238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789192835235264:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-28T14:52:15.073341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789194653196163:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51549
2025-11-28T14:52:15.077890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789194653196163:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.077908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789193127205036:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.077563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789194038636429:2132] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:15.078823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235264:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:15.078123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789189743669042:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.095403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235253:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.086195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789193127205032:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.095471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789192835235253:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:15.100944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789192835235265:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-28T14:52:15.101922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789192835235253:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.102340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789192835235245:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.102466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:15.102419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789194038636438:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48943
2025-11-28T14:52:15.102597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789192835235245:2107] [node 4] ICP22 created new session: [1:7577789192835235269:2107]
2025-11-28T14:52:15.102698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789192835235269:2107] [node 4] ICS09 handshake done sender: [1:7577789192835235253:2121] self: [1:7577789192835235252:0] peer: [4:7577789194038636437:0] socket: 409 qp: -1
2025-11-28T14:52:15.102761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789192835235269:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:15.102993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:15.103111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789194038636429:2132] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:15.103423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789189743669042:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10551) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789194038636429:2132] held: no
2025-11-28T14:52:15.103507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789189743669042:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.103535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:15.103566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789189743669042:2048] [node 1] ICP22 created new session: [4:7577789194038636439:2048]
2025-11-28T14:52:15.103598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789194038636439:2048] [node 1] ICS09 handshake done sender: [4:7577789194038636429:2132] self: [4:7577789194038636437:0] peer: [1:7577789192835235252:0] socket: 419 qp: -1
2025-11-28T14:52:15.103621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789194038636439:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:15.103687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:15.114860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789194945054616:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.115537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789194945054631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21525
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.115658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789194945054632:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21525
2025-11-28T14:52:15.115801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789194945054628:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.115751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235262:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.152214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789194945054618:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.182881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789194945054626:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.183014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789194945054631:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.183313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789190650087251:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.188751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235249:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.188803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789192835235249:2119] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:15.190052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789192835235249:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.190322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789194945054636:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21525
2025-11-28T14:52:15.191314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789188540267877:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.191992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789194945054626:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:15.191390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:15.191452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789188540267877:2048] [node 2] ICP22 created new session: [1:7577789192835235274:2048]
2025-11-28T14:52:15.191506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789192835235274:2048] [node 2] ICS09 handshake done sender: [1:7577789192835235249:2119] self: [1:7577789192835235248:0] peer: [2:7577789194945054634:0] socket: 402 qp: -1
2025-11-28T14:52:15.191549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789192835235274:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:15.191587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:15.192671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789190650087251:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.192736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:15.192780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789190650087251:2048] [node 1] ICP22 created new session: [2:7577789194945054637:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.193764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434291:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.192825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789194945054637:2048] [node 1] ICS09 handshake done sender: [2:7577789194945054626:2116] self: [2:7577789194945054634:0] peer: [1:7577789192835235248:0] socket: 414 qp: -1
2025-11-28T14:52:15.192871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789194945054637:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:15.192910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:15.195755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789190650087251:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:15.196045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789193894434291:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.196239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789189599466915:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.197729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235260:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.197775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789192835235260:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:15.199164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789192835235260:2123] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:15.199663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789192835235247:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.199731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:15.199783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789192835235247:2109] [node 6] ICP22 created new session: [1:7577789192835235278:2109]
2025-11-28T14:52:15.199846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789192835235278:2109] [node 6] ICS09 handshake done sender: [1:7577789192835235260:2123] self: [1:7577789192835235259:0] peer: [6:7577789193894434292:0] socket: 418 qp: -1
2025-11-28T14:52:15.199887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789192835235278:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:15.199934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:15.200358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434293:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.200976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789193894434291:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:15.201497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789189599466915:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.201565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:15.201608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789189599466915:2048] [node 1] ICP22 created new session: [6:7577789193894434294:2048]
2025-11-28T14:52:15.201678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789193894434294:2048] [node 1] ICS09 handshake done sender: [6:7577789193894434291:2117] self: [6:7577789193894434292:0] peer: [1:7577789192835235259:0] socket: 448 qp: -1
2025-11-28T14:52:15.201714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789193894434294:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:15.201746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:15.202024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789189599466915:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:15.204805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789194653196155:2121] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.216216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235261:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.249368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789193127205026:2111] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.285822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789188832237660:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.286446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789194038636436:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.286597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789189743669044:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.298262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789194653196152:2120] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.299263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789194653196162:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.299811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789194653196165:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51549
2025-11-28T14:52:15.299960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789194653196166:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51549
2025-11-28T14:52:15.300088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789190358228780:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.300297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789194653196165:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.301252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789190358228779:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.301398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789194653196166:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.301650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789194653196148:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.301951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789194038636428:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.310962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789194945054618:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.329064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789194945054618:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:15.318673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235255:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.318735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789192835235255:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:15.301986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789194038636428:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:15.330105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789194945054613:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.330793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434297:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.329510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789194038636428:2131] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:15.330156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:15.330137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789194038636423:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.334828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789192835235255:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:15.331167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789194945054613:2108] [node 5] ICP22 created new session: [2:7577789194945054641:2108]
2025-11-28T14:52:15.330178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:15.330215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789194038636423:2108] [node 5] ICP22 created new session: [4:7577789194038636444:2108]
2025-11-28T14:52:15.330248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789194038636444:2108] [node 5] ICS09 handshake done sender: [4:7577789194038636428:2131] self: [4:7577789194038636427:0] peer: [5:7577789194653196169:0] socket: 425 qp: -1
2025-11-28T14:52:15.330273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789194038636444:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:15.330299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:15.341300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789192835235246:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.341371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:15.341421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789192835235246:2108] [node 5] ICP22 created new session: [1:7577789192835235282:2108]
2025-11-28T14:52:15.341474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789192835235282:2108] [node 5] ICS09 handshake done sender: [1:7577789192835235255:2122] self: [1:7577789192835235254:0] peer: [5:7577789194653196168:0] socket: 434 qp: -1
2025-11-28T14:52:15.341511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789192835235282:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:15.331220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789194945054641:2108] [node 5] ICS09 handshake done sender: [2:7577789194945054618:2113] self: [2:7577789194945054617:0] peer: [5:7577789194653196167:0] socket: 408 qp: -1
2025-11-28T14:52:15.331262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789194945054641:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:15.331299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.343740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434298:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.341575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:15.344306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789190358228779:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10551) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789194653196165:2126] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.344701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789194653196170:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51549
2025-11-28T14:52:15.344897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789193894434297:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.345067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789193894434277:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.345843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789194653196171:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51549
2025-11-28T14:52:15.350930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789194038636434:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.350994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789194038636434:2134] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.352406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789194653196172:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.360568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789194038636434:2134] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:15.354291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434300:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.360914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789194038636424:2111] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.362239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789193894434297:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:15.360807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789194653196162:2124] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:15.363953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789193127205027:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.365131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789193127205026:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.374363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789193894434277:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.361000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:15.374368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235265:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.395671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789194653196166:2127] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.362960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789194038636424:2111] [node 6] ICP22 created new session: [4:7577789194038636448:2111]
2025-11-28T14:52:15.395479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789192835235251:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.365261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789193127205033:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.378750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789194038636448:2111] [node 6] ICS09 handshake done sender: [4:7577789194038636434:2134] self: [4:7577789194038636432:0] peer: [6:7577789193894434299:0] socket: 454 qp: -1
2025-11-28T14:52:15.386487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789194038636448:2111] [node 6] ICS10 traffic start
2025-11-28T14:52:15.386556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:15.399142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789194038636450:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48943
2025-11-28T14:52:15.374437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:15.365785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789193127205040:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63046
2025-11-28T14:52:15.395530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789192835235251:2120] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:15.378797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789193894434277:2108] [node 4] ICP22 created new session: [6:7577789193894434301:2108]
2025-11-28T14:52:15.365894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789193127205041:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63046
2025-11-28T14:52:15.378665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789193127205026:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:15.403750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789193127205041:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.408011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789193127205019:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.408081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:15.408131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789193127205019:2107] [node 4] ICP22 created new session: [3:7577789193127205045:2107]
2025-11-28T14:52:15.401073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789194038636436:2135] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:15.402323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789189743669044:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.402378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:15.402428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789189743669044:2050] [node 3] ICP22 created new session: [4:7577789194038636451:2050]
2025-11-28T14:52:15.402487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789194038636451:2050] [node 3] ICS09 handshake done sender: [4:7577789194038636436:2135] self: [4:7577789194038636442:0] peer: [3:7577789193127205024:0] socket: 437 qp: -1
2025-11-28T14:52:15.402522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789194038636451:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:15.402564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:15.401840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789192835235251:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:15.408556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789194653196165:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:15.404104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789194038636424:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(43021396ns) {HandleHandshakeStatus:329(42992601ns) {DropIncomingHandshake:442(1014ns) DropOutgoingHandshake:457(1391ns) SwitchToState:179(1017ns) ProcessPendingSessionEvents:495(4281002ns) {ForwardSessionEventToSession:612(2163569ns) ForwardSessionEventToSession:612(319966ns) ForwardSessionEventToSession:612(2995ns) ForwardSessionEventToSession:612(1752687ns) ForwardSessionEventToSession:612(4625ns) ForwardSessionEventToSession:612(2425ns) ForwardSessionEventToSession:612(5411ns) ForwardSessionEventToSession:612(5504ns) ForwardSessionEventToSession:612(2814ns) ForwardSessionEventToSession:612(5307ns)}}}
2025-11-28T14:52:15.398752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789193894434301:2108] [node 4] ICS09 handshake done sender: [6:7577789193894434297:2119] self: [6:7577789193894434299:0] peer: [4:7577789194038636432:0] socket: 455 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.408192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789193127205045:2107] [node 4] ICS09 handshake done sender: [3:7577789193127205026:2111] self: [3:7577789193127205024:0] peer: [4:7577789194038636442:0] socket: 435 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.425087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789194038636453:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48943
2025-11-28T14:52:15.410652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789188540267878:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.408251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789193127205045:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:15.399649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789193894434301:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:15.421221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:15.408295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:15.426013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789190358228780:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.426083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:15.402123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:15.424106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789188540267878:2049] [node 3] ICP22 created new session: [1:7577789192835235286:2049]
2025-11-28T14:52:15.427337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789190358228780:2049] [node 2] ICP22 created new session: [5:7577789194653196174:2049]
2025-11-28T14:52:15.411883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789193127205044:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63046
2025-11-28T14:52:15.412553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789193127205032:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:15.425964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789192835235286:2049] [node 3] ICS09 handshake done sender: [1:7577789192835235251:2120] self: [1:7577789192835235250:0] peer: [3:7577789193127205039:0] socket: 403 qp: -1
2025-11-28T14:52:15.431372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789194653196174:2049] [node 2] ICS09 handshake done sender: [5:7577789194653196162:2124] self: [5:7577789194653196167:0] peer: [2:7577789194945054617:0] socket: 431 qp: -1
2025-11-28T14:52:15.414528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789193894434277:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(39950182ns) {HandleHandshakeStatus:329(39945534ns) {DropIncomingHandshake:442(1249ns) DropOutgoingHandshake:457(9287ns) SwitchToState:179(1006ns) ProcessPendingSessionEvents:495(5706095ns) {ForwardSessionEventToSession:612(5638237ns) ForwardSessionEventToSession:612(10443ns) ForwardSessionEventToSession:612(3210ns) ForwardSessionEventToSession:612(3458ns) ForwardSessionEventToSession:612(5639ns) ForwardSessionEventToSession:612(3055ns) ForwardSessionEventToSession:612(3776ns) ForwardSessionEventToSession:612(15936ns) ForwardSessionEventToSession:612(3079ns) ForwardSessionEventToSession:612(8764ns)}}}
2025-11-28T14:52:15.412906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789188832237660:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.431427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789194653196174:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:15.426166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789192835235286:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:15.425633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434303:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.412973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:15.433343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:15.426219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:15.413024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789188832237660:2048] [node 1] ICP22 created new session: [3:7577789193127205049:2048]
2025-11-28T14:52:15.413080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789193127205049:2048] [node 1] ICS09 handshake done sender: [3:7577789193127205032:2115] self: [3:7577789193127205039:0] peer: [1:7577789192835235250:0] socket: 423 qp: -1
2025-11-28T14:52:15.432036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789188540267878:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(21161152ns) {HandleHandshakeStatus:329(21158384ns) {DropIncomingHandshake:442(15053ns) DropOutgoingHandshake:457(660ns) SwitchToState:179(1202ns) ProcessPendingSessionEvents:495(158399ns) {ForwardSessionEventToSession:612(89465ns) ForwardSessionEventToSession:612(11410ns) ForwardSessionEventToSession:612(3993ns) ForwardSessionEventToSession:612(3560ns) ForwardSessionEventToSession:612(6380ns) ForwardSessionEventToSession:612(2830ns) ForwardSessionEventToSession:612(10432ns) ForwardSessionEventToSession:612(7291ns) ForwardSessionEventToSession:612(2795ns) ForwardSessionEventToSession:612(6231ns)}}}
2025-11-28T14:52:15.413116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789193127205049:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:15.413147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:15.434365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789194653196148:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.434422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:15.434489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789194653196148:2108] [node 4] ICP22 created new session: [5:7577789194653196176:2108]
2025-11-28T14:52:15.434552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789194653196176:2108] [node 4] ICS09 handshake done sender: [5:7577789194653196166:2127] self: [5:7577789194653196169:0] peer: [4:7577789194038636427:0] socket: 451 qp: -1
2025-11-28T14:52:15.434603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789194653196176:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:15.434632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:15.435840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789190358228779:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.435881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:15.435956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789190358228779:2048] [node 1] ICP22 created new session: [5:7577789194653196178:2048]
2025-11-28T14:52:15.436016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789194653196178:2048] [node 1] ICS09 handshake done sender: [5:7577789194653196165:2126] self: [5:7577789194653196168:0] peer: [1:7577789192835235254:0] socket: 421 qp: -1
2025-11-28T14:52:15.436051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789194653196178:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:15.436080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.437683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789194653196180:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.438272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434304:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.443392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789194945054620:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.475299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789194038636430:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.475474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789189743669043:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.481239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789194653196151:2119] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.506751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789193127205023:2110] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.533265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789194945054630:2119] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.535542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789193894434298:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.534422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789194945054620:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.534578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789194945054627:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.534666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789194945054632:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.535024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789194945054613:2108] [node 5] ICP18 handshake (actor [2:7577789194945054627:2117]) is too old
2025-11-28T14:52:15.546433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789189599466916:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.570480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789194945054620:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:15.570907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789194038636455:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48943
2025-11-28T14:52:15.571379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789194945054630:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.571480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789194945054612:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.571538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:15.571597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789194945054612:2107] [node 4] ICP22 created new session: [2:7577789194945054646:2107]
2025-11-28T14:52:15.571794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789194945054646:2107] [node 4] ICS09 handshake done sender: [2:7577789194945054620:2114] self: [2:7577789194945054619:0] peer: [4:7577789194038636454:0] socket: 412 qp: -1
2025-11-28T14:52:15.571939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789194945054646:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:15.572008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:15.571399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789194038636430:2133] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:15.571658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789189743669043:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.571698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:15.571774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789189743669043:2049] [node 2] ICP22 created new session: [4:7577789194038636456:2049]
2025-11-28T14:52:15.571969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789194038636456:2049] [node 2] ICS09 handshake done sender: [4:7577789194038636430:2133] self: [4:7577789194038636454:0] peer: [2:7577789194945054619:0] socket: 420 qp: -1
2025-11-28T14:52:15.572005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789194038636456:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:15.572051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:15.576674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789193127205031:2114] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.597639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789194945054630:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.614932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789193894434303:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.614944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434306:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.619293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789189599466917:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.619639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789193894434298:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:15.619742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789189599466916:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.619799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:15.618153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789194945054614:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.619993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789189599466916:2049] [node 2] ICP22 created new session: [6:7577789193894434308:2049]
2025-11-28T14:52:15.620124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789193127205031:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.620448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789194653196161:2123] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.618243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:15.620024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789193894434308:2049] [node 2] ICS09 handshake done sender: [6:7577789193894434298:2120] self: [6:7577789193894434305:0] peer: [2:7577789194945054629:0] socket: 457 qp: -1
2025-11-28T14:52:15.633254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789193127205031:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:15.619307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789194945054614:2109] [node 6] ICP22 created new session: [2:7577789194945054649:2109]
2025-11-28T14:52:15.620053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789193894434308:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:15.619365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789194945054649:2109] [node 6] ICS09 handshake done sender: [2:7577789194945054630:2119] self: [2:7577789194945054629:0] peer: [6:7577789193894434305:0] socket: 456 qp: -1
2025-11-28T14:52:15.619504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789194945054649:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:15.619642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.652291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789194653196161:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.620125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:15.651054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789189599466916:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(29074041ns) {HandleHandshakeStatus:329(29072123ns) {DropIncomingHandshake:442(739ns) DropOutgoingHandshake:457(4922ns) SwitchToState:179(197ns) ProcessPendingSessionEvents:495(28570458ns) {ForwardSessionEventToSession:612(321956ns) ForwardSessionEventToSession:612(7861ns) ForwardSessionEventToSession:612(2405ns) ForwardSessionEventToSession:612(1922ns) ForwardSessionEventToSession:612(12516145ns) ForwardSessionEventToSession:612(5330ns) ForwardSessionEventToSession:612(4414288ns) ForwardSessionEventToSession:612(8353507ns) ForwardSessionEventToSession:612(4414ns) ForwardSessionEventToSession:612(2915042ns)}}}
2025-11-28T14:52:15.651241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789193894434304:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.651555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434310:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.652010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789193894434276:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.652219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789193894434303:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:15.652490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789193127205021:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.652683Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789194653196161:2123] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:15.652591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:15.652675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789193127205021:2109] [node 6] ICP22 created new session: [3:7577789193127205053:2109]
2025-11-28T14:52:15.653015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789194653196147:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.652726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789193127205053:2109] [node 6] ICS09 handshake done sender: [3:7577789193127205031:2114] self: [3:7577789193127205029:0] peer: [6:7577789193894434307:0] socket: 428 qp: -1
2025-11-28T14:52:15.653072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:15.652770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789193127205053:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:15.652806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:15.653129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789194653196147:2109] [node 6] ICP22 created new session: [5:7577789194653196183:2109]
2025-11-28T14:52:15.653591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789189599466917:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.653402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789194653196183:2109] [node 6] ICS09 handshake done sender: [5:7577789194653196161:2123] self: [5:7577789194653196160:0] peer: [6:7577789193894434311:0] socket: 440 qp: -1
2025-11-28T14:52:15.653473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789194653196183:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:15.653511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:15.654406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:15.654739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789189599466917:2050] [node 3] ICP22 created new session: [6:7577789193894434313:2050]
2025-11-28T14:52:15.664617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789193894434313:2050] [node 3] ICS09 handshake done sender: [6:7577789193894434303:2122] self: [6:7577789193894434307:0] peer: [3:7577789193127205029:0] socket: 468 qp: -1
2025-11-28T14:52:15.665126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789193894434313:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:15.666504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.669744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789193894434276:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789193894434304:2123] held: no
2025-11-28T14:52:15.670072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789193894434312:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45079
2025-11-28T14:52:15.671166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789193894434304:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:15.671300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789193894434276:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.671657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:15.671721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789193894434276:2106] [node 5] ICP22 created new session: [6:7577789193894434315:2106]
2025-11-28T14:52:15.671776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789193894434315:2106] [node 5] ICS09 handshake done sender: [6:7577789193894434304:2123] self: [6:7577789193894434311:0] peer: [5:7577789194653196160:0] socket: 470 qp: -1
2025-11-28T14:52:15.671820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789193894434315:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:15.671859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:15.673246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789194945054622:2115] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.691690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789193127205040:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.691824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789188832237661:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.692053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789194945054622:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.692550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789194945054622:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.692755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789190650087252:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.692831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:15.692888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789190650087252:2049] [node 3] ICP22 created new session: [2:7577789194945054652:2049]
2025-11-28T14:52:15.692950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789194945054652:2049] [node 3] ICS09 handshake done sender: [2:7577789194945054622:2115] self: [2:7577789194945054621:0] peer: [3:7577789193127205055:0] socket: 407 qp: -1
2025-11-28T14:52:15.692896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789193127205056:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63046
2025-11-28T14:52:15.693001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789194945054652:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:15.693054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:15.694045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789193127205040:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:15.694522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789188832237661:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5460) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789193127205040:2118] held: no
2025-11-28T14:52:15.694631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789188832237661:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.694674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:15.694713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789188832237661:2049] [node 2] ICP22 created new session: [3:7577789193127205057:2049]
2025-11-28T14:52:15.694751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789193127205057:2049] [node 2] ICS09 handshake done sender: [3:7577789193127205040:2118] self: [3:7577789193127205055:0] peer: [2:7577789194945054621:0] socket: 462 qp: -1
2025-11-28T14:52:15.694782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789193127205057:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:15.694813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:15.697389Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789193127205030:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:15.698135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789194653196180:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.698304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789190358228781:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:15.698540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789193127205030:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:15.698973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789193127205030:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:15.699203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789194653196186:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51549
2025-11-28T14:52:15.699229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789193127205020:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:15.699342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:15.699572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789194653196180:2131] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:15.699406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789193127205020:2108] [node 5] ICP22 created new session: [3:7577789193127205060:2108]
2025-11-28T14:52:15.699530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789193127205060:2108] [node 5] ICS09 handshake done sender: [3:7577789193127205030:2113] self: [3:7577789193127205028:0] peer: [5:7577789194653196185:0] socket: 430 qp: -1
2025-11-28T14:52:15.699571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789193127205060:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:15.699801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789190358228781:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:15.699614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:15.699848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:15.699879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789190358228781:2050] [node 3] ICP22 created new session: [5:7577789194653196187:2050]
2025-11-28T14:52:15.699941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789194653196187:2050] [node 3] ICS09 handshake done sender: [5:7577789194653196180:2131] self: [5:7577789194653196185:0] peer: [3:7577789193127205028:0] socket: 469 qp: -1
2025-11-28T14:52:15.699975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789194653196187:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:15.700013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:15.743490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:15.743690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789194945054637:2048] [node 1] ICS01 socket: 414 reason# EndOfStream
2025-11-28T14:52:15.743726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789190650087251:2048] [node 1] ICP30 unregister session Session# [2:7577789194945054637:2048] VirtualId# [2:7577789194945054634:0]
2025-11-28T14:52:15.743748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789194945054637:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.743821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:15.743879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:15.744445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:15.744674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789193127205049:2048] [node 1] ICS01 socket: 423 reason# EndOfStream
2025-11-28T14:52:15.744732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789188832237660:2048] [node 1] ICP30 unregister session Session# [3:7577789193127205049:2048] VirtualId# [3:7577789193127205039:0]
2025-11-28T14:52:15.744761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789193127205049:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.744848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:15.744913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:15.747145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:15.747401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789194038636439:2048] [node 1] ICS01 socket: 419 reason# EndOfStream
2025-11-28T14:52:15.747456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789189743669042:2048] [node 1] ICP30 unregister session Session# [4:7577789194038636439:2048] VirtualId# [4:7577789194038636437:0]
2025-11-28T14:52:15.747498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789194038636439:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.747614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:15.747681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:15.747825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:15.747980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789194653196178:2048] [node 1] ICS01 socket: 421 reason# EndOfStream
2025-11-28T14:52:15.748026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789190358228779:2048] [node 1] ICP30 unregister session Session# [5:7577789194653196178:2048] VirtualId# [5:7577789194653196168:0]
2025-11-28T14:52:15.748075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789194653196178:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.748148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:15.748213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:15.748711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:15.748950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789193894434294:2048] [node 1] ICS07 socket disconnect 448 reason# EndOfStream
2025-11-28T14:52:15.748985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789193894434294:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.749053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:15.749096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789193894434294:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:15.749806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789189599466915:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10551) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:15.749951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789193894434294:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:15.749988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789189599466915:2048] [node 1] ICP30 unregister session Session# [6:7577789193894434294:2048] VirtualId# [6:7577789193894434292:0]
2025-11-28T14:52:15.750041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:15.750076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789189599466915:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.750111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.757128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:15.757399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789193127205057:2049] [node 2] ICS01 socket: 462 reason# EndOfStream
2025-11-28T14:52:15.757453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789188832237661:2049] [node 2] ICP30 unregister session Session# [3:7577789193127205057:2049] VirtualId# [3:7577789193127205055:0]
2025-11-28T14:52:15.757488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789193127205057:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.757586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:15.757689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:15.760034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789189599466915:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:15.761655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:15.761923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789194038636456:2049] [node 2] ICS01 socket: 420 reason# EndOfStream
2025-11-28T14:52:15.761978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789189743669043:2049] [node 2] ICP30 unregister session Session# [4:7577789194038636456:2049] VirtualId# [4:7577789194038636454:0]
2025-11-28T14:52:15.762017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789194038636456:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.762143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:15.762203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:15.763700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:15.763955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789193894434308:2049] [node 2] ICS07 socket disconnect 457 reason# EndOfStream
2025-11-28T14:52:15.764010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789193894434308:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.764096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:15.764154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789193894434308:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:15.764462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:15.764591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789194653196174:2049] [node 2] ICS01 socket: 431 reason# EndOfStream
2025-11-28T14:52:15.764632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789190358228780:2049] [node 2] ICP30 unregister session Session# [5:7577789194653196174:2049] VirtualId# [5:7577789194653196167:0]
2025-11-28T14:52:15.764653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789194653196174:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.764709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:15.764744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:15.764871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789189599466916:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5460) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:15.765004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789193894434308:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:15.765063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789189599466916:2049] [node 2] ICP30 unregister session Session# [6:7577789193894434308:2049] VirtualId# [6:7577789193894434305:0]
2025-11-28T14:52:15.765129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:15.765160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789189599466916:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5460) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.765190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5460) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.773164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:15.773453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789194038636451:2050] [node 3] ICS01 socket: 437 reason# EndOfStream
2025-11-28T14:52:15.773545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789189743669044:2050] [node 3] ICP30 unregister session Session# [4:7577789194038636451:2050] VirtualId# [4:7577789194038636442:0]
2025-11-28T14:52:15.773590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789194038636451:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.773684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:15.773743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:15.773951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:15.774168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789194653196187:2050] [node 3] ICS07 socket disconnect 469 reason# EndOfStream
2025-11-28T14:52:15.774207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789194653196187:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.774302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:15.774360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789194653196187:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:15.774717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:15.774894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789193894434313:2050] [node 3] ICS07 socket disconnect 468 reason# EndOfStream
2025-11-28T14:52:15.774917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789193894434313:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.774973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:15.774992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789190358228781:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18166) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:15.775341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789193894434313:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:15.775373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789189599466916:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:15.776445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789194653196187:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:15.776488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789190358228781:2050] [node 3] ICP30 unregister session Session# [5:7577789194653196187:2050] VirtualId# [5:7577789194653196185:0]
2025-11-28T14:52:15.776562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:15.776596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789190358228781:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.776629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.777295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789189599466917:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18166) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:15.777336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789193894434313:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:15.777394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789189599466917:2050] [node 3] ICP30 unregister session Session# [6:7577789193894434313:2050] VirtualId# [6:7577789193894434307:0]
2025-11-28T14:52:15.777436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:15.777462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789189599466917:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.777471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.786686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789190358228781:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:15.787659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789189599466917:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:15.821931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:15.822101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789194653196176:2108] [node 4] ICS07 socket disconnect 451 reason# EndOfStream
2025-11-28T14:52:15.822140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789194653196176:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.822262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:15.822310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789194653196176:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:15.823021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789194653196148:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12223) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:15.823060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:15.823063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789194653196176:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:15.823131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789194653196148:2108] [node 4] ICP30 unregister session Session# [5:7577789194653196176:2108] VirtualId# [5:7577789194653196169:0]
2025-11-28T14:52:15.823199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:15.823240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789194653196148:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12223) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.823318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789193894434301:2108] [node 4] ICS01 socket: 455 reason# EndOfStream
2025-11-28T14:52:15.823261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12223) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:15.823391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789193894434277:2108] [node 4] ICP30 unregister session Session# [6:7577789193894434301:2108] VirtualId# [6:7577789193894434299:0]
2025-11-28T14:52:15.823444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789193894434301:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.823546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:15.823620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:15.832378Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:15.832647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789193894434315:2106] [node 5] ICS01 socket: 470 reason# EndOfStream
2025-11-28T14:52:15.832697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789193894434276:2106] [node 5] ICP30 unregister session Session# [6:7577789193894434315:2106] VirtualId# [6:7577789193894434311:0]
2025-11-28T14:52:15.832731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789193894434315:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:15.832819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:15.832894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:15.987914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789192547238306:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:15.987937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789192547238307:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:16.010052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789200389129232:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.010101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789200389129233:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:16.030942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789197940781016:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.030973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789197940781017:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:16.061213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789197872650712:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.061273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789197872650713:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:16.061393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789197872650714:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:16.093410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789196882027408:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.093415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789196882027409:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:16.093736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789196882027410:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:16.122718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789200140442661:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:16.122779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789200140442662:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:16.122975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789200140442660:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.266618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789196842205665:2107] [node 6] ICP01 ready to work
2025-11-28T14:52:16.264505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789200140442721:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:16.266793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789200389129295:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:16.267959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789197940781080:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:16.267671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789197872650776:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:16.268307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789197940781079:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:16.268311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789200389129296:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:16.268516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789196842205666:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:16.269253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789200389129297:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:16.269805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789196882027480:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:16.269674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789197872650777:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:16.270088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789197940781081:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:16.269252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789200140442723:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:16.270763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789196842205667:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:16.277792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789192547238307:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(78897777ns) {Configure:104(78892459ns) {ProcessConfigured:135(3108778ns) {StartInitialHandshake:157(3105979ns) {DropHandshakes:470(521948ns) {DropIncomingHandshake:442(892ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(1058ns) PrepareNewSessionHandshake:477(2550ns) {SwitchToState:179(911ns)}}}}}
2025-11-28T14:52:16.278309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789192547238306:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(77501897ns) {Configure:104(77491386ns) {ProcessConfigured:135(801657ns) {StartInitialHandshake:157(798943ns) {DropHandshakes:470(2104ns) {DropIncomingHandshake:442(631ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(16764ns) PrepareNewSessionHandshake:477(1796ns) {SwitchToState:179(605ns)}}}}}
2025-11-28T14:52:16.278470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789196882027409:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(72580472ns) {Configure:104(72576287ns) {ProcessConfigured:135(356426ns) {StartInitialHandshake:157(354466ns) {DropHandshakes:470(1117ns) {DropIncomingHandshake:442(392ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(364ns) PrepareNewSessionHandshake:477(836ns) {SwitchToState:179(292ns)}}}}}
2025-11-28T14:52:16.278928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789196882027408:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39448691ns) {Configure:104(39441938ns) {ProcessConfigured:135(499411ns) {StartInitialHandshake:157(497381ns) {DropHandshakes:470(917ns) {DropIncomingHandshake:442(237ns) DropOutgoingHandshake:457(170ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:477(1309ns) {SwitchToState:179(454ns)}}}}}
2025-11-28T14:52:16.279141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789200140442660:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17504759ns) {Configure:104(17499564ns) {ProcessConfigured:135(172448ns) {StartInitialHandshake:157(170192ns) {DropHandshakes:470(921ns) {DropIncomingHandshake:442(121ns) DropOutgoingHandshake:457(164ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(2899ns) {SwitchToState:179(1393ns)}}}}}
2025-11-28T14:52:16.279176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789196882027486:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:16.279698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789197872650712:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18370218ns) {Configure:104(18355245ns) {ProcessConfigured:135(578791ns) {StartInitialHandshake:157(576509ns) {DropHandshakes:470(1121ns) {DropIncomingHandshake:442(394ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(739ns) PrepareNewSessionHandshake:477(1518ns) {SwitchToState:179(577ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.291481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442738:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.290883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789197940781095:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27918
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.296062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789196882027495:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28742
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.295983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789197940781098:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27918
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.299692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789197940781096:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27918
No RDMA context found for GID: 2025-11-28T14:52:16.296454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789196882027496:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28742
::ffff:127.1.0.0
2025-11-28T14:52:16.301446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789196882027497:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28742
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.301944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789196842205681:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51833
2025-11-28T14:52:16.304224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789196882027495:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.304789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789197940781097:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27918
2025-11-28T14:52:16.306136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789197940781098:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.304784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789196842205684:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51833
2025-11-28T14:52:16.304886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789196882027480:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.306819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789200389129312:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33851
2025-11-28T14:52:16.306709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789200140442738:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.306226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789196882027496:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.306832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789200389129310:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33851
2025-11-28T14:52:16.306928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789196882027410:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.305069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789196842205683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51833
2025-11-28T14:52:16.306893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789200389129313:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33851
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.306969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789197940781095:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.305765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789196842205682:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51833
2025-11-28T14:52:16.306997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789200140442660:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.306881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789197872650782:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.307360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789197940781016:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.306929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789197872650782:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:16.322604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789200389129314:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33851
2025-11-28T14:52:16.306840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789196842205685:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51833
2025-11-28T14:52:16.315128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789197940781085:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.306911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205683:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.315181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789197940781085:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:16.307340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205684:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.322604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205673:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.333276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789197940781085:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:16.322667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789196842205673:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.322710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205685:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.330329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205669:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.330363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789196842205669:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:16.333940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789197940781102:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27918
2025-11-28T14:52:16.333682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789196842205669:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:16.334178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789197940781080:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.334222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789197872650782:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:16.334165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789196842205673:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:16.334361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789197940781095:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:16.335052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789192547238307:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.334401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:16.335046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789197872650776:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.335116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:16.334470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789197940781080:2108] [node 5] ICP22 created new session: [3:7577789197940781103:2108]
2025-11-28T14:52:16.335098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:16.335383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789192547238307:2049] [node 3] ICP22 created new session: [1:7577789196842205691:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.335374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789197872650776:2108] [node 5] ICP22 created new session: [4:7577789197872650792:2108]
2025-11-28T14:52:16.334762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789197940781103:2108] [node 5] ICS09 handshake done sender: [3:7577789197940781085:2110] self: [3:7577789197940781082:0] peer: [5:7577789196882027499:0] socket: 415 qp: -1
2025-11-28T14:52:16.335540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789196842205665:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.335988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789197872650792:2108] [node 5] ICS09 handshake done sender: [4:7577789197872650782:2112] self: [4:7577789197872650781:0] peer: [5:7577789196882027498:0] socket: 409 qp: -1
2025-11-28T14:52:16.334886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789197940781103:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:16.336693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442740:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
2025-11-28T14:52:16.336022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789197872650792:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:16.334940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:16.335582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:16.336388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:16.335194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789197940781016:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.335783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789196842205691:2049] [node 3] ICS09 handshake done sender: [1:7577789196842205669:2110] self: [1:7577789196842205668:0] peer: [3:7577789197940781100:0] socket: 413 qp: -1
2025-11-28T14:52:16.335824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789196842205691:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:16.335516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:16.335979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789196842205665:2107] [node 6] ICP22 created new session: [1:7577789196842205692:2107]
2025-11-28T14:52:16.335650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789197940781016:2048] [node 1] ICP22 created new session: [3:7577789197940781105:2048]
2025-11-28T14:52:16.336023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:16.338147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789200140442738:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:16.336342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789197940781105:2048] [node 1] ICS09 handshake done sender: [3:7577789197940781095:2115] self: [3:7577789197940781100:0] peer: [1:7577789196842205668:0] socket: 430 qp: -1
2025-11-28T14:52:16.336607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789196842205692:2107] [node 6] ICS09 handshake done sender: [1:7577789196842205673:2112] self: [1:7577789196842205672:0] peer: [6:7577789200140442739:0] socket: 417 qp: -1
2025-11-28T14:52:16.336384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789197940781105:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:16.338553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789200140442660:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.336636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789196842205692:2107] [node 6] ICS10 traffic start
2025-11-28T14:52:16.336725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:16.336413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:16.338744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:16.339345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789200140442660:2048] [node 1] ICP22 created new session: [6:7577789200140442741:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.339511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789200140442741:2048] [node 1] ICS09 handshake done sender: [6:7577789200140442738:2115] self: [6:7577789200140442739:0] peer: [1:7577789196842205672:0] socket: 434 qp: -1
2025-11-28T14:52:16.340348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789197940781108:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27918
2025-11-28T14:52:16.339657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789200140442741:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.339700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:16.345414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442743:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.350900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789200140442743:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.350979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789197940781108:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.351231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789200140442662:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.351202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789197872650796:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42267
2025-11-28T14:52:16.351335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789197872650795:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42267
2025-11-28T14:52:16.352052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789197872650795:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.351599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789197940781094:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.351635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789197940781094:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:16.352252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789197872650714:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.352904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789197940781093:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.352934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789197940781093:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.353651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789197872650796:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.353794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789197940781094:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:16.353828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789197872650712:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.353757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442746:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
2025-11-28T14:52:16.354008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789200140442743:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.353963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789197940781093:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:16.354078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789197940781079:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.354361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789197872650799:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42267
2025-11-28T14:52:16.354365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205676:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.354138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:16.354401Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789196842205676:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:16.354247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789200140442662:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.354236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789197940781079:2109] [node 6] ICP22 created new session: [3:7577789197940781113:2109]
2025-11-28T14:52:16.354410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:16.355333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789200140442662:2050] [node 3] ICP22 created new session: [6:7577789200140442747:2050]
2025-11-28T14:52:16.356292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789197872650795:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:16.355250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789197940781113:2109] [node 6] ICS09 handshake done sender: [3:7577789197940781094:2114] self: [3:7577789197940781091:0] peer: [6:7577789200140442745:0] socket: 457 qp: -1
2025-11-28T14:52:16.357775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789196842205676:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:16.355448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789197940781081:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.356113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789200140442747:2050] [node 3] ICS09 handshake done sender: [6:7577789200140442743:2117] self: [6:7577789200140442745:0] peer: [3:7577789197940781091:0] socket: 443 qp: -1
2025-11-28T14:52:16.355502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789197940781113:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:16.358184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789197872650714:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.355549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:16.358234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:16.356154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789200140442747:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:16.356121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:16.358873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789196842205666:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.357911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:16.356263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789197940781081:2107] [node 4] ICP22 created new session: [3:7577789197940781115:2107]
2025-11-28T14:52:16.358936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:16.357640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789197940781115:2107] [node 4] ICS09 handshake done sender: [3:7577789197940781093:2113] self: [3:7577789197940781092:0] peer: [4:7577789197872650797:0] socket: 458 qp: -1
2025-11-28T14:52:16.358984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789196842205666:2108] [node 4] ICP22 created new session: [1:7577789196842205698:2108]
2025-11-28T14:52:16.357683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789197940781115:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.358921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789197872650714:2050] [node 3] ICP22 created new session: [4:7577789197872650802:2050]
2025-11-28T14:52:16.357845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.358969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789197872650802:2050] [node 3] ICS09 handshake done sender: [4:7577789197872650795:2116] self: [4:7577789197872650797:0] peer: [3:7577789197940781092:0] socket: 445 qp: -1
2025-11-28T14:52:16.359022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789196842205698:2108] [node 4] ICS09 handshake done sender: [1:7577789196842205676:2113] self: [1:7577789196842205674:0] peer: [4:7577789197872650798:0] socket: 459 qp: -1
2025-11-28T14:52:16.358996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789197872650802:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:16.359053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789196842205698:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:16.359029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:16.359077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:16.359071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789197872650800:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42267
2025-11-28T14:52:16.359154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789197872650801:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42267
2025-11-28T14:52:16.360575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789197872650796:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:16.360957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789197872650712:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31178) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789197872650796:2117] held: no
2025-11-28T14:52:16.361120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789197872650712:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.361159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:16.361483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789197872650712:2048] [node 1] ICP22 created new session: [4:7577789197872650804:2048]
2025-11-28T14:52:16.361523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789197872650804:2048] [node 1] ICS09 handshake done sender: [4:7577789197872650796:2117] self: [4:7577789197872650798:0] peer: [1:7577789196842205674:0] socket: 460 qp: -1
2025-11-28T14:52:16.361615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789197872650804:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:16.361810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.362036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442750:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
2025-11-28T14:52:16.362306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789197872650800:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.363511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789196882027483:2119] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.364524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205682:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.373204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789200140442750:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.376242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789200140442723:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.376510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789196882027500:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28742
2025-11-28T14:52:16.376753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789197872650787:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.376794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789197872650787:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.377455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789196882027501:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28742
2025-11-28T14:52:16.377827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789197872650787:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.378054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789197872650777:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.377998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789196882027502:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28742
2025-11-28T14:52:16.378565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789196882027503:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28742
2025-11-28T14:52:16.378123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:16.379199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442752:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
2025-11-28T14:52:16.378696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789196882027496:2125] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:16.379847Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789200140442723:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7077) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789200140442750:2119] held: no
2025-11-28T14:52:16.378907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789196882027495:2124] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:16.379217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789197872650777:2110] [node 6] ICP22 created new session: [4:7577789197872650809:2110]
2025-11-28T14:52:16.379560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789197872650809:2110] [node 6] ICS09 handshake done sender: [4:7577789197872650787:2115] self: [4:7577789197872650786:0] peer: [6:7577789200140442751:0] socket: 422 qp: -1
2025-11-28T14:52:16.379742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789197872650809:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:16.380195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:16.380820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789196882027410:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.380932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:16.380895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789200140442750:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:16.380990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789196882027410:2050] [node 3] ICP22 created new session: [5:7577789196882027504:2050]
2025-11-28T14:52:16.381036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789196882027504:2050] [node 3] ICS09 handshake done sender: [5:7577789196882027496:2125] self: [5:7577789196882027499:0] peer: [3:7577789197940781082:0] socket: 426 qp: -1
2025-11-28T14:52:16.381075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789196882027504:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:16.381218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:16.381252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789200140442723:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.381306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:16.381378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789200140442723:2107] [node 4] ICP22 created new session: [6:7577789200140442753:2107]
2025-11-28T14:52:16.381454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789200140442753:2107] [node 4] ICS09 handshake done sender: [6:7577789200140442750:2119] self: [6:7577789200140442751:0] peer: [4:7577789197872650786:0] socket: 438 qp: -1
2025-11-28T14:52:16.381484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789200140442753:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:16.381535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:16.381914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789196882027480:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.381993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:16.382034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789196882027480:2108] [node 4] ICP22 created new session: [5:7577789196882027506:2108]
2025-11-28T14:52:16.382080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789196882027506:2108] [node 4] ICS09 handshake done sender: [5:7577789196882027495:2124] self: [5:7577789196882027498:0] peer: [4:7577789197872650781:0] socket: 416 qp: -1
2025-11-28T14:52:16.382121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789196882027506:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:16.382187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:16.382858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789196882027502:2129] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.383867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789196882027487:2121] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.392783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789196882027408:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.393102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789197940781096:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.393193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205677:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.393257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789197940781080:2108] [node 5] ICP18 handshake (actor [3:7577789197940781096:2116]) is too old
2025-11-28T14:52:16.393237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789196842205677:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.394492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789196842205677:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:16.394475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442755:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.394730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789196842205667:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.394832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:16.394843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789196882027510:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28742
2025-11-28T14:52:16.394929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789196842205667:2109] [node 5] ICP22 created new session: [1:7577789196842205702:2109]
2025-11-28T14:52:16.394984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789196842205702:2109] [node 5] ICS09 handshake done sender: [1:7577789196842205677:2114] self: [1:7577789196842205675:0] peer: [5:7577789196882027508:0] socket: 461 qp: -1
2025-11-28T14:52:16.395025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789196842205702:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:16.395071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:16.395886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789196882027502:2129] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:16.396111Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789196882027408:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31178) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789196882027502:2129] held: no
2025-11-28T14:52:16.396225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789196882027408:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.396265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:16.396313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789196882027408:2048] [node 1] ICP22 created new session: [5:7577789196882027511:2048]
2025-11-28T14:52:16.396386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789196882027511:2048] [node 1] ICS09 handshake done sender: [5:7577789196882027502:2129] self: [5:7577789196882027508:0] peer: [1:7577789196842205675:0] socket: 468 qp: -1
2025-11-28T14:52:16.396437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789196882027511:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:16.396501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:16.397508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789200389129302:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.419331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789200389129315:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33851
2025-11-28T14:52:16.419416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789197940781097:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.419562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789200389129310:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.419568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789197940781017:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.419674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789200389129314:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.419769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789200389129315:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.419851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789200389129302:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.420435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789200389129302:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.420664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789197940781118:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27918
2025-11-28T14:52:16.420792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789200389129233:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.420868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:16.420930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789200389129233:2049] [node 3] ICP22 created new session: [2:7577789200389129320:2049]
2025-11-28T14:52:16.420939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789197940781097:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:16.420977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789200389129320:2049] [node 3] ICS09 handshake done sender: [2:7577789200389129302:2111] self: [2:7577789200389129299:0] peer: [3:7577789197940781117:0] socket: 420 qp: -1
2025-11-28T14:52:16.421012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789200389129320:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:16.421087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:16.421046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789197940781017:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.421271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:16.421331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789197940781017:2049] [node 2] ICP22 created new session: [3:7577789197940781119:2049]
2025-11-28T14:52:16.421370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789197940781119:2049] [node 2] ICS09 handshake done sender: [3:7577789197940781097:2117] self: [3:7577789197940781117:0] peer: [2:7577789200389129299:0] socket: 440 qp: -1
2025-11-28T14:52:16.421394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789197940781119:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:16.421438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:16.422045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789196882027484:2120] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.442096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789200389129312:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.443137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442756:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
2025-11-28T14:52:16.453808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789196842205671:2111] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.495402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789200389129303:2112] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.518662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789200389129313:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.518829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789200389129232:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.519167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205671:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.519702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789196842205671:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:16.519251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789196882027497:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.519475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789196882027409:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.519906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789200389129303:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.519937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789192547238306:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.520073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:16.520130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789192547238306:2048] [node 2] ICP22 created new session: [1:7577789196842205705:2048]
2025-11-28T14:52:16.520191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789196842205705:2048] [node 2] ICS09 handshake done sender: [1:7577789196842205671:2111] self: [1:7577789196842205670:0] peer: [2:7577789200389129324:0] socket: 414 qp: -1
2025-11-28T14:52:16.520234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789196842205705:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:16.520289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.520427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789200389129325:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33851
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.520819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789196882027514:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28742
2025-11-28T14:52:16.521023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789200389129303:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:16.521225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789196882027497:2126] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:16.521323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789200389129313:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:16.521583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789196882027409:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.521668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:16.521733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789196882027409:2049] [node 2] ICP22 created new session: [5:7577789196882027515:2049]
2025-11-28T14:52:16.521738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789200389129295:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.521781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789196882027515:2049] [node 2] ICS09 handshake done sender: [5:7577789196882027497:2126] self: [5:7577789196882027513:0] peer: [2:7577789200389129300:0] socket: 444 qp: -1
2025-11-28T14:52:16.521794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:16.521825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789196882027515:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:16.521833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789200389129295:2108] [node 5] ICP22 created new session: [2:7577789200389129327:2108]
2025-11-28T14:52:16.521878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789200389129327:2108] [node 5] ICS09 handshake done sender: [2:7577789200389129303:2112] self: [2:7577789200389129300:0] peer: [5:7577789196882027513:0] socket: 431 qp: -1
2025-11-28T14:52:16.521892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:16.521909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789200389129327:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:16.522001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:16.522559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789200389129232:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.522596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:16.522628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789200389129232:2048] [node 1] ICP22 created new session: [2:7577789200389129329:2048]
2025-11-28T14:52:16.522668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789200389129329:2048] [node 1] ICS09 handshake done sender: [2:7577789200389129313:2117] self: [2:7577789200389129324:0] peer: [1:7577789196842205670:0] socket: 436 qp: -1
2025-11-28T14:52:16.522711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789200389129329:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:16.522741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.525279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789197872650811:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42267
2025-11-28T14:52:16.532386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789200389129301:2110] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.533245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789196842205681:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.534770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789192547238306:2048] [node 2] ICP18 handshake (actor [1:7577789196842205681:2115]) is too old
2025-11-28T14:52:16.548588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789200140442733:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.549468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789196882027503:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.565155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789196882027490:2122] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.569592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789196882027490:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.570306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789196882027490:2122] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:16.568885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789200140442755:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.570537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789196882027486:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.569198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789200140442721:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.574646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:16.575080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789196882027486:2109] [node 6] ICP22 created new session: [5:7577789196882027519:2109]
2025-11-28T14:52:16.575147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789196882027519:2109] [node 6] ICS09 handshake done sender: [5:7577789196882027490:2122] self: [5:7577789196882027488:0] peer: [6:7577789200140442757:0] socket: 425 qp: -1
2025-11-28T14:52:16.575213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789196882027519:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.575947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:16.580124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442758:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
2025-11-28T14:52:16.580260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789200140442721:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18032) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789200140442755:2121] held: no
2025-11-28T14:52:16.581045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789200140442755:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:16.581628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789200140442721:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.581685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:16.581750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789200140442721:2106] [node 5] ICP22 created new session: [6:7577789200140442759:2106]
2025-11-28T14:52:16.581808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789200140442759:2106] [node 5] ICS09 handshake done sender: [6:7577789200140442755:2121] self: [6:7577789200140442757:0] peer: [5:7577789196882027488:0] socket: 466 qp: -1
2025-11-28T14:52:16.581858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789200140442759:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:16.581926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:16.584628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789200389129305:2113] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.593209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789200140442756:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.593463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789200140442661:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.594093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789200389129305:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.594688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789200389129305:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:16.594807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789200140442762:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57981
2025-11-28T14:52:16.595062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789200389129296:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.595123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:16.595170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789200389129296:2109] [node 6] ICP22 created new session: [2:7577789200389129333:2109]
2025-11-28T14:52:16.595139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789200140442756:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:16.595771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789200389129333:2109] [node 6] ICS09 handshake done sender: [2:7577789200389129305:2113] self: [2:7577789200389129304:0] peer: [6:7577789200140442761:0] socket: 423 qp: -1
2025-11-28T14:52:16.595801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789200140442661:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.596110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789200389129311:2115] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:16.596122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789200389129333:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:16.596163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:16.596174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:16.596268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789200140442661:2049] [node 2] ICP22 created new session: [6:7577789200140442763:2049]
2025-11-28T14:52:16.596721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789200140442763:2049] [node 2] ICS09 handshake done sender: [6:7577789200140442756:2122] self: [6:7577789200140442761:0] peer: [2:7577789200389129304:0] socket: 439 qp: -1
2025-11-28T14:52:16.596757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789200140442763:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:16.596806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:16.597847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789197872650811:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.598001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789197872650713:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:16.598184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789200389129311:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:16.598604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789200389129311:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:16.598786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789200389129297:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:16.598852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:16.598851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789197872650813:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42267
2025-11-28T14:52:16.598904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789200389129297:2107] [node 4] ICP22 created new session: [2:7577789200389129336:2107]
2025-11-28T14:52:16.598965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789200389129336:2107] [node 4] ICS09 handshake done sender: [2:7577789200389129311:2115] self: [2:7577789200389129309:0] peer: [4:7577789197872650812:0] socket: 412 qp: -1
2025-11-28T14:52:16.599007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789200389129336:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:16.599049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:16.599804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789197872650811:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:16.600077Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789197872650713:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789197872650811:2121] held: no
2025-11-28T14:52:16.600197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789197872650713:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:16.600240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:16.600290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789197872650713:2049] [node 2] ICP22 created new session: [4:7577789197872650814:2049]
2025-11-28T14:52:16.600334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789197872650814:2049] [node 2] ICS09 handshake done sender: [4:7577789197872650811:2121] self: [4:7577789197872650812:0] peer: [2:7577789200389129309:0] socket: 429 qp: -1
2025-11-28T14:52:16.600379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789197872650814:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:16.600455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:16.619056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:16.619275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789200389129329:2048] [node 1] ICS01 socket: 436 reason# EndOfStream
2025-11-28T14:52:16.619320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789200389129232:2048] [node 1] ICP30 unregister session Session# [2:7577789200389129329:2048] VirtualId# [2:7577789200389129324:0]
2025-11-28T14:52:16.619360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789200389129329:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.619465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:16.619515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:16.620029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:16.620235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789197940781105:2048] [node 1] ICS07 socket disconnect 430 reason# EndOfStream
2025-11-28T14:52:16.620268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789197940781105:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.620350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:16.620422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789197940781105:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:16.623087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:16.623312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789200140442741:2048] [node 1] ICS01 socket: 434 reason# EndOfStream
2025-11-28T14:52:16.623581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789197940781016:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31178) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:16.623635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789197940781105:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:16.623368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789200140442660:2048] [node 1] ICP30 unregister session Session# [6:7577789200140442741:2048] VirtualId# [6:7577789200140442739:0]
2025-11-28T14:52:16.623530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789200140442741:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.623990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:16.624065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:16.632166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:16.632425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789197872650804:2048] [node 1] ICS01 socket: 460 reason# EndOfStream
2025-11-28T14:52:16.632488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789197872650712:2048] [node 1] ICP30 unregister session Session# [4:7577789197872650804:2048] VirtualId# [4:7577789197872650798:0]
2025-11-28T14:52:16.632532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789197872650804:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.632626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:16.632667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:16.633397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789197940781016:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31178) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:16.633469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789197940781105:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:16.633509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789197940781016:2048] [node 1] ICP30 unregister session Session# [3:7577789197940781105:2048] VirtualId# [3:7577789197940781100:0]
2025-11-28T14:52:16.633575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:16.633612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789197940781016:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31178) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.633626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31178) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.635073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:16.635450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789196882027511:2048] [node 1] ICS01 socket: 468 reason# EndOfStream
2025-11-28T14:52:16.635537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789196882027408:2048] [node 1] ICP30 unregister session Session# [5:7577789196882027511:2048] VirtualId# [5:7577789196882027508:0]
2025-11-28T14:52:16.635608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789196882027511:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.635787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:16.635887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:16.640549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:16.640775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789197940781119:2049] [node 2] ICS07 socket disconnect 440 reason# EndOfStream
2025-11-28T14:52:16.640803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789197940781119:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.640895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:16.640938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789197940781119:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:16.644475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789197940781017:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:16.644516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789197940781119:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:16.645755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789197940781017:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:16.646563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789197940781119:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:16.646660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789197940781017:2049] [node 2] ICP30 unregister session Session# [3:7577789197940781119:2049] VirtualId# [3:7577789197940781117:0]
2025-11-28T14:52:16.646720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:16.646750Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789197940781017:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.646765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.647830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789197940781016:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:16.647757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:16.647889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789196882027515:2049] [node 2] ICS01 socket: 444 reason# EndOfStream
2025-11-28T14:52:16.647928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789196882027409:2049] [node 2] ICP30 unregister session Session# [5:7577789196882027515:2049] VirtualId# [5:7577789196882027513:0]
2025-11-28T14:52:16.647956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789196882027515:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.648020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:16.648052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:16.647868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:16.648019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789197872650814:2049] [node 2] ICS01 socket: 429 reason# EndOfStream
2025-11-28T14:52:16.648041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789197872650713:2049] [node 2] ICP30 unregister session Session# [4:7577789197872650814:2049] VirtualId# [4:7577789197872650812:0]
2025-11-28T14:52:16.648059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789197872650814:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.648115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:16.648140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:16.650224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:16.650390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789197940781017:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:16.650448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789200140442763:2049] [node 2] ICS07 socket disconnect 439 reason# EndOfStream
2025-11-28T14:52:16.650483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789200140442763:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.650559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:16.650604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789200140442763:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:16.652655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789200140442661:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:16.652833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789200140442763:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:16.652870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789200140442661:2049] [node 2] ICP30 unregister session Session# [6:7577789200140442763:2049] VirtualId# [6:7577789200140442761:0]
2025-11-28T14:52:16.652909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:16.652934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789200140442661:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.652943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.662987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789200140442661:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:16.703705Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:16.703938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789197872650802:2050] [node 3] ICS07 socket disconnect 445 reason# EndOfStream
2025-11-28T14:52:16.703995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789197872650802:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.704103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:16.704155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789197872650802:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:16.705208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789197872650714:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3693) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:16.705264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789197872650802:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:16.705569Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:16.705298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789197872650714:2050] [node 3] ICP30 unregister session Session# [4:7577789197872650802:2050] VirtualId# [4:7577789197872650797:0]
2025-11-28T14:52:16.705794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789196882027504:2050] [node 3] ICS07 socket disconnect 426 reason# EndOfStream
2025-11-28T14:52:16.705344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:16.705823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789196882027504:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.705399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789197872650714:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.705894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:16.705939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789196882027504:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:16.705415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.706358Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:16.706713Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789196882027410:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3693) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:16.706764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789196882027504:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:16.706794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789196882027410:2050] [node 3] ICP30 unregister session Session# [5:7577789196882027504:2050] VirtualId# [5:7577789196882027499:0]
2025-11-28T14:52:16.706830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:16.706570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789200140442747:2050] [node 3] ICS07 socket disconnect 443 reason# EndOfStream
2025-11-28T14:52:16.707016Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789196882027410:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.707039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.706615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789200140442747:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.706759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:16.706799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789200140442747:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:16.708162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789200140442662:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3693) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:16.708317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789200140442747:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:16.708360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789200140442662:2050] [node 3] ICP30 unregister session Session# [6:7577789200140442747:2050] VirtualId# [6:7577789200140442745:0]
2025-11-28T14:52:16.708424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:16.708448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789200140442662:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.708457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.717756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789196882027410:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:16.718511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789200140442662:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:16.752399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:16.752626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789196882027506:2108] [node 4] ICS07 socket disconnect 416 reason# EndOfStream
2025-11-28T14:52:16.752658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789196882027506:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.752752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:16.752803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789196882027506:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:16.753485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:16.753591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789196882027480:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7077) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:16.753661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789196882027506:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:16.753693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789200140442753:2107] [node 4] ICS01 socket: 438 reason# EndOfStream
2025-11-28T14:52:16.753765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789196882027480:2108] [node 4] ICP30 unregister session Session# [5:7577789196882027506:2108] VirtualId# [5:7577789196882027498:0]
2025-11-28T14:52:16.753751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789200140442723:2107] [node 4] ICP30 unregister session Session# [6:7577789200140442753:2107] VirtualId# [6:7577789200140442751:0]
2025-11-28T14:52:16.753831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:16.753798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789200140442753:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.753878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:16.753897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789196882027480:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7077) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.753934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:16.753921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7077) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:16.765633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:16.765789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789200140442759:2106] [node 5] ICS01 socket: 466 reason# EndOfStream
2025-11-28T14:52:16.765822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789200140442721:2106] [node 5] ICP30 unregister session Session# [6:7577789200140442759:2106] VirtualId# [6:7577789200140442757:0]
2025-11-28T14:52:16.765847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789200140442759:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:16.765921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:16.765959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:16.823849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789197499939658:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:16.823855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789197499939659:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:16.842639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789197895688220:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:16.842652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789197895688219:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.862552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789197083129766:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.862555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789197083129767:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:16.888052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789199061229858:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.888430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789199061229860:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:16.888518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789199061229859:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:16.909699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789196761372166:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.909735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789196761372168:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:16.909816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789196761372167:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:16.932974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789200052204980:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:16.933014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789200052204981:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:16.933041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789200052204982:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:17.011343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789204347172337:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:17.030426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789202190655596:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:17.042027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789201794907044:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:17.046902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789201378097139:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:17.050075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789202190655597:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:17.050094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789201378097140:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:17.050988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789201794907045:2129] [node 6] ICP01 ready to work
2025-11-28T14:52:17.051841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789203356197235:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:17.052640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789201378097141:2116] [node 6] ICP01 ready to work
2025-11-28T14:52:17.052425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789202190655598:2113] [node 6] ICP01 ready to work
2025-11-28T14:52:17.052038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789204347172340:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:17.048719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789201056339538:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:17.057995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789201056339539:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:17.058248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789201794907046:2110] [node 5] ICP01 ready to work
2025-11-28T14:52:17.058349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789203356197238:2114] [node 6] ICP01 ready to work
2025-11-28T14:52:17.058399Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789199061229860:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(60230979ns) {Configure:104(60222365ns) {ProcessConfigured:135(5333675ns) {StartInitialHandshake:157(5329664ns) {DropHandshakes:470(2207ns) {DropIncomingHandshake:442(553ns) DropOutgoingHandshake:457(342ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:477(3630ns) {SwitchToState:179(1575ns)}}}}}
2025-11-28T14:52:17.060878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789199061229859:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(62858144ns) {Configure:104(62851247ns) {ProcessConfigured:135(692623ns) {StartInitialHandshake:157(689955ns) {DropHandshakes:470(7367ns) {DropIncomingHandshake:442(637ns) DropOutgoingHandshake:457(376ns)} GenerateSessionVirtualId:988(667ns) PrepareNewSessionHandshake:477(1918ns) {SwitchToState:179(797ns)}}}}}
2025-11-28T14:52:17.061903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789199061229858:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(63462701ns) {Configure:104(63453762ns) {ProcessConfigured:135(199787ns) {StartInitialHandshake:157(197754ns) {DropHandshakes:470(1529ns) {DropIncomingHandshake:442(416ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:477(1028ns) {SwitchToState:179(187ns)}}}}}
2025-11-28T14:52:17.062338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789197499939658:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(72046970ns) {Configure:104(72040350ns) {ProcessConfigured:135(460157ns) {StartInitialHandshake:157(457489ns) {DropHandshakes:470(2278ns) {DropIncomingHandshake:442(535ns) DropOutgoingHandshake:457(428ns)} GenerateSessionVirtualId:988(902ns) PrepareNewSessionHandshake:477(1803ns) {SwitchToState:179(969ns)}}}}}
2025-11-28T14:52:17.062862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789197499939659:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(75250423ns) {Configure:104(72883887ns) {ProcessConfigured:135(1239721ns) {StartInitialHandshake:157(1237097ns) {DropHandshakes:470(1179ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:477(2630ns) {SwitchToState:179(917ns)}}}}}
2025-11-28T14:52:17.066010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789197895688219:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(73413903ns) {Configure:104(73408397ns) {ProcessConfigured:135(1406813ns) {StartInitialHandshake:157(1404894ns) {DropHandshakes:470(1233ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(389ns) PrepareNewSessionHandshake:477(1404ns) {SwitchToState:179(328ns)}}}}}
2025-11-28T14:52:17.067570Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789197083129767:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46172805ns) {Configure:104(46154129ns) {ProcessConfigured:135(770900ns) {StartInitialHandshake:157(769527ns) {DropHandshakes:470(704ns) {DropIncomingHandshake:442(167ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:477(1659ns) {SwitchToState:179(592ns)}}}}}
2025-11-28T14:52:17.067585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789200052204982:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40156719ns) {Configure:104(40149737ns) {ProcessConfigured:135(790209ns) {StartInitialHandshake:157(788694ns) {DropHandshakes:470(225737ns) {DropIncomingHandshake:442(481ns) DropOutgoingHandshake:457(974ns)} GenerateSessionVirtualId:988(1184ns) PrepareNewSessionHandshake:477(1604ns) {SwitchToState:179(784ns)}}}}}
2025-11-28T14:52:17.067533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789197895688220:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(76729303ns) {Configure:104(76720344ns) {ProcessConfigured:135(964139ns) {StartInitialHandshake:157(961871ns) {DropHandshakes:470(1106ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(388ns) PrepareNewSessionHandshake:477(1797ns) {SwitchToState:179(221ns)}}}}}
2025-11-28T14:52:17.070130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789197083129766:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(76671072ns) {Configure:104(76666611ns) {ProcessConfigured:135(1171752ns) {StartInitialHandshake:157(886836ns) {DropHandshakes:470(1472ns) {DropIncomingHandshake:442(251ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(770ns) PrepareNewSessionHandshake:477(1251ns) {SwitchToState:179(201ns)}}}}}
2025-11-28T14:52:17.071528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789196761372167:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46355719ns) {Configure:104(46350177ns) {ProcessConfigured:135(918361ns) {StartInitialHandshake:157(916375ns) {DropHandshakes:470(1110ns) {DropIncomingHandshake:442(353ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(515ns) PrepareNewSessionHandshake:477(1827ns) {SwitchToState:179(966ns)}}}}}
2025-11-28T14:52:17.070616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789204347172337:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36505817ns) {Configure:104(36500912ns) {ProcessConfigured:135(508351ns) {StartInitialHandshake:157(506875ns) {DropHandshakes:470(890ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(401ns) PrepareNewSessionHandshake:477(1334ns) {SwitchToState:179(328ns)}}}}}
2025-11-28T14:52:17.071265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789200052204981:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36794994ns) {Configure:104(33831548ns) {ProcessConfigured:135(2048354ns) {StartInitialHandshake:157(2046405ns) {DropHandshakes:470(1094ns) {DropIncomingHandshake:442(257ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(361ns) PrepareNewSessionHandshake:477(2845ns) {SwitchToState:179(1316ns)}}}}}
2025-11-28T14:52:17.071907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789196761372166:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(69564040ns) {Configure:104(62950002ns) {ProcessConfigured:135(2103223ns) {StartInitialHandshake:157(2101566ns) {DropHandshakes:470(1337ns) {DropIncomingHandshake:442(532ns) DropOutgoingHandshake:457(339ns)} GenerateSessionVirtualId:988(496ns) PrepareNewSessionHandshake:477(1765ns) {SwitchToState:179(599ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.087789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789201794907055:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.088645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789201794907046:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24502222ns) {Configure:104(24496461ns) {ProcessConfigured:135(2065383ns) {StartInitialHandshake:157(2059709ns) {DropHandshakes:470(2454ns) {DropIncomingHandshake:442(581ns) DropOutgoingHandshake:457(469ns)} GenerateSessionVirtualId:988(1175ns) PrepareNewSessionHandshake:477(3943ns) {SwitchToState:179(1724ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.099408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789201794907055:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.099312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789202190655607:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53787
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.099446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789203356197250:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-28T14:52:17.100194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789201378097151:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58950
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.100921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789201378097141:2116] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34567961ns) {Configure:104(34561881ns) {ProcessConfigured:135(1434433ns) {StartInitialHandshake:157(1431723ns) {DropHandshakes:470(2586ns) {DropIncomingHandshake:442(564ns) DropOutgoingHandshake:457(391ns)} GenerateSessionVirtualId:988(471ns) PrepareNewSessionHandshake:477(1433ns) {SwitchToState:179(535ns)}}}}}
2025-11-28T14:52:17.100210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172354:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.106130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789201794907061:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61260
2025-11-28T14:52:17.109162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789202190655598:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33673492ns) {Configure:104(33667298ns) {ProcessConfigured:135(734988ns) {StartInitialHandshake:157(732660ns) {DropHandshakes:470(4978ns) {DropIncomingHandshake:442(626ns) DropOutgoingHandshake:457(465ns)} GenerateSessionVirtualId:988(1285ns) PrepareNewSessionHandshake:477(1336ns) {SwitchToState:179(590ns)}}}}}
No RDMA context found for GID: 2025-11-28T14:52:17.100922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789201378097150:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58950
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.111659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789202190655609:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53787
2025-11-28T14:52:17.115030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789201056339539:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40591075ns) {Configure:104(40042186ns) {ProcessConfigured:135(2813667ns) {StartInitialHandshake:157(2809821ns) {DropHandshakes:470(2786ns) {DropIncomingHandshake:442(893ns) DropOutgoingHandshake:457(783ns)} GenerateSessionVirtualId:988(1072ns) PrepareNewSessionHandshake:477(3214ns) {SwitchToState:179(1458ns)}}}}}
2025-11-28T14:52:17.114769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789201794907061:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.113842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789203356197250:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.116336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789201794907063:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61260
No RDMA context found for GID: 2025-11-28T14:52:17.116915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789204347172354:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:17.109229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789201378097150:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.113263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789202190655607:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.118718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789200052204980:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.123299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789203356197251:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.113184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789201378097157:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58950
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.123677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789201794907052:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.114880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789202190655608:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53787
2025-11-28T14:52:17.123727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789201794907052:2139] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:17.114760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789201378097157:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.131945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789203356197254:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.126541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789201794907064:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61260
2025-11-28T14:52:17.117590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789202190655609:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.117033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789201378097158:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58950
2025-11-28T14:52:17.118918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789202190655608:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.128848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789201794907050:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.128888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789201794907050:2138] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:17.137529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789203356197253:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-28T14:52:17.123018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789197083129766:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.129890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789201794907052:2139] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:17.123375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789202190655610:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53787
2025-11-28T14:52:17.133335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789202190655616:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53787
2025-11-28T14:52:17.130232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789201794907045:2129] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.123262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789201378097151:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.130284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:17.130339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789201794907045:2129] [node 6] ICP22 created new session: [1:7577789201794907067:2129]
2025-11-28T14:52:17.128783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789201378097160:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58950
2025-11-28T14:52:17.130419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789201794907067:2129] [node 6] ICS09 handshake done sender: [1:7577789201794907052:2139] self: [1:7577789201794907051:0] peer: [6:7577789204347172355:0] socket: 435 qp: -1
2025-11-28T14:52:17.130456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789201794907067:2129] [node 6] ICS10 traffic start
2025-11-28T14:52:17.130495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:17.147937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789201794907050:2138] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:17.163957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789197499939659:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.164022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:17.166856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789197499939659:2049] [node 3] ICP22 created new session: [1:7577789201794907071:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.167045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789201794907071:2049] [node 3] ICS09 handshake done sender: [1:7577789201794907050:2138] self: [1:7577789201794907048:0] peer: [3:7577789201378097161:0] socket: 431 qp: -1
2025-11-28T14:52:17.168821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172356:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
2025-11-28T14:52:17.167098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789201794907071:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:17.167189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:17.171960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789201378097163:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58950
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.172207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789201056339553:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-28T14:52:17.172963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789201378097157:2131] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:17.173121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789204347172354:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:17.173418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789197083129766:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.173497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:17.173564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789197083129766:2048] [node 1] ICP22 created new session: [3:7577789201378097164:2048]
2025-11-28T14:52:17.173630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789201378097164:2048] [node 1] ICS09 handshake done sender: [3:7577789201378097157:2131] self: [3:7577789201378097161:0] peer: [1:7577789201794907048:0] socket: 439 qp: -1
2025-11-28T14:52:17.173673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789201378097164:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:17.173723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:17.173659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789200052204980:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.173725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:17.173777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789200052204980:2048] [node 1] ICP22 created new session: [6:7577789204347172357:2048]
2025-11-28T14:52:17.173823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789204347172357:2048] [node 1] ICS09 handshake done sender: [6:7577789204347172354:2115] self: [6:7577789204347172355:0] peer: [1:7577789201794907051:0] socket: 436 qp: -1
2025-11-28T14:52:17.174187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789201056339553:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.173864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789204347172357:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:17.174030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:17.174749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789196761372166:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.175369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789201056339555:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-28T14:52:17.175047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789201794907060:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.175093Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789201794907060:2142] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:17.176495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789201794907060:2142] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.176857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789201056339556:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-28T14:52:17.177195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789201056339553:2128] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:17.178690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789201794907046:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.180701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789201378097147:2126] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.179507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789196761372166:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.192098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:17.192183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789196761372166:2048] [node 1] ICP22 created new session: [5:7577789201056339557:2048]
2025-11-28T14:52:17.198224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789201056339557:2048] [node 1] ICS09 handshake done sender: [5:7577789201056339553:2128] self: [5:7577789201056339554:0] peer: [1:7577789201794907057:0] socket: 463 qp: -1
2025-11-28T14:52:17.198301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789201056339557:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:17.198500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:17.197689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:17.197786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789201794907046:2110] [node 5] ICP22 created new session: [1:7577789201794907075:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.198185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789201794907075:2110] [node 5] ICS09 handshake done sender: [1:7577789201794907060:2142] self: [1:7577789201794907057:0] peer: [5:7577789201056339554:0] socket: 462 qp: -1
2025-11-28T14:52:17.198432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789201794907075:2110] [node 5] ICS10 traffic start
2025-11-28T14:52:17.198502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:17.202004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789201056339559:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-28T14:52:17.202325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789201794907046:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(21773137ns) {HandleHandshakeStatus:329(21769951ns) {DropIncomingHandshake:442(667578ns) DropOutgoingHandshake:457(1062ns) SwitchToState:179(716ns) ProcessPendingSessionEvents:495(1283514ns) {ForwardSessionEventToSession:612(1219669ns) ForwardSessionEventToSession:612(9511ns) ForwardSessionEventToSession:612(10259ns) ForwardSessionEventToSession:612(2428ns) ForwardSessionEventToSession:612(2600ns) ForwardSessionEventToSession:612(11126ns) ForwardSessionEventToSession:612(2274ns) ForwardSessionEventToSession:612(2760ns) ForwardSessionEventToSession:612(9720ns) ForwardSessionEventToSession:612(2533ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.202714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789196761372166:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22667103ns) {HandleHandshakeStatus:329(22664101ns) {DropIncomingHandshake:442(2525ns) DropOutgoingHandshake:457(13236ns) SwitchToState:179(426ns) ProcessPendingSessionEvents:495(2820622ns) {ForwardSessionEventToSession:612(1067797ns) ForwardSessionEventToSession:612(11802ns) ForwardSessionEventToSession:612(419777ns) ForwardSessionEventToSession:612(287529ns) ForwardSessionEventToSession:612(3885ns) ForwardSessionEventToSession:612(74099ns) ForwardSessionEventToSession:612(3086ns) ForwardSessionEventToSession:612(2579ns) ForwardSessionEventToSession:612(390443ns) ForwardSessionEventToSession:612(4104ns)}}}
2025-11-28T14:52:17.202708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789203356197255:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-28T14:52:17.203401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789201794907063:2144] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.203724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789197499939659:2049] [node 3] ICP18 handshake (actor [1:7577789201794907063:2144]) is too old
2025-11-28T14:52:17.204519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789203356197255:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.208575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789204347172342:2110] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.237131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789201378097158:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.242314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789201794907049:2137] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.253758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789202190655610:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.253961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789197895688219:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.254363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789201794907049:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.254888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789201794907049:2137] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.255140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789197499939658:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.255198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:17.255264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789197499939658:2048] [node 2] ICP22 created new session: [1:7577789201794907078:2048]
2025-11-28T14:52:17.255166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789202190655621:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53787
2025-11-28T14:52:17.255401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789201794907078:2048] [node 2] ICS09 handshake done sender: [1:7577789201794907049:2137] self: [1:7577789201794907047:0] peer: [2:7577789202190655620:0] socket: 428 qp: -1
2025-11-28T14:52:17.255363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789201378097143:2124] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.255450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789201794907078:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:17.265608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:17.276043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789203356197251:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.279952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789202190655610:2135] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:17.280064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789199061229860:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.283738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789201378097143:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.283985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789197895688219:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.284078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:17.284131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789197895688219:2048] [node 1] ICP22 created new session: [2:7577789202190655622:2048]
2025-11-28T14:52:17.284180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789202190655622:2048] [node 1] ICS09 handshake done sender: [2:7577789202190655610:2135] self: [2:7577789202190655620:0] peer: [1:7577789201794907047:0] socket: 432 qp: -1
2025-11-28T14:52:17.284228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789202190655622:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:17.284280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:17.283866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789197499939658:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(24843859ns) {HandleHandshakeStatus:329(24840778ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(770ns) SwitchToState:179(531ns) ProcessPendingSessionEvents:495(10363448ns) {ForwardSessionEventToSession:612(10269246ns) ForwardSessionEventToSession:612(10148ns) ForwardSessionEventToSession:612(13860ns) ForwardSessionEventToSession:612(10102ns) ForwardSessionEventToSession:612(9491ns) ForwardSessionEventToSession:612(2893ns) ForwardSessionEventToSession:612(3149ns) ForwardSessionEventToSession:612(14517ns) ForwardSessionEventToSession:612(3243ns) ForwardSessionEventToSession:612(4121ns)}}}
2025-11-28T14:52:17.284436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789201378097143:2124] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.284663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789201378097139:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.284906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:17.285189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789201378097139:2107] [node 4] ICP22 created new session: [3:7577789201378097169:2107]
2025-11-28T14:52:17.285244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789201378097169:2107] [node 4] ICS09 handshake done sender: [3:7577789201378097143:2124] self: [3:7577789201378097142:0] peer: [4:7577789203356197257:0] socket: 443 qp: -1
2025-11-28T14:52:17.285166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789203356197258:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-28T14:52:17.285317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789201378097169:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:17.285374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:17.285536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789203356197251:2134] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:17.285793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789199061229860:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.285983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:17.286228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789199061229860:2050] [node 3] ICP22 created new session: [4:7577789203356197259:2050]
2025-11-28T14:52:17.289730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789203356197259:2050] [node 3] ICS09 handshake done sender: [4:7577789203356197251:2134] self: [4:7577789203356197257:0] peer: [3:7577789201378097142:0] socket: 444 qp: -1
2025-11-28T14:52:17.289850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789203356197259:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:17.289932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.290897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172360:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.298120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172361:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
2025-11-28T14:52:17.298671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789204347172361:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.298937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789204347172340:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.299203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789203356197247:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.299250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789203356197247:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:17.300664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789203356197247:2131] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.301670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789203356197238:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.302870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172363:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
2025-11-28T14:52:17.302845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:17.302906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789203356197238:2114] [node 6] ICP22 created new session: [4:7577789203356197263:2114]
2025-11-28T14:52:17.302989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789203356197263:2114] [node 6] ICS09 handshake done sender: [4:7577789203356197247:2131] self: [4:7577789203356197245:0] peer: [6:7577789204347172362:0] socket: 414 qp: -1
2025-11-28T14:52:17.303034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789203356197263:2114] [node 6] ICS10 traffic start
2025-11-28T14:52:17.303104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.305565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789201056339561:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-28T14:52:17.304772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789204347172361:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:17.305449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789204347172340:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8199) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789204347172361:2118] held: no
2025-11-28T14:52:17.307823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789204347172340:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.308447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:17.308528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789204347172340:2107] [node 4] ICP22 created new session: [6:7577789204347172364:2107]
2025-11-28T14:52:17.308860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789201056339561:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.309135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789201056339538:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.308609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789204347172364:2107] [node 4] ICS09 handshake done sender: [6:7577789204347172361:2118] self: [6:7577789204347172362:0] peer: [4:7577789203356197245:0] socket: 473 qp: -1
2025-11-28T14:52:17.308660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789204347172364:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:17.309452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789203356197249:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.308737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:17.309480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789203356197249:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:17.310361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789203356197249:2132] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.310620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789203356197235:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.310721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:17.310717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789201056339563:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-28T14:52:17.310777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789203356197235:2108] [node 5] ICP22 created new session: [4:7577789203356197267:2108]
2025-11-28T14:52:17.310854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789203356197267:2108] [node 5] ICS09 handshake done sender: [4:7577789203356197249:2132] self: [4:7577789203356197248:0] peer: [5:7577789201056339562:0] socket: 424 qp: -1
2025-11-28T14:52:17.310894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789203356197267:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:17.310943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:17.310998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789201056339561:2132] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:17.311196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789201056339538:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.311256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:17.311685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789201056339538:2108] [node 4] ICP22 created new session: [5:7577789201056339564:2108]
2025-11-28T14:52:17.311838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789201056339564:2108] [node 4] ICS09 handshake done sender: [5:7577789201056339561:2132] self: [5:7577789201056339562:0] peer: [4:7577789203356197248:0] socket: 476 qp: -1
2025-11-28T14:52:17.311901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789201056339564:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:17.311964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:17.312387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789201056339538:2108] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.313133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789202190655601:2128] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.314307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789201794907064:2145] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.314467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789197499939658:2048] [node 2] ICP18 handshake (actor [1:7577789201794907064:2145]) is too old
2025-11-28T14:52:17.316564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.344761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789201056339567:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-28T14:52:17.345023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789204347172366:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.345296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789204347172337:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.345520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789201056339552:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.345556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789201056339552:2127] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:17.346389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789201056339552:2127] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.346583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789201056339539:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.346651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:17.346633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172368:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
2025-11-28T14:52:17.346737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789201056339539:2109] [node 6] ICP22 created new session: [5:7577789201056339569:2109]
2025-11-28T14:52:17.346839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789201056339569:2109] [node 6] ICS09 handshake done sender: [5:7577789201056339552:2127] self: [5:7577789201056339551:0] peer: [6:7577789204347172367:0] socket: 430 qp: -1
2025-11-28T14:52:17.346873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789201056339569:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:17.346907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:17.346910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789204347172366:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:17.347122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789204347172337:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.347200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:17.347237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789204347172337:2106] [node 5] ICP22 created new session: [6:7577789204347172369:2106]
2025-11-28T14:52:17.347293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789204347172369:2106] [node 5] ICS09 handshake done sender: [6:7577789204347172366:2120] self: [6:7577789204347172367:0] peer: [5:7577789201056339551:0] socket: 437 qp: -1
2025-11-28T14:52:17.347318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789204347172369:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:17.347384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:17.366863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789201378097146:2125] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.393274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789202190655616:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.405865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789202190655602:2129] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.433609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789201378097160:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.433819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789197083129767:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.434127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789202190655602:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.434660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789202190655602:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:17.434908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789197895688220:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.434987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.435085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789197895688220:2049] [node 3] ICP22 created new session: [2:7577789202190655627:2049]
2025-11-28T14:52:17.435139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789202190655627:2049] [node 3] ICS09 handshake done sender: [2:7577789202190655602:2129] self: [2:7577789202190655600:0] peer: [3:7577789201378097172:0] socket: 447 qp: -1
2025-11-28T14:52:17.435225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789202190655627:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:17.435274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789201378097173:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58950
2025-11-28T14:52:17.435266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:17.435591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789201378097160:2133] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:17.436437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789197083129767:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7122) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789201378097160:2133] held: no
2025-11-28T14:52:17.436575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789197083129767:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.436660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:17.436733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789197083129767:2049] [node 2] ICP22 created new session: [3:7577789201378097174:2049]
2025-11-28T14:52:17.436814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789201378097174:2049] [node 2] ICS09 handshake done sender: [3:7577789201378097160:2133] self: [3:7577789201378097172:0] peer: [2:7577789202190655600:0] socket: 448 qp: -1
2025-11-28T14:52:17.436861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789201378097174:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:17.436915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.438801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172371:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
2025-11-28T14:52:17.443078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789202190655606:2131] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.460459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789203356197253:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.460731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789199061229859:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.461065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789202190655606:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.461571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789202190655606:2131] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:17.461818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789202190655596:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.461890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:17.462001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789202190655596:2107] [node 4] ICP22 created new session: [2:7577789202190655631:2107]
2025-11-28T14:52:17.462062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789202190655631:2107] [node 4] ICS09 handshake done sender: [2:7577789202190655606:2131] self: [2:7577789202190655604:0] peer: [4:7577789203356197269:0] socket: 451 qp: -1
2025-11-28T14:52:17.462093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789203356197270:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-28T14:52:17.462102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789202190655631:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:17.462158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:17.462380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789203356197253:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:17.462737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789199061229859:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7122) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789203356197253:2135] held: no
2025-11-28T14:52:17.463581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789199061229859:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.463649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:17.463834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789199061229859:2049] [node 2] ICP22 created new session: [4:7577789203356197271:2049]
2025-11-28T14:52:17.463896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789203356197271:2049] [node 2] ICS09 handshake done sender: [4:7577789203356197253:2135] self: [4:7577789203356197269:0] peer: [2:7577789202190655604:0] socket: 454 qp: -1
2025-11-28T14:52:17.463939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789203356197271:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:17.463984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:17.468428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789201794907054:2140] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.496344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789203356197254:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.496673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789199061229858:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.496932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789201794907054:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.497449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789201794907054:2140] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.497668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789201794907044:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.497738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:17.497777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789201794907044:2107] [node 4] ICP22 created new session: [1:7577789201794907081:2107]
2025-11-28T14:52:17.497786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789203356197274:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-28T14:52:17.497909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789201794907081:2107] [node 4] ICS09 handshake done sender: [1:7577789201794907054:2140] self: [1:7577789201794907053:0] peer: [4:7577789203356197273:0] socket: 458 qp: -1
2025-11-28T14:52:17.497956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789201794907081:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:17.497991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:17.498166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789203356197254:2136] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:17.498354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789199061229858:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789203356197254:2136] held: no
2025-11-28T14:52:17.499284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789199061229858:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.499465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:17.499534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789199061229858:2048] [node 1] ICP22 created new session: [4:7577789203356197275:2048]
2025-11-28T14:52:17.499590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789203356197275:2048] [node 1] ICS09 handshake done sender: [4:7577789203356197254:2136] self: [4:7577789203356197273:0] peer: [1:7577789201794907053:0] socket: 459 qp: -1
2025-11-28T14:52:17.499637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789203356197275:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:17.499684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:17.502851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [6:7577789204347172348:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.524743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789201056339555:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.525082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789201056339539:2109] [node 6] ICP18 handshake (actor [5:7577789201056339555:2129]) is too old
2025-11-28T14:52:17.529770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789201378097149:2127] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.546846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789201056339559:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.547038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789196761372168:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.547380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789201378097149:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.548178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789201378097149:2127] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.548415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789201056339572:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-28T14:52:17.548608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789201378097140:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.548756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:17.549184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789201056339559:2131] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:17.549136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789201378097140:2108] [node 5] ICP22 created new session: [3:7577789201378097177:2108]
2025-11-28T14:52:17.549217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789201378097177:2108] [node 5] ICS09 handshake done sender: [3:7577789201378097149:2127] self: [3:7577789201378097148:0] peer: [5:7577789201056339571:0] socket: 442 qp: -1
2025-11-28T14:52:17.549404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789196761372168:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.549629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:17.549256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789201378097177:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:17.549673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789196761372168:2050] [node 3] ICP22 created new session: [5:7577789201056339573:2050]
2025-11-28T14:52:17.549445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:17.549736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789201056339573:2050] [node 3] ICS09 handshake done sender: [5:7577789201056339559:2131] self: [5:7577789201056339571:0] peer: [3:7577789201378097148:0] socket: 468 qp: -1
2025-11-28T14:52:17.549772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789201056339573:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:17.549813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:17.551455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789196761372168:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:17.556264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789201378097153:2130] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.582580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789204347172360:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.582890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789200052204982:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.583175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789201378097153:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.583708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789201378097153:2130] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.583993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789201378097141:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.584127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:17.584175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172373:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
2025-11-28T14:52:17.584211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789201378097141:2116] [node 6] ICP22 created new session: [3:7577789201378097180:2116]
2025-11-28T14:52:17.584303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789201378097180:2116] [node 6] ICS09 handshake done sender: [3:7577789201378097153:2130] self: [3:7577789201378097152:0] peer: [6:7577789204347172372:0] socket: 420 qp: -1
2025-11-28T14:52:17.584349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789201378097180:2116] [node 6] ICS10 traffic start
2025-11-28T14:52:17.584490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789204347172360:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:17.584456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:17.585008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789200052204982:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18150) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789204347172360:2117] held: no
2025-11-28T14:52:17.585306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789200052204982:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.585377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:17.585435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789200052204982:2050] [node 3] ICP22 created new session: [6:7577789204347172374:2050]
2025-11-28T14:52:17.585585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789204347172374:2050] [node 3] ICS09 handshake done sender: [6:7577789204347172360:2117] self: [6:7577789204347172372:0] peer: [3:7577789201378097152:0] socket: 472 qp: -1
2025-11-28T14:52:17.585649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789204347172374:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:17.585715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:17.586909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789202190655605:2130] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.599629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789201056339567:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.599856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789196761372167:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.600154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789202190655605:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:17.600771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789202190655605:2130] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:17.600887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789201056339576:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-28T14:52:17.601000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789202190655597:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.601070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:17.601140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789202190655597:2108] [node 5] ICP22 created new session: [2:7577789202190655634:2108]
2025-11-28T14:52:17.601167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789201056339567:2134] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:17.601191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789202190655634:2108] [node 5] ICS09 handshake done sender: [2:7577789202190655605:2130] self: [2:7577789202190655603:0] peer: [5:7577789201056339575:0] socket: 478 qp: -1
2025-11-28T14:52:17.601334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789202190655634:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:17.601620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789196761372167:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.601427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:17.601759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:17.601901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789196761372167:2049] [node 2] ICP22 created new session: [5:7577789201056339577:2049]
2025-11-28T14:52:17.602017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789201056339577:2049] [node 2] ICS09 handshake done sender: [5:7577789201056339567:2134] self: [5:7577789201056339575:0] peer: [2:7577789202190655603:0] socket: 479 qp: -1
2025-11-28T14:52:17.602076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789201056339577:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:17.602199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:17.602598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789196761372167:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:17.605505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789202190655613:2136] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:17.606280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789204347172371:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.606535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789200052204981:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:17.606864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789202190655613:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:17.607351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789202190655613:2136] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:17.607554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789202190655598:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:17.607588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789204347172377:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:883
2025-11-28T14:52:17.607635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:17.607701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789202190655598:2113] [node 6] ICP22 created new session: [2:7577789202190655637:2113]
2025-11-28T14:52:17.607747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789202190655637:2113] [node 6] ICS09 handshake done sender: [2:7577789202190655613:2136] self: [2:7577789202190655611:0] peer: [6:7577789204347172376:0] socket: 450 qp: -1
2025-11-28T14:52:17.607792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789202190655637:2113] [node 6] ICS10 traffic start
2025-11-28T14:52:17.607843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:17.607889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789204347172371:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:17.608404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789200052204981:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7122) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789204347172371:2122] held: no
2025-11-28T14:52:17.608571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789200052204981:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:17.608649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:17.608700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789200052204981:2049] [node 2] ICP22 created new session: [6:7577789204347172378:2049]
2025-11-28T14:52:17.608786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789204347172378:2049] [node 2] ICS09 handshake done sender: [6:7577789204347172371:2122] self: [6:7577789204347172376:0] peer: [2:7577789202190655611:0] socket: 445 qp: -1
2025-11-28T14:52:17.608821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789204347172378:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:17.608871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:17.629876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:17.630104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789202190655622:2048] [node 1] ICS07 socket disconnect 432 reason# EndOfStream
2025-11-28T14:52:17.630143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789202190655622:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.630231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:17.630284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789202190655622:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:17.631374Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:17.631587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789201378097164:2048] [node 1] ICS07 socket disconnect 439 reason# EndOfStream
2025-11-28T14:52:17.631625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789201378097164:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.631705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:17.631756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789201378097164:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:17.633143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789197895688219:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:17.633195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789202190655622:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:17.634881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789197083129766:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:17.634967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:17.634943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789201378097164:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:17.635546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789197895688219:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:17.635523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789203356197275:2048] [node 1] ICS01 socket: 459 reason# EndOfStream
2025-11-28T14:52:17.635674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789199061229858:2048] [node 1] ICP30 unregister session Session# [4:7577789203356197275:2048] VirtualId# [4:7577789203356197273:0]
2025-11-28T14:52:17.635712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789203356197275:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.635768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789202190655622:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:17.635815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:17.635820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789197895688219:2048] [node 1] ICP30 unregister session Session# [2:7577789202190655622:2048] VirtualId# [2:7577789202190655620:0]
2025-11-28T14:52:17.635864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:17.635868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:17.635918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789197895688219:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.635934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.636257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789197083129766:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:17.636358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789201378097164:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:17.636419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789197083129766:2048] [node 1] ICP30 unregister session Session# [3:7577789201378097164:2048] VirtualId# [3:7577789201378097161:0]
2025-11-28T14:52:17.636465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:17.636492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789197083129766:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.636506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.636742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:17.636965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789201056339557:2048] [node 1] ICS07 socket disconnect 463 reason# EndOfStream
2025-11-28T14:52:17.637000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789201056339557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.637075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:17.637123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789201056339557:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:17.637884Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:17.637968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789196761372166:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:17.638067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789201056339557:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:17.638112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789196761372166:2048] [node 1] ICP30 unregister session Session# [5:7577789201056339557:2048] VirtualId# [5:7577789201056339554:0]
2025-11-28T14:52:17.638167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:17.638208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789196761372166:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.638228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.638239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789204347172357:2048] [node 1] ICS07 socket disconnect 436 reason# EndOfStream
2025-11-28T14:52:17.638281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789204347172357:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.638352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:17.638406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789204347172357:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:17.639157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789200052204980:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:17.639194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789204347172357:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:17.639268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789200052204980:2048] [node 1] ICP30 unregister session Session# [6:7577789204347172357:2048] VirtualId# [6:7577789204347172355:0]
2025-11-28T14:52:17.639298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:17.639339Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789200052204980:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.639347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19695) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.646231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:17.646244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789197083129766:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:17.646449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789201378097174:2049] [node 2] ICS01 socket: 448 reason# EndOfStream
2025-11-28T14:52:17.646508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789197083129767:2049] [node 2] ICP30 unregister session Session# [3:7577789201378097174:2049] VirtualId# [3:7577789201378097172:0]
2025-11-28T14:52:17.646548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789201378097174:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.646648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:17.646707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:17.648282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789196761372166:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:17.648357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:17.648824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789203356197271:2049] [node 2] ICS01 socket: 454 reason# EndOfStream
2025-11-28T14:52:17.648866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789199061229859:2049] [node 2] ICP30 unregister session Session# [4:7577789203356197271:2049] VirtualId# [4:7577789203356197269:0]
2025-11-28T14:52:17.648939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789203356197271:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.649009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:17.649051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:17.649302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789200052204980:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:17.649468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:17.649594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789201056339577:2049] [node 2] ICS07 socket disconnect 479 reason# EndOfStream
2025-11-28T14:52:17.649619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789201056339577:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.649683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:17.649724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789201056339577:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:17.650451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789196761372167:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7122) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:17.650549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789201056339577:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:17.650584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:17.650576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789196761372167:2049] [node 2] ICP30 unregister session Session# [5:7577789201056339577:2049] VirtualId# [5:7577789201056339575:0]
2025-11-28T14:52:17.650622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:17.650645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789196761372167:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7122) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.650673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7122) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.650781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789204347172378:2049] [node 2] ICS01 socket: 445 reason# EndOfStream
2025-11-28T14:52:17.650835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789200052204981:2049] [node 2] ICP30 unregister session Session# [6:7577789204347172378:2049] VirtualId# [6:7577789204347172376:0]
2025-11-28T14:52:17.650878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789204347172378:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.650957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:17.651005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:17.660520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:17.660567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789196761372167:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:17.660760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789203356197259:2050] [node 3] ICS07 socket disconnect 444 reason# EndOfStream
2025-11-28T14:52:17.660798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789203356197259:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.660885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:17.660944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789203356197259:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:17.661617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:17.661746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789201056339573:2050] [node 3] ICS07 socket disconnect 468 reason# EndOfStream
2025-11-28T14:52:17.661764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789201056339573:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.661814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:17.661844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789201056339573:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:17.662027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789199061229860:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:17.662222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789203356197259:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:17.662286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789199061229860:2050] [node 3] ICP30 unregister session Session# [4:7577789203356197259:2050] VirtualId# [4:7577789203356197257:0]
2025-11-28T14:52:17.662367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:17.662410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789199061229860:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.662453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.662787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789196761372168:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:17.662903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789201056339573:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:17.662933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789196761372168:2050] [node 3] ICP30 unregister session Session# [5:7577789201056339573:2050] VirtualId# [5:7577789201056339571:0]
2025-11-28T14:52:17.662995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:17.663023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789196761372168:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.662964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:17.663053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.663170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789204347172374:2050] [node 3] ICS01 socket: 472 reason# EndOfStream
2025-11-28T14:52:17.663215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789200052204982:2050] [node 3] ICP30 unregister session Session# [6:7577789204347172374:2050] VirtualId# [6:7577789204347172372:0]
2025-11-28T14:52:17.663263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789204347172374:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.663346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:17.663392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:17.672846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789196761372168:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:17.727384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:17.727584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789201056339564:2108] [node 4] ICS07 socket disconnect 476 reason# EndOfStream
2025-11-28T14:52:17.727614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789201056339564:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.727725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:17.727775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789201056339564:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:17.728653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789201056339538:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8199) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:17.728770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789201056339564:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:17.728816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789201056339538:2108] [node 4] ICP30 unregister session Session# [5:7577789201056339564:2108] VirtualId# [5:7577789201056339562:0]
2025-11-28T14:52:17.728732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:17.728861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:17.728927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789201056339538:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8199) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.728960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789204347172364:2107] [node 4] ICS01 socket: 473 reason# EndOfStream
2025-11-28T14:52:17.728968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8199) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.729108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789204347172340:2107] [node 4] ICP30 unregister session Session# [6:7577789204347172364:2107] VirtualId# [6:7577789204347172362:0]
2025-11-28T14:52:17.729145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789204347172364:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.729235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:17.729302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:17.758973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:17.759234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789204347172369:2106] [node 5] ICS07 socket disconnect 437 reason# EndOfStream
2025-11-28T14:52:17.759282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789204347172369:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:17.759387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:17.759443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789204347172369:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:17.760139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789204347172337:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11649) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:17.760184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789204347172369:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:17.760253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789204347172337:2106] [node 5] ICP30 unregister session Session# [6:7577789204347172369:2106] VirtualId# [6:7577789204347172367:0]
2025-11-28T14:52:17.760304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:17.760330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789204347172337:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11649) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:17.760343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11649) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:17.824032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789204122371156:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:17.824092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789204122371155:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:17.860903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789204684247023:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:17.861194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789204684247022:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:17.885809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789202546928844:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:17.885882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789202546928843:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:17.913549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789201422772988:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:17.913584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789201422772990:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:17.913919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789201422772989:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:17.943474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789202758056251:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:17.943721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789202758056252:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:17.943761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789202758056253:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:17.975352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789204060990023:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:17.975480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789204060990025:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:17.975504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789204060990024:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:18.085739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789208417338514:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:18.093031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789208417338515:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:18.128549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789208355957379:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:18.128741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789206841896229:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:18.128527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789208417338516:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:18.130896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789207053023612:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:18.130939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789208979214383:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:18.132654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789208979214382:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:18.141685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789206841896231:2133] [node 5] ICP01 ready to work
2025-11-28T14:52:18.136533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789208979214384:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:18.136654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789207053023611:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:18.141771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789205717740368:2126] [node 6] ICP01 ready to work
2025-11-28T14:52:18.141924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789208355957381:2104] [node 4] ICP01 ready to work
2025-11-28T14:52:18.146316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789206841896230:2122] [node 6] ICP01 ready to work
2025-11-28T14:52:18.145671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789205717740369:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:18.152209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789201422772989:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(74526102ns) {Configure:104(74518482ns) {ProcessConfigured:135(1044446ns) {StartInitialHandshake:157(1042225ns) {DropHandshakes:470(2734ns) {DropIncomingHandshake:442(714ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(818ns) PrepareNewSessionHandshake:477(1188ns) {SwitchToState:179(556ns)}}}}}
2025-11-28T14:52:18.152614Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789204122371156:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(96489104ns) {Configure:104(95729843ns) {ProcessConfigured:135(407620ns) {StartInitialHandshake:157(405427ns) {DropHandshakes:470(1197ns) {DropIncomingHandshake:442(300ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(3082ns) PrepareNewSessionHandshake:477(1849ns) {SwitchToState:179(685ns)}}}}}
2025-11-28T14:52:18.153258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789204060990024:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(79904406ns) {Configure:104(75019612ns) {ProcessConfigured:135(477504ns) {StartInitialHandshake:157(475495ns) {DropHandshakes:470(1172ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(522ns) PrepareNewSessionHandshake:477(830ns) {SwitchToState:179(347ns)}}}}}
2025-11-28T14:52:18.152988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789201422772990:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(38322808ns) {Configure:104(38310165ns) {ProcessConfigured:135(668641ns) {StartInitialHandshake:157(665763ns) {DropHandshakes:470(148227ns) {DropIncomingHandshake:442(378ns) DropOutgoingHandshake:457(474ns)} GenerateSessionVirtualId:988(1035ns) PrepareNewSessionHandshake:477(2211ns) {SwitchToState:179(868ns)}}}}}
2025-11-28T14:52:18.153562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789204060990023:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(66553964ns) {Configure:104(66546162ns) {ProcessConfigured:135(792285ns) {StartInitialHandshake:157(789984ns) {DropHandshakes:470(197694ns) {DropIncomingHandshake:442(683ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(634ns) PrepareNewSessionHandshake:477(1543ns) {SwitchToState:179(101ns)}}}}}
2025-11-28T14:52:18.154895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789202758056252:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44267880ns) {Configure:104(44260682ns) {ProcessConfigured:135(174197ns) {StartInitialHandshake:157(172489ns) {DropHandshakes:470(1042ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(883ns) {SwitchToState:179(378ns)}}}}}
2025-11-28T14:52:18.155119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789204122371155:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(96631241ns) {Configure:104(96626971ns) {ProcessConfigured:135(1242200ns) {StartInitialHandshake:157(1239741ns) {DropHandshakes:470(1227ns) {DropIncomingHandshake:442(383ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(1389ns) {SwitchToState:179(606ns)}}}}}
2025-11-28T14:52:18.155474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789204060990025:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(67926702ns) {Configure:104(67917601ns) {ProcessConfigured:135(1674739ns) {StartInitialHandshake:157(1672316ns) {DropHandshakes:470(1572ns) {DropIncomingHandshake:442(717ns) DropOutgoingHandshake:457(299ns)} GenerateSessionVirtualId:988(422ns) PrepareNewSessionHandshake:477(1220ns) {SwitchToState:179(474ns)}}}}}
2025-11-28T14:52:18.155885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789204684247022:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(94994142ns) {Configure:104(94986918ns) {ProcessConfigured:135(1805892ns) {StartInitialHandshake:157(1804724ns) {DropHandshakes:470(1014ns) {DropIncomingHandshake:442(290ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(325ns) PrepareNewSessionHandshake:477(5044ns) {SwitchToState:179(334ns)}}}}}
2025-11-28T14:52:18.156539Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789202546928843:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(89833514ns) {Configure:104(89827549ns) {ProcessConfigured:135(1768911ns) {StartInitialHandshake:157(1763390ns) {DropHandshakes:470(1306ns) {DropIncomingHandshake:442(285ns) DropOutgoingHandshake:457(444ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(304852ns) {SwitchToState:179(1291ns)}}}}}
2025-11-28T14:52:18.156207Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789208417338515:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(45676954ns) {Configure:104(45668109ns) {ProcessConfigured:135(1057877ns) {StartInitialHandshake:157(925574ns) {DropHandshakes:470(1409ns) {DropIncomingHandshake:442(362ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(1169ns) PrepareNewSessionHandshake:477(1121ns) {SwitchToState:179(113ns)}}}}}
2025-11-28T14:52:18.156799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789204684247023:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(83329339ns) {Configure:104(83318512ns) {ProcessConfigured:135(682893ns) {StartInitialHandshake:157(680891ns) {DropHandshakes:470(1218ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(331ns)} GenerateSessionVirtualId:988(423ns) PrepareNewSessionHandshake:477(957ns) {SwitchToState:179(461ns)}}}}}
2025-11-28T14:52:18.155976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789202758056251:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(83605606ns) {Configure:104(83590657ns) {ProcessConfigured:135(1187494ns) {StartInitialHandshake:157(1186102ns) {DropHandshakes:470(1729ns) {DropIncomingHandshake:442(413ns) DropOutgoingHandshake:457(140ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(1186ns) {SwitchToState:179(109ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.156222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789208417338514:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(45342537ns) {Configure:104(42300526ns) {ProcessConfigured:135(1181419ns) {StartInitialHandshake:157(620690ns) {DropHandshakes:470(1761ns) {DropIncomingHandshake:442(553ns) DropOutgoingHandshake:457(103ns)} GenerateSessionVirtualId:988(894ns) PrepareNewSessionHandshake:477(1367ns) {SwitchToState:179(723ns)}}}}}
2025-11-28T14:52:18.156220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789207053023612:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21601340ns) {Configure:104(21594653ns) {ProcessConfigured:135(1664080ns) {StartInitialHandshake:157(1662467ns) {DropHandshakes:470(784ns) {DropIncomingHandshake:442(165ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(360ns) PrepareNewSessionHandshake:477(1369ns) {SwitchToState:179(405ns)}}}}}
2025-11-28T14:52:18.159177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789202546928844:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24390918ns) {Configure:104(24383237ns) {ProcessConfigured:135(1861305ns) {StartInitialHandshake:157(1859275ns) {DropHandshakes:470(1194ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(165ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(1492ns) {SwitchToState:179(1309ns)}}}}}
2025-11-28T14:52:18.156485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789202758056253:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(82756257ns) {Configure:104(76254976ns) {ProcessConfigured:135(802911ns) {StartInitialHandshake:157(801363ns) {DropHandshakes:470(1334ns) {DropIncomingHandshake:442(383ns) DropOutgoingHandshake:457(116ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:477(1322ns) {SwitchToState:179(245ns)}}}}}
2025-11-28T14:52:18.159505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789208417338517:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2099
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.161842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789208417338517:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.165403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789208417338529:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2099
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.165480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789208417338530:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2099
2025-11-28T14:52:18.168668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789205717740380:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41818
2025-11-28T14:52:18.168677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789208979214399:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22805
2025-11-28T14:52:18.168727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789206841896243:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19804
2025-11-28T14:52:18.170806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957397:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.170826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789208979214398:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22805
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.175402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789208979214399:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.169281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789206841896242:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19804
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.176213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789207053023626:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-28T14:52:18.176147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789208417338529:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.176106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789208355957397:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.186350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789208979214398:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.186687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789207053023627:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-28T14:52:18.169323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789206841896246:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19804
2025-11-28T14:52:18.186440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789208355957379:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.175734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206841896242:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.205634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789208979214382:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18159473ns) {IncomingHandshake:261(18157807ns) {DropIncomingHandshake:442(3354ns) PrepareNewSessionHandshake:477(6838ns) {SwitchToState:179(5547ns)} GenerateSessionVirtualId:988(1314ns) IssueIncomingHandshakeReply:193(13941ns)}}
2025-11-28T14:52:18.186760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789207053023628:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-28T14:52:18.196713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957398:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.186682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789208417338534:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2099
2025-11-28T14:52:18.178260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206841896243:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.213261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789208979214401:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22805
2025-11-28T14:52:18.186353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789206841896248:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19804
2025-11-28T14:52:18.213286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789207053023617:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.213426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789208979214400:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22805
2025-11-28T14:52:18.226437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789208979214389:2112] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.213339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789207053023617:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:18.238755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957400:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.226009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789207053023617:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:18.239261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789208355957397:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:18.239318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789207053023611:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.239990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789208355957379:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.241631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789208417338530:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.240051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:18.239379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:18.240172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789208355957379:2105] [node 5] ICP22 created new session: [6:7577789208355957401:2105]
2025-11-28T14:52:18.239906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789207053023611:2110] [node 6] ICP22 created new session: [5:7577789207053023630:2110]
2025-11-28T14:52:18.240442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789208355957401:2105] [node 5] ICS09 handshake done sender: [6:7577789208355957397:2115] self: [6:7577789208355957399:0] peer: [5:7577789207053023613:0] socket: 438 qp: -1
2025-11-28T14:52:18.240476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789208355957401:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:18.241813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.240016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789207053023630:2110] [node 6] ICS09 handshake done sender: [5:7577789207053023617:2111] self: [5:7577789207053023613:0] peer: [6:7577789208355957399:0] socket: 431 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.240161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789207053023630:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:18.240233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:18.242644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789205717740381:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41818
2025-11-28T14:52:18.242692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789205717740382:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41818
2025-11-28T14:52:18.244308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789205717740381:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.244581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789205717740382:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.246665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789208417338520:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.273290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206841896246:2144] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.273502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789202546928843:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.273742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789208417338520:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.274308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789208417338520:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.274476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789206841896252:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19804
2025-11-28T14:52:18.274664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789204122371156:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.274845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:18.274966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789206841896246:2144] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:18.274920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789204122371156:2049] [node 3] ICP22 created new session: [1:7577789208417338538:2049]
2025-11-28T14:52:18.275029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789208417338538:2049] [node 3] ICS09 handshake done sender: [1:7577789208417338520:2111] self: [1:7577789208417338518:0] peer: [3:7577789206841896251:0] socket: 443 qp: -1
2025-11-28T14:52:18.275072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789208417338538:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:18.275174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:18.275525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789202546928843:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.275581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:18.275637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789202546928843:2048] [node 1] ICP22 created new session: [3:7577789206841896253:2048]
2025-11-28T14:52:18.275685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789206841896253:2048] [node 1] ICS09 handshake done sender: [3:7577789206841896246:2144] self: [3:7577789206841896251:0] peer: [1:7577789208417338518:0] socket: 448 qp: -1
2025-11-28T14:52:18.275719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789206841896253:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:18.275759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.275983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957404:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.280774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789207053023620:2113] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.310888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789208417338525:2113] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.317959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206841896248:2145] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.333601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789207053023627:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.333811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789202758056251:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.334124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789208417338525:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.334582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789208417338525:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.334789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789208417338515:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.334847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789207053023634:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-28T14:52:18.334845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:18.334882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789208417338515:2108] [node 5] ICP22 created new session: [1:7577789208417338542:2108]
2025-11-28T14:52:18.334945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789208417338542:2108] [node 5] ICS09 handshake done sender: [1:7577789208417338525:2113] self: [1:7577789208417338522:0] peer: [5:7577789207053023633:0] socket: 453 qp: -1
2025-11-28T14:52:18.334982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789208417338542:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:18.335010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:18.335176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789207053023627:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:18.335362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789202758056251:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.335420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.335643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789202758056251:2048] [node 1] ICP22 created new session: [5:7577789207053023635:2048]
2025-11-28T14:52:18.335678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789207053023635:2048] [node 1] ICS09 handshake done sender: [5:7577789207053023627:2117] self: [5:7577789207053023633:0] peer: [1:7577789208417338522:0] socket: 457 qp: -1
2025-11-28T14:52:18.335876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789205717740385:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41818
2025-11-28T14:52:18.335733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789207053023635:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:18.335786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:18.349263Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789206841896241:2141] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.358274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789207053023628:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.358517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789202758056253:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.358794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206841896241:2141] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.359224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789206841896241:2141] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.359414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789206841896231:2133] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.359543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:18.359585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789207053023638:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-28T14:52:18.359580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789206841896231:2133] [node 5] ICP22 created new session: [3:7577789206841896257:2133]
2025-11-28T14:52:18.359649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789206841896257:2133] [node 5] ICS09 handshake done sender: [3:7577789206841896241:2141] self: [3:7577789206841896238:0] peer: [5:7577789207053023637:0] socket: 456 qp: -1
2025-11-28T14:52:18.359677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789206841896257:2133] [node 5] ICS10 traffic start
2025-11-28T14:52:18.359867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789207053023628:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:18.359751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:18.360131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789202758056253:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.360207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:18.360265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789202758056253:2050] [node 3] ICP22 created new session: [5:7577789207053023639:2050]
2025-11-28T14:52:18.360453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789207053023639:2050] [node 3] ICS09 handshake done sender: [5:7577789207053023628:2118] self: [5:7577789207053023637:0] peer: [3:7577789206841896238:0] socket: 458 qp: -1
2025-11-28T14:52:18.360593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789207053023639:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.360806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:18.360863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957405:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.361287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789202758056253:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:18.377098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789206841896236:2138] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.385333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789208417338534:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.385534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789204122371156:2049] [node 3] ICP18 handshake (actor [1:7577789208417338534:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.386314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789208979214404:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22805
2025-11-28T14:52:18.402117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789208979214394:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.421567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789208355957398:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.421770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789204060990024:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.422018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789208979214394:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.422431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789208979214394:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.422815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.422763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789208979214384:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.422834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:18.423050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789208355957398:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:18.422889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789208979214384:2109] [node 6] ICP22 created new session: [2:7577789208979214406:2109]
2025-11-28T14:52:18.422946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789208979214406:2109] [node 6] ICS09 handshake done sender: [2:7577789208979214394:2115] self: [2:7577789208979214392:0] peer: [6:7577789208355957406:0] socket: 435 qp: -1
2025-11-28T14:52:18.422989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789208979214406:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:18.423030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:18.423368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789204060990024:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.423428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:18.424107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789204060990024:2049] [node 2] ICP22 created new session: [6:7577789208355957408:2049]
2025-11-28T14:52:18.424166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789208355957408:2049] [node 2] ICS09 handshake done sender: [6:7577789208355957398:2116] self: [6:7577789208355957406:0] peer: [2:7577789208979214392:0] socket: 450 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.424186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789208355957408:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:18.424216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:18.424432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789206841896261:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19804
2025-11-28T14:52:18.424500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789204060990024:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:18.427382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789208417338521:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.435265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789208979214400:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.435514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789204684247022:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.435703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789208417338521:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.436145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789208417338521:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.436332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789208979214410:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22805
2025-11-28T14:52:18.436307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789204122371155:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.436395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:18.436432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789204122371155:2048] [node 2] ICP22 created new session: [1:7577789208417338546:2048]
2025-11-28T14:52:18.436486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789208417338546:2048] [node 2] ICS09 handshake done sender: [1:7577789208417338521:2112] self: [1:7577789208417338519:0] peer: [2:7577789208979214409:0] socket: 432 qp: -1
2025-11-28T14:52:18.436530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789208417338546:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:18.436564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:18.436645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789208979214400:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:18.436906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789204684247022:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.437103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:18.437181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789204684247022:2048] [node 1] ICP22 created new session: [2:7577789208979214411:2048]
2025-11-28T14:52:18.437237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789208979214411:2048] [node 1] ICS09 handshake done sender: [2:7577789208979214400:2118] self: [2:7577789208979214409:0] peer: [1:7577789208417338519:0] socket: 452 qp: -1
2025-11-28T14:52:18.437264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789208979214411:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:18.437332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.437694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789204684247022:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:18.437745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789205717740386:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41818
2025-11-28T14:52:18.439458Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [5:7577789207053023618:2112] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.465548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789208979214401:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.471374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789208979214393:2114] [node 5] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.482274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789207053023626:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.482536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789202758056252:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.482833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789208979214393:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.483384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789208979214393:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.483590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789208979214383:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.483718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:18.483776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789208979214383:2108] [node 5] ICP22 created new session: [2:7577789208979214416:2108]
2025-11-28T14:52:18.483702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789207053023642:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-28T14:52:18.483831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789208979214416:2108] [node 5] ICS09 handshake done sender: [2:7577789208979214393:2114] self: [2:7577789208979214391:0] peer: [5:7577789207053023641:0] socket: 441 qp: -1
2025-11-28T14:52:18.483875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789208979214416:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:18.483910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:18.484080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789207053023626:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:18.484995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789202758056252:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5465) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789207053023626:2116] held: no
2025-11-28T14:52:18.485376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789202758056252:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.485416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:18.485612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789206841896234:2137] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.485540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789202758056252:2049] [node 2] ICP22 created new session: [5:7577789207053023643:2049]
2025-11-28T14:52:18.485627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789207053023643:2049] [node 2] ICS09 handshake done sender: [5:7577789207053023626:2116] self: [5:7577789207053023641:0] peer: [2:7577789208979214391:0] socket: 455 qp: -1
2025-11-28T14:52:18.485652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789207053023643:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:18.485690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:18.486452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789205717740380:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.486657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789201422772990:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.486899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206841896234:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.494685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789206841896234:2137] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.502361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789206841896229:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.507875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:18.507829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789205717740388:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41818
2025-11-28T14:52:18.507960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789206841896229:2107] [node 4] ICP22 created new session: [3:7577789206841896263:2107]
2025-11-28T14:52:18.508046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789206841896263:2107] [node 4] ICS09 handshake done sender: [3:7577789206841896234:2137] self: [3:7577789206841896232:0] peer: [4:7577789205717740387:0] socket: 439 qp: -1
2025-11-28T14:52:18.508092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789206841896263:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:18.508159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:18.508387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789205717740380:2133] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:18.508764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789201422772990:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23629) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789205717740380:2133] held: no
2025-11-28T14:52:18.508919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789201422772990:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.508972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:18.509023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789201422772990:2050] [node 3] ICP22 created new session: [4:7577789205717740389:2050]
2025-11-28T14:52:18.509097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789205717740389:2050] [node 3] ICS09 handshake done sender: [4:7577789205717740380:2133] self: [4:7577789205717740387:0] peer: [3:7577789206841896232:0] socket: 442 qp: -1
2025-11-28T14:52:18.509148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789205717740389:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:18.509192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.510229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789207053023645:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-28T14:52:18.512526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789207053023645:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.512737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789207053023612:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.512951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789205717740378:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.512986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789205717740378:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:18.514295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789208417338527:2115] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.514242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789205717740378:2132] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:18.515845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789205717740369:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.535217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789207053023612:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23203) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789207053023645:2122] held: no
2025-11-28T14:52:18.531021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:18.532762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789205717740369:2108] [node 5] ICP22 created new session: [4:7577789205717740393:2108]
2025-11-28T14:52:18.535206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789205717740393:2108] [node 5] ICS09 handshake done sender: [4:7577789205717740378:2132] self: [4:7577789205717740376:0] peer: [5:7577789207053023646:0] socket: 423 qp: -1
2025-11-28T14:52:18.535637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789205717740393:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:18.535718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.0
2025-11-28T14:52:18.535926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789207053023647:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-28T14:52:18.535886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789208355957404:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.536227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789204060990023:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.536423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789205717740369:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(20336413ns) {HandleHandshakeStatus:329(20334393ns) {DropIncomingHandshake:442(6799704ns) DropOutgoingHandshake:457(1958ns) SwitchToState:179(748ns) ProcessPendingSessionEvents:495(344965ns) {ForwardSessionEventToSession:612(72147ns) ForwardSessionEventToSession:612(26853ns) ForwardSessionEventToSession:612(2819ns) ForwardSessionEventToSession:612(213480ns) ForwardSessionEventToSession:612(3607ns) ForwardSessionEventToSession:612(1657ns) ForwardSessionEventToSession:612(2176ns) ForwardSessionEventToSession:612(3393ns) ForwardSessionEventToSession:612(2799ns) ForwardSessionEventToSession:612(9111ns)}}}
2025-11-28T14:52:18.536420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789207053023645:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:18.536473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789208417338527:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.536799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789207053023612:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.536740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957411:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.537032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789208417338527:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:18.536897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:18.537130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789207053023612:2108] [node 4] ICP22 created new session: [5:7577789207053023648:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.537219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789207053023648:2108] [node 4] ICS09 handshake done sender: [5:7577789207053023645:2122] self: [5:7577789207053023646:0] peer: [4:7577789205717740376:0] socket: 430 qp: -1
2025-11-28T14:52:18.537274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789208417338516:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.537530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957412:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.537482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789207053023648:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:18.537555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:18.537348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:18.537590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789208417338516:2109] [node 6] ICP22 created new session: [1:7577789208417338549:2109]
2025-11-28T14:52:18.537639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789208417338549:2109] [node 6] ICS09 handshake done sender: [1:7577789208417338527:2115] self: [1:7577789208417338524:0] peer: [6:7577789208355957410:0] socket: 470 qp: -1
2025-11-28T14:52:18.537794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789208417338549:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:18.537855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:18.537839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789208355957404:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:18.539223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789204060990023:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.539274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:18.539424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789208355957411:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.539456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789204060990023:2048] [node 1] ICP22 created new session: [6:7577789208355957413:2048]
2025-11-28T14:52:18.539610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789208355957413:2048] [node 1] ICS09 handshake done sender: [6:7577789208355957404:2118] self: [6:7577789208355957410:0] peer: [1:7577789208417338524:0] socket: 471 qp: -1
2025-11-28T14:52:18.539663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789208355957413:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:18.539776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:18.539790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789208355957381:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.540280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789204060990023:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:18.540331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789205717740377:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.540395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789205717740377:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:18.542125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789205717740377:2131] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.542269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957416:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.542325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789205717740368:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.542403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:18.542447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789205717740368:2126] [node 6] ICP22 created new session: [4:7577789205717740397:2126]
2025-11-28T14:52:18.542530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789208355957411:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:18.542528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789205717740397:2126] [node 6] ICS09 handshake done sender: [4:7577789205717740377:2131] self: [4:7577789205717740374:0] peer: [6:7577789208355957415:0] socket: 465 qp: -1
2025-11-28T14:52:18.542565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789205717740397:2126] [node 6] ICS10 traffic start
2025-11-28T14:52:18.542662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:18.542713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789208355957381:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.542902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:18.542992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789208355957381:2104] [node 4] ICP22 created new session: [6:7577789208355957417:2104]
2025-11-28T14:52:18.543062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789208355957417:2104] [node 4] ICS09 handshake done sender: [6:7577789208355957411:2121] self: [6:7577789208355957415:0] peer: [4:7577789205717740374:0] socket: 486 qp: -1
2025-11-28T14:52:18.543098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789208355957417:2104] [node 4] ICS10 traffic start
2025-11-28T14:52:18.543155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:18.545622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789208417338526:2114] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.571644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789205717740385:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.571936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789201422772988:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.572312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789208417338526:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.572843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789208417338526:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.573152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789208417338514:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.573230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:18.573241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789205717740400:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41818
2025-11-28T14:52:18.573298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789208417338514:2107] [node 4] ICP22 created new session: [1:7577789208417338552:2107]
2025-11-28T14:52:18.573366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789208417338552:2107] [node 4] ICS09 handshake done sender: [1:7577789208417338526:2114] self: [1:7577789208417338523:0] peer: [4:7577789205717740399:0] socket: 474 qp: -1
2025-11-28T14:52:18.573402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789208417338552:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:18.573442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:18.573530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789205717740385:2136] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:18.573745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789201422772988:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789205717740385:2136] held: no
2025-11-28T14:52:18.574412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789201422772988:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.574478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:18.574521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789201422772988:2048] [node 1] ICP22 created new session: [4:7577789205717740401:2048]
2025-11-28T14:52:18.574603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789205717740401:2048] [node 1] ICS09 handshake done sender: [4:7577789205717740385:2136] self: [4:7577789205717740399:0] peer: [1:7577789208417338523:0] socket: 475 qp: -1
2025-11-28T14:52:18.578227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789205717740401:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:18.578314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:18.579577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789206841896240:2140] [node 6] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.588998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789208355957405:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.589230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789204060990025:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.589553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206841896240:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.590108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789206841896240:2140] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.590358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789206841896230:2122] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.590446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:18.590509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789206841896230:2122] [node 6] ICP22 created new session: [3:7577789206841896266:2122]
2025-11-28T14:52:18.590498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789208355957420:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49991
2025-11-28T14:52:18.590601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789206841896266:2122] [node 6] ICS09 handshake done sender: [3:7577789206841896240:2140] self: [3:7577789206841896237:0] peer: [6:7577789208355957419:0] socket: 478 qp: -1
2025-11-28T14:52:18.590652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789206841896266:2122] [node 6] ICS10 traffic start
2025-11-28T14:52:18.590728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789208355957405:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:18.590967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789204060990025:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.590698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:18.591022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:18.591156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789204060990025:2050] [node 3] ICP22 created new session: [6:7577789208355957421:2050]
2025-11-28T14:52:18.591469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789208355957421:2050] [node 3] ICS09 handshake done sender: [6:7577789208355957405:2119] self: [6:7577789208355957419:0] peer: [3:7577789206841896237:0] socket: 454 qp: -1
2025-11-28T14:52:18.591493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789208355957421:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:18.591527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:18.592569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789204060990025:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:18.594212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [3:7577789206841896239:2139] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.595050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789208979214404:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.604430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789208979214387:2111] [node 3] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.620665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206841896261:2147] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.620925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789202546928844:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.621311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789208979214387:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.621831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789208979214387:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.621991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789206841896269:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19804
2025-11-28T14:52:18.622033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789204684247023:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.622265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:18.622304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789204684247023:2049] [node 3] ICP22 created new session: [2:7577789208979214420:2049]
2025-11-28T14:52:18.622389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789208979214420:2049] [node 3] ICS09 handshake done sender: [2:7577789208979214387:2111] self: [2:7577789208979214385:0] peer: [3:7577789206841896268:0] socket: 480 qp: -1
2025-11-28T14:52:18.622421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789208979214420:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:18.622425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789206841896261:2147] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:18.622471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:18.622780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789202546928844:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5465) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789206841896261:2147] held: no
2025-11-28T14:52:18.622956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789202546928844:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.623026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:18.623081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789202546928844:2049] [node 2] ICP22 created new session: [3:7577789206841896270:2049]
2025-11-28T14:52:18.623203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789206841896270:2049] [node 2] ICS09 handshake done sender: [3:7577789206841896261:2147] self: [3:7577789206841896268:0] peer: [2:7577789208979214385:0] socket: 437 qp: -1
2025-11-28T14:52:18.623244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789206841896270:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:18.623304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:18.623971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789208979214390:2113] [node 4] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:52:18.624802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789205717740386:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.624979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789201422772989:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:18.625268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789208979214390:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:18.625698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789208979214390:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:18.625851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789205717740404:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41818
2025-11-28T14:52:18.625887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789208979214382:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:18.625937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:18.626026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789208979214382:2107] [node 4] ICP22 created new session: [2:7577789208979214423:2107]
2025-11-28T14:52:18.626079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789208979214423:2107] [node 4] ICS09 handshake done sender: [2:7577789208979214390:2113] self: [2:7577789208979214388:0] peer: [4:7577789205717740403:0] socket: 429 qp: -1
2025-11-28T14:52:18.626130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789208979214423:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:18.626138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789205717740386:2137] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:18.626187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:18.626368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789201422772989:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5465) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789205717740386:2137] held: no
2025-11-28T14:52:18.626869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789201422772989:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:18.627164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:18.627205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789201422772989:2049] [node 2] ICP22 created new session: [4:7577789205717740405:2049]
2025-11-28T14:52:18.627233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789205717740405:2049] [node 2] ICS09 handshake done sender: [4:7577789205717740386:2137] self: [4:7577789205717740403:0] peer: [2:7577789208979214388:0] socket: 434 qp: -1
2025-11-28T14:52:18.627256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789205717740405:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:18.627303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:18.630086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:18.630317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789208979214411:2048] [node 1] ICS07 socket disconnect 452 reason# EndOfStream
2025-11-28T14:52:18.630351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789208979214411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.630470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:18.630564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789208979214411:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:18.631063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:18.631272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789206841896253:2048] [node 1] ICS07 socket disconnect 448 reason# EndOfStream
2025-11-28T14:52:18.631339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789206841896253:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.631426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:18.631481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789206841896253:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:18.632406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789202546928843:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:18.632621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789204684247022:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:18.632551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789206841896253:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:18.633040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789202546928843:2048] [node 1] ICP30 unregister session Session# [3:7577789206841896253:2048] VirtualId# [3:7577789206841896251:0]
2025-11-28T14:52:18.633044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789208979214411:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:18.633376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:18.633410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789202546928843:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.633421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.633417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:18.633741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789205717740401:2048] [node 1] ICS01 socket: 475 reason# EndOfStream
2025-11-28T14:52:18.633810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789201422772988:2048] [node 1] ICP30 unregister session Session# [4:7577789205717740401:2048] VirtualId# [4:7577789205717740399:0]
2025-11-28T14:52:18.633917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789205717740401:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.634001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:18.634050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:18.634149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789204684247022:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:18.634188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789208979214411:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:18.634208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789204684247022:2048] [node 1] ICP30 unregister session Session# [2:7577789208979214411:2048] VirtualId# [2:7577789208979214409:0]
2025-11-28T14:52:18.634253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:18.634275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789204684247022:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.634302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.634386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:18.634627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789207053023635:2048] [node 1] ICS07 socket disconnect 457 reason# EndOfStream
2025-11-28T14:52:18.634660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789207053023635:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.634739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:18.634802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789207053023635:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:18.635291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:18.635461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789208355957413:2048] [node 1] ICS07 socket disconnect 471 reason# EndOfStream
2025-11-28T14:52:18.635431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789202758056251:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:18.635504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789208355957413:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.635573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789207053023635:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:18.635561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:18.635618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789208355957413:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:18.635646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789202758056251:2048] [node 1] ICP30 unregister session Session# [5:7577789207053023635:2048] VirtualId# [5:7577789207053023633:0]
2025-11-28T14:52:18.635690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:18.635717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789202758056251:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.635763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.636192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789204060990023:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:18.636296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789208355957413:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:18.636334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789204060990023:2048] [node 1] ICP30 unregister session Session# [6:7577789208355957413:2048] VirtualId# [6:7577789208355957410:0]
2025-11-28T14:52:18.636411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:18.636449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789204060990023:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.636475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.642495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:18.642734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789206841896270:2049] [node 2] ICS01 socket: 437 reason# EndOfStream
2025-11-28T14:52:18.642781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789202546928844:2049] [node 2] ICP30 unregister session Session# [3:7577789206841896270:2049] VirtualId# [3:7577789206841896268:0]
2025-11-28T14:52:18.642824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789206841896270:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.642912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:18.642978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:18.643586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789202546928843:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:18.644323Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:18.644580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789205717740405:2049] [node 2] ICS01 socket: 434 reason# EndOfStream
2025-11-28T14:52:18.644708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789201422772989:2049] [node 2] ICP30 unregister session Session# [4:7577789205717740405:2049] VirtualId# [4:7577789205717740403:0]
2025-11-28T14:52:18.644746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789205717740405:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.644852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:18.644904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:18.645158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:18.645382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789207053023643:2049] [node 2] ICS01 socket: 455 reason# EndOfStream
2025-11-28T14:52:18.645432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789202758056252:2049] [node 2] ICP30 unregister session Session# [5:7577789207053023643:2049] VirtualId# [5:7577789207053023641:0]
2025-11-28T14:52:18.645467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789207053023643:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.645546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:18.645598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:18.645650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789202758056251:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:18.645716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:18.645874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789208355957408:2049] [node 2] ICS07 socket disconnect 450 reason# EndOfStream
2025-11-28T14:52:18.645904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789208355957408:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.645986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:18.646030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789208355957408:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:18.646562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789204060990024:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5465) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:18.646659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789204060990023:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:18.646663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789208355957408:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:18.646702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789204060990024:2049] [node 2] ICP30 unregister session Session# [6:7577789208355957408:2049] VirtualId# [6:7577789208355957406:0]
2025-11-28T14:52:18.646742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:18.646778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789204060990024:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5465) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.646789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5465) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.653887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:18.654135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789205717740389:2050] [node 3] ICS01 socket: 442 reason# EndOfStream
2025-11-28T14:52:18.654188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789201422772990:2050] [node 3] ICP30 unregister session Session# [4:7577789205717740389:2050] VirtualId# [4:7577789205717740387:0]
2025-11-28T14:52:18.654227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789205717740389:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.654306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:18.654367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:18.654869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:18.655020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789208355957421:2050] [node 3] ICS01 socket: 454 reason# EndOfStream
2025-11-28T14:52:18.655052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789204060990025:2050] [node 3] ICP30 unregister session Session# [6:7577789208355957421:2050] VirtualId# [6:7577789208355957419:0]
2025-11-28T14:52:18.655072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789208355957421:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.655138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:18.655203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:18.655842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:18.656044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789207053023639:2050] [node 3] ICS07 socket disconnect 458 reason# EndOfStream
2025-11-28T14:52:18.656085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789207053023639:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.656171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:18.656224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789207053023639:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:18.656897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789204060990024:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:18.656960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789202758056253:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23629) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:18.657021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789207053023639:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:18.657071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789202758056253:2050] [node 3] ICP30 unregister session Session# [5:7577789207053023639:2050] VirtualId# [5:7577789207053023637:0]
2025-11-28T14:52:18.657121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:18.657158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789202758056253:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.657179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.667169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789202758056253:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:18.700844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:18.701041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789207053023648:2108] [node 4] ICS01 socket: 430 reason# EndOfStream
2025-11-28T14:52:18.701076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789207053023612:2108] [node 4] ICP30 unregister session Session# [5:7577789207053023648:2108] VirtualId# [5:7577789207053023646:0]
2025-11-28T14:52:18.701120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789207053023648:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.701215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:18.701268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:18.701637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:18.701813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789208355957417:2104] [node 4] ICS07 socket disconnect 486 reason# EndOfStream
2025-11-28T14:52:18.701837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789208355957417:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.701893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:18.701944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789208355957417:2104] [node 4] ICS15 start handshake
2025-11-28T14:52:18.702620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789208355957381:2104] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23203) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:18.702677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789208355957417:2104] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:18.702750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789208355957381:2104] [node 4] ICP30 unregister session Session# [6:7577789208355957417:2104] VirtualId# [6:7577789208355957415:0]
2025-11-28T14:52:18.702801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:18.702837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789208355957381:2104] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.702847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:18.712246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789208355957381:2104] [node 4] ICP33 wake up from error state
2025-11-28T14:52:18.723612Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:18.723875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789208355957401:2105] [node 5] ICS01 socket: 438 reason# EndOfStream
2025-11-28T14:52:18.723944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789208355957379:2105] [node 5] ICP30 unregister session Session# [6:7577789208355957401:2105] VirtualId# [6:7577789208355957399:0]
2025-11-28T14:52:18.723985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789208355957401:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:18.724120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:18.724175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:18.806484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789206062143799:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:18.806521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789206062143800:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:18.821416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789206347043847:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:18.821462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789206347043848:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:18.836401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789206494803724:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:18.836450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789206494803725:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:18.852745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789204931979153:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:18.852809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789204931979154:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:18.852850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789204931979155:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:18.868851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789204942906080:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:18.868892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789204942906081:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:18.868918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789204942906082:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:18.883937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789206047681240:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:18.883981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789206047681241:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:18.884013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789206047681242:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:18.992835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789206047681303:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:18.992893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789206494803788:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:18.993698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789206347043919:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:18.993729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789204942906145:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:18.993946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789206494803789:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:18.993986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789206347043920:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:18.993987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789204931979216:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:18.994040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789204942906146:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:18.994051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789206047681305:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:18.994244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789206347043921:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:18.994362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789206494803790:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:18.994394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789204931979217:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:18.995813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789206062143799:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49376286ns) {Configure:104(49365125ns) {ProcessConfigured:135(175142ns) {StartInitialHandshake:157(172648ns) {DropHandshakes:470(3696ns) {DropIncomingHandshake:442(967ns) DropOutgoingHandshake:457(526ns)} GenerateSessionVirtualId:988(877ns) PrepareNewSessionHandshake:477(1306ns) {SwitchToState:179(355ns)}}}}}
2025-11-28T14:52:18.996338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789206062143890:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:18.996760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789206062143891:2133] [node 5] ICP01 ready to work
2025-11-28T14:52:18.997305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789206062143892:2124] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:19.006085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789210789771100:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.006475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789210642011228:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60165
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.006641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789210642011229:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60165
2025-11-28T14:52:19.006761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789210642011230:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60165
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.008525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789209226946524:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64068
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.008994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789209226946525:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64068
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.010055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209237873456:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21069
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.010286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789210357111197:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.010431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789210357111198:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53260
2025-11-28T14:52:19.010470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209237873456:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.010574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789210357111199:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.010707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789210357111200:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53260
2025-11-28T14:52:19.011869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789210357111198:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.012016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789210357111199:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.012173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789210357111202:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53260
2025-11-28T14:52:19.013414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789210789771100:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.013705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789210789771101:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
2025-11-28T14:52:19.013708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789210357111197:2141] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.013907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789210789771102:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
2025-11-28T14:52:19.014039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789210789771103:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
2025-11-28T14:52:19.013952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789210642011228:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.014337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789210642011230:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.014586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789210789771101:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.014814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789210642011229:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.014940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789206494803725:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.015150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789210642011234:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60165
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.015315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789210642011235:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60165
2025-11-28T14:52:19.015536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789206347043927:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.015564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789206347043927:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.016718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789209226946524:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.016799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789206347043927:2121] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.016952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789210789771106:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
2025-11-28T14:52:19.017086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789209226946525:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.017361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789206347043848:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:19.017433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
::ffff:127.1.0.0
2025-11-28T14:52:19.017610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789206347043848:2049] [node 3] ICP22 created new session: [2:7577789210642011240:2049]
2025-11-28T14:52:19.017635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789210789771102:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.017688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789210642011240:2049] [node 3] ICS09 handshake done sender: [2:7577789206347043927:2121] self: [2:7577789206347043926:0] peer: [3:7577789210789771105:0] socket: 445 qp: -1
2025-11-28T14:52:19.017690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789210789771101:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:19.017731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789210642011240:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:19.017776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.017711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789209226946529:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64068
2025-11-28T14:52:19.017938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789210357111200:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.017962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789204931979155:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.018161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789204931979154:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.018198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789206494803725:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.018250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.018316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789206494803725:2049] [node 2] ICP22 created new session: [3:7577789210789771107:2049]
2025-11-28T14:52:19.018359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789210789771107:2049] [node 2] ICS09 handshake done sender: [3:7577789210789771101:2117] self: [3:7577789210789771105:0] peer: [2:7577789206347043926:0] socket: 460 qp: -1
2025-11-28T14:52:19.018401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789210789771107:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:19.018428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.018499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789210357111202:2145] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.018601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789210642011235:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.018798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789209226946529:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.018898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789204931979153:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.019106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789210357111192:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.019139Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789210357111192:2138] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.019264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789210642011234:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.019586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789206347043925:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.019614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789206347043925:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.019879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789210357111192:2138] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.020574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206494803796:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.020612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789206494803796:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.020620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789209226946533:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64068
2025-11-28T14:52:19.020777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789206062143890:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.020844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.020894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789206062143890:2107] [node 4] ICP22 created new session: [1:7577789210357111211:2107]
2025-11-28T14:52:19.020949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789210357111211:2107] [node 4] ICS09 handshake done sender: [1:7577789210357111192:2138] self: [1:7577789210357111191:0] peer: [4:7577789209226946532:0] socket: 461 qp: -1
2025-11-28T14:52:19.021009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789210357111211:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:19.021046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.021372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789210789771103:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.021474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789209226946529:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.021415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789206347043847:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.021736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789206347043925:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:19.021676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209237873458:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21069
2025-11-28T14:52:19.021932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789204931979153:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.022035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.022135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789206494803724:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.022092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789204931979153:2048] [node 1] ICP22 created new session: [4:7577789209226946535:2048]
2025-11-28T14:52:19.022264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789209226946535:2048] [node 1] ICS09 handshake done sender: [4:7577789209226946529:2117] self: [4:7577789209226946532:0] peer: [1:7577789210357111191:0] socket: 464 qp: -1
2025-11-28T14:52:19.022303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789209226946535:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:19.022341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789206347043919:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.022335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:19.022332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789206062143889:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.022389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.022364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789206062143889:2136] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.022514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789206494803796:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:19.022546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789206347043919:2107] [node 4] ICP22 created new session: [2:7577789210642011246:2107]
2025-11-28T14:52:19.022595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789210642011246:2107] [node 4] ICS09 handshake done sender: [2:7577789206347043925:2120] self: [2:7577789206347043924:0] peer: [4:7577789209226946531:0] socket: 444 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:19.022630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789210642011246:2107] [node 4] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:19.022690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.023006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209237873459:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21069
2025-11-28T14:52:19.023278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789206494803788:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.023321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.023357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789206494803788:2107] [node 4] ICP22 created new session: [3:7577789210789771113:2107]
2025-11-28T14:52:19.023394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789210789771113:2107] [node 4] ICS09 handshake done sender: [3:7577789206494803796:2113] self: [3:7577789206494803795:0] peer: [4:7577789209226946530:0] socket: 430 qp: -1
2025-11-28T14:52:19.023420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789210789771113:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:19.023448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.023827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789206062143894:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.023860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789206062143894:2137] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.025327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789209226946534:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64068
2025-11-28T14:52:19.025623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789206062143889:2136] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:19.025915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209237873458:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.026873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209237873460:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21069
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.026062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789206062143894:2137] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:19.027062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789204942906081:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.027080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648615:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
2025-11-28T14:52:19.027416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789210642011249:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60165
2025-11-28T14:52:19.027408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789206062143799:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.028037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.028094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789206062143799:2048] [node 2] ICP22 created new session: [1:7577789210357111216:2048]
2025-11-28T14:52:19.028152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789210357111216:2048] [node 2] ICS09 handshake done sender: [1:7577789206062143889:2136] self: [1:7577789206062143888:0] peer: [2:7577789210642011245:0] socket: 454 qp: -1
2025-11-28T14:52:19.028240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789210357111216:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:19.028286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.028819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209237873459:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.029209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789204942906082:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.029285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648616:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
2025-11-28T14:52:19.029344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789210789771116:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:760
2025-11-28T14:52:19.029409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789206062143800:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.029349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789206347043929:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.029391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789206347043929:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.029488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:19.029522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789206062143800:2049] [node 3] ICP22 created new session: [1:7577789210357111218:2049]
2025-11-28T14:52:19.029914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789210357111218:2049] [node 3] ICS09 handshake done sender: [1:7577789206062143894:2137] self: [1:7577789206062143893:0] peer: [3:7577789210789771112:0] socket: 459 qp: -1
2025-11-28T14:52:19.029962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789210357111218:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:19.029995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.030007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789204931979154:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1515) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789209226946525:2116] held: no
2025-11-28T14:52:19.030143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789204931979155:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63490) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789209226946524:2115] held: no
2025-11-28T14:52:19.030341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789209226946525:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.030808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789209226946537:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64068
2025-11-28T14:52:19.031092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206494803798:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.031119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789206494803798:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.031646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789210789771103:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:19.031674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789210642011234:2127] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:19.031841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789204931979154:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.031843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210342648616:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.031924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.031956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789204931979154:2049] [node 2] ICP22 created new session: [4:7577789209226946540:2049]
2025-11-28T14:52:19.031987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789209226946540:2049] [node 2] ICS09 handshake done sender: [4:7577789209226946525:2116] self: [4:7577789209226946531:0] peer: [2:7577789206347043924:0] socket: 447 qp: -1
2025-11-28T14:52:19.032018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789209226946540:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:19.032133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.032131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789206494803724:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.032127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789206047681242:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.032168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.032204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789206494803724:2048] [node 1] ICP22 created new session: [3:7577789210789771118:2048]
2025-11-28T14:52:19.032240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789210789771118:2048] [node 1] ICS09 handshake done sender: [3:7577789210789771103:2119] self: [3:7577789210789771112:0] peer: [1:7577789206062143893:0] socket: 463 qp: -1
2025-11-28T14:52:19.032314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789210789771118:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:19.032356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.033791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789209226946524:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.033833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209237873463:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21069
2025-11-28T14:52:19.034500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648618:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
2025-11-28T14:52:19.034616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789206347043847:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.034738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.034785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789206347043847:2048] [node 1] ICP22 created new session: [2:7577789210642011251:2048]
2025-11-28T14:52:19.034789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789206494803800:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.034843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210342648615:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.035068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789204931979155:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.034828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789210642011251:2048] [node 1] ICS09 handshake done sender: [2:7577789210642011234:2127] self: [2:7577789210642011245:0] peer: [1:7577789206062143888:0] socket: 466 qp: -1
2025-11-28T14:52:19.035120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789206047681241:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.035118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:19.034811Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789206494803800:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.035047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789210642011251:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:19.035165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789204931979155:2050] [node 3] ICP22 created new session: [4:7577789209226946542:2050]
2025-11-28T14:52:19.035191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:19.035195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789209226946542:2050] [node 3] ICS09 handshake done sender: [4:7577789209226946524:2115] self: [4:7577789209226946530:0] peer: [3:7577789206494803795:0] socket: 446 qp: -1
2025-11-28T14:52:19.035221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789209226946542:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:19.035241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.035728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789206494803798:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.035798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789206347043847:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:19.035916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209237873464:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21069
2025-11-28T14:52:19.036089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209237873460:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.036339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648620:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
2025-11-28T14:52:19.036337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789206494803789:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.036414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.036455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789206494803789:2108] [node 5] ICP22 created new session: [3:7577789210789771121:2108]
2025-11-28T14:52:19.036487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789210789771121:2108] [node 5] ICS09 handshake done sender: [3:7577789206494803798:2114] self: [3:7577789206494803797:0] peer: [5:7577789209237873462:0] socket: 457 qp: -1
2025-11-28T14:52:19.036518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789210789771121:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:19.036541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.036672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789210642011227:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.036710Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789210642011227:2123] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.036862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209237873465:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21069
2025-11-28T14:52:19.036975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209237873459:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:19.038004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789204942906080:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.037740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789206347043929:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:19.038083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789204942906082:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.038128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:19.038210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789204942906082:2050] [node 3] ICP22 created new session: [5:7577789209237873467:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.038263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209237873467:2050] [node 3] ICS09 handshake done sender: [5:7577789209237873459:2119] self: [5:7577789209237873462:0] peer: [3:7577789206494803797:0] socket: 475 qp: -1
2025-11-28T14:52:19.038291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209237873467:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:19.038334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.038347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789206347043920:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.038421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.038351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648621:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
2025-11-28T14:52:19.038475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789206347043920:2108] [node 5] ICP22 created new session: [2:7577789210642011254:2108]
2025-11-28T14:52:19.038474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789206494803800:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:19.038520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789210642011254:2108] [node 5] ICS09 handshake done sender: [2:7577789206347043929:2122] self: [2:7577789206347043928:0] peer: [5:7577789209237873461:0] socket: 471 qp: -1
2025-11-28T14:52:19.038555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789210642011254:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:19.038597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.039093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210342648620:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.039234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210342648616:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:19.039389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789206494803790:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.039420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.039448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789206494803790:2109] [node 6] ICP22 created new session: [3:7577789210789771123:2109]
2025-11-28T14:52:19.039481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789210789771123:2109] [node 6] ICS09 handshake done sender: [3:7577789206494803800:2115] self: [3:7577789206494803799:0] peer: [6:7577789210342648617:0] socket: 455 qp: -1
2025-11-28T14:52:19.039501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789210789771123:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:19.039555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:19.039697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789206047681242:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.039821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:19.039865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789206047681242:2050] [node 3] ICP22 created new session: [6:7577789210342648622:2050]
2025-11-28T14:52:19.039955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210342648622:2050] [node 3] ICS09 handshake done sender: [6:7577789210342648616:2117] self: [6:7577789210342648617:0] peer: [3:7577789206494803799:0] socket: 482 qp: -1
2025-11-28T14:52:19.039985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210342648622:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:19.040037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.040205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789210642011227:2123] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:19.040721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789206347043921:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.040783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.040824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789206347043921:2109] [node 6] ICP22 created new session: [2:7577789210642011256:2109]
2025-11-28T14:52:19.040864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789210642011256:2109] [node 6] ICS09 handshake done sender: [2:7577789210642011227:2123] self: [2:7577789210642011226:0] peer: [6:7577789210342648619:0] socket: 479 qp: -1
2025-11-28T14:52:19.040895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789210642011256:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:19.040933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:19.041041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789206047681240:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.041368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210342648618:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.041618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789204942906082:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:19.041589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789210357111194:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.041629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789210357111194:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.042453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789210357111196:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.042487Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789210357111196:2140] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.044112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648624:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
2025-11-28T14:52:19.044161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789204942906081:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1515) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789209237873458:2118] held: no
2025-11-28T14:52:19.044435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789210357111194:2139] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:19.044498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209237873464:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.044506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789206047681241:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1515) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789210342648615:2116] held: no
2025-11-28T14:52:19.044717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209237873458:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:19.044783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789206047681305:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.044852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789210357111196:2140] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:19.045133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210342648615:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:19.045120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789206062143891:2133] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.045278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789204942906081:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.045181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.045331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.045249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789206062143891:2133] [node 5] ICP22 created new session: [1:7577789210357111223:2133]
2025-11-28T14:52:19.045380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789204942906081:2049] [node 2] ICP22 created new session: [5:7577789209237873472:2049]
2025-11-28T14:52:19.045314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789210357111223:2133] [node 5] ICS09 handshake done sender: [1:7577789210357111194:2139] self: [1:7577789210357111193:0] peer: [5:7577789209237873466:0] socket: 467 qp: -1
2025-11-28T14:52:19.045449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209237873472:2049] [node 2] ICS09 handshake done sender: [5:7577789209237873458:2118] self: [5:7577789209237873461:0] peer: [2:7577789206347043928:0] socket: 472 qp: -1
2025-11-28T14:52:19.045352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789210357111223:2133] [node 5] ICS10 traffic start
2025-11-28T14:52:19.045480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209237873472:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:19.045393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.045542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.045798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789206062143892:2124] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.045782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789209226946523:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.045800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789206047681241:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.045857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.045847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.045819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789209226946523:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.045942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789206062143892:2124] [node 6] ICP22 created new session: [1:7577789210357111225:2124]
2025-11-28T14:52:19.045891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789206047681241:2049] [node 2] ICP22 created new session: [6:7577789210342648629:2049]
2025-11-28T14:52:19.045967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210342648629:2049] [node 2] ICS09 handshake done sender: [6:7577789210342648615:2116] self: [6:7577789210342648619:0] peer: [2:7577789210642011226:0] socket: 480 qp: -1
2025-11-28T14:52:19.046036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789210357111225:2124] [node 6] ICS09 handshake done sender: [1:7577789210357111196:2140] self: [1:7577789210357111195:0] peer: [6:7577789210342648625:0] socket: 486 qp: -1
2025-11-28T14:52:19.045999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210342648629:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:19.046146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.046082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789210357111225:2124] [node 6] ICS10 traffic start
2025-11-28T14:52:19.046113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.046915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789209226946544:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64068
2025-11-28T14:52:19.047134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789204942906145:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.047227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789209226946523:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.047382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209237873469:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21069
2025-11-28T14:52:19.047469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648628:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
2025-11-28T14:52:19.047585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789204931979217:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.047642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.047693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789204931979217:2109] [node 6] ICP22 created new session: [4:7577789209226946547:2109]
2025-11-28T14:52:19.047874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789209226946547:2109] [node 6] ICS09 handshake done sender: [4:7577789209226946523:2114] self: [4:7577789209226946522:0] peer: [6:7577789210342648627:0] socket: 483 qp: -1
2025-11-28T14:52:19.048031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789209226946547:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:19.048062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.049385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789209226946546:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64068
2025-11-28T14:52:19.050086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789209226946521:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.050115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789209226946521:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.050173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789204942906080:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3280) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789209237873460:2120] held: no
2025-11-28T14:52:19.050710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209237873460:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:19.050991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789209226946521:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:19.051515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789204931979216:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.051553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.051584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789204931979216:2108] [node 5] ICP22 created new session: [4:7577789209226946550:2108]
2025-11-28T14:52:19.051618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789209226946550:2108] [node 5] ICS09 handshake done sender: [4:7577789209226946521:2113] self: [4:7577789209226946520:0] peer: [5:7577789209237873474:0] socket: 442 qp: -1
2025-11-28T14:52:19.051644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789209226946550:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:19.051669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.052443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210342648620:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:19.052746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789206047681240:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3280) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789210342648620:2119] held: no
2025-11-28T14:52:19.052907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789206047681240:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.052952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.053005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789206047681240:2048] [node 1] ICP22 created new session: [6:7577789210342648633:2048]
2025-11-28T14:52:19.053063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210342648633:2048] [node 1] ICS09 handshake done sender: [6:7577789210342648620:2119] self: [6:7577789210342648625:0] peer: [1:7577789210357111195:0] socket: 487 qp: -1
2025-11-28T14:52:19.053095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210342648633:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:19.053137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:19.053504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789204942906080:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.053581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.053619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789204942906080:2048] [node 1] ICP22 created new session: [5:7577789209237873475:2048]
2025-11-28T14:52:19.053652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209237873475:2048] [node 1] ICS09 handshake done sender: [5:7577789209237873460:2120] self: [5:7577789209237873466:0] peer: [1:7577789210357111193:0] socket: 478 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:19.053676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209237873475:2048] [node 1] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:19.053707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:19.053836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648631:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.054321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789209226946546:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.054410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648632:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
2025-11-28T14:52:19.054451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789204931979216:2108] [node 5] ICP18 handshake (actor [4:7577789209226946546:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.054530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209237873477:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21069
2025-11-28T14:52:19.054971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210342648618:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:19.055007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789209226946544:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.055260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789204931979217:2109] [node 6] ICP18 handshake (actor [4:7577789209226946544:2121]) is too old
2025-11-28T14:52:19.055406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210342648632:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.055455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789206047681305:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.055523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.055867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789206047681305:2107] [node 4] ICP22 created new session: [6:7577789210342648635:2107]
2025-11-28T14:52:19.055955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210342648635:2107] [node 4] ICS09 handshake done sender: [6:7577789210342648618:2118] self: [6:7577789210342648627:0] peer: [4:7577789209226946522:0] socket: 484 qp: -1
2025-11-28T14:52:19.056014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210342648635:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:19.056045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.056310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209237873464:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:19.056659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789204942906145:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789209237873464:2122] held: no
2025-11-28T14:52:19.056767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789204942906145:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.056829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.056890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789204942906145:2108] [node 4] ICP22 created new session: [5:7577789209237873478:2108]
2025-11-28T14:52:19.056940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209237873478:2108] [node 4] ICS09 handshake done sender: [5:7577789209237873464:2122] self: [5:7577789209237873474:0] peer: [4:7577789209226946520:0] socket: 448 qp: -1
2025-11-28T14:52:19.057038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209237873478:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:19.057068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.057860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789206047681303:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.058139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209237873452:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.058176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789209237873452:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.059082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209237873452:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.059346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210342648638:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24846
2025-11-28T14:52:19.059457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789204942906146:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.059551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.059602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789204942906146:2109] [node 6] ICP22 created new session: [5:7577789209237873481:2109]
2025-11-28T14:52:19.059669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209237873481:2109] [node 6] ICS09 handshake done sender: [5:7577789209237873452:2116] self: [5:7577789209237873451:0] peer: [6:7577789210342648637:0] socket: 494 qp: -1
2025-11-28T14:52:19.059721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209237873481:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:19.059759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:19.059819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210342648632:2124] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:19.060039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789206047681303:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.060080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.060120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789206047681303:2106] [node 5] ICP22 created new session: [6:7577789210342648639:2106]
2025-11-28T14:52:19.060171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210342648639:2106] [node 5] ICS09 handshake done sender: [6:7577789210342648632:2124] self: [6:7577789210342648637:0] peer: [5:7577789209237873451:0] socket: 449 qp: -1
2025-11-28T14:52:19.060207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210342648639:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:19.060236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.082516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:19.082757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789210642011251:2048] [node 1] ICS07 socket disconnect 466 reason# EndOfStream
2025-11-28T14:52:19.082794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789210642011251:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.082893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.082965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789210642011251:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:19.083593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:19.083797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789210789771118:2048] [node 1] ICS01 socket: 463 reason# EndOfStream
2025-11-28T14:52:19.083844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789206494803724:2048] [node 1] ICP30 unregister session Session# [3:7577789210789771118:2048] VirtualId# [3:7577789210789771112:0]
2025-11-28T14:52:19.083879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789210789771118:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.083953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.083995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.085375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789206347043847:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3280) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:19.085418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789210642011251:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:19.085965Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:19.086207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789209226946535:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-11-28T14:52:19.086321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789204931979153:2048] [node 1] ICP30 unregister session Session# [4:7577789209226946535:2048] VirtualId# [4:7577789209226946532:0]
2025-11-28T14:52:19.086358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789209226946535:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.086445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.086506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.086450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789206347043847:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3280) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.086501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789210642011251:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.086539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789206347043847:2048] [node 1] ICP30 unregister session Session# [2:7577789210642011251:2048] VirtualId# [2:7577789210642011245:0]
2025-11-28T14:52:19.086587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.086613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789206347043847:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3280) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.086624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3280) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.086773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:19.087054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210342648633:2048] [node 1] ICS01 socket: 487 reason# EndOfStream
2025-11-28T14:52:19.087138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789206047681240:2048] [node 1] ICP30 unregister session Session# [6:7577789210342648633:2048] VirtualId# [6:7577789210342648625:0]
2025-11-28T14:52:19.087176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210342648633:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.087234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.087277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.087523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:19.087757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789209237873475:2048] [node 1] ICS01 socket: 478 reason# EndOfStream
2025-11-28T14:52:19.087813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789204942906080:2048] [node 1] ICP30 unregister session Session# [5:7577789209237873475:2048] VirtualId# [5:7577789209237873466:0]
2025-11-28T14:52:19.087852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789209237873475:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.087924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.087974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.111291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:19.111437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789210789771107:2049] [node 2] ICS01 socket: 460 reason# EndOfStream
2025-11-28T14:52:19.111476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789206494803725:2049] [node 2] ICP30 unregister session Session# [3:7577789210789771107:2049] VirtualId# [3:7577789210789771105:0]
2025-11-28T14:52:19.111498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789210789771107:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.111568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:19.111611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:19.113414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:19.113583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789209226946540:2049] [node 2] ICS01 socket: 447 reason# EndOfStream
2025-11-28T14:52:19.113631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789204931979154:2049] [node 2] ICP30 unregister session Session# [4:7577789209226946540:2049] VirtualId# [4:7577789209226946531:0]
2025-11-28T14:52:19.113659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789209226946540:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.113718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:19.113749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:19.114107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:19.114342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789209237873472:2049] [node 2] ICS01 socket: 472 reason# EndOfStream
2025-11-28T14:52:19.114385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789204942906081:2049] [node 2] ICP30 unregister session Session# [5:7577789209237873472:2049] VirtualId# [5:7577789209237873461:0]
2025-11-28T14:52:19.114414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789209237873472:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.114482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:19.114543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:19.114674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:19.114898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210342648629:2049] [node 2] ICS01 socket: 480 reason# EndOfStream
2025-11-28T14:52:19.114938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789206047681241:2049] [node 2] ICP30 unregister session Session# [6:7577789210342648629:2049] VirtualId# [6:7577789210342648619:0]
2025-11-28T14:52:19.114963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210342648629:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.115032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:19.115086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:19.134117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:19.134281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789209226946542:2050] [node 3] ICS01 socket: 446 reason# EndOfStream
2025-11-28T14:52:19.134323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789204931979155:2050] [node 3] ICP30 unregister session Session# [4:7577789209226946542:2050] VirtualId# [4:7577789209226946530:0]
2025-11-28T14:52:19.134351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789209226946542:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.134422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:19.134473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:19.134831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:19.135066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789209237873467:2050] [node 3] ICS01 socket: 475 reason# EndOfStream
2025-11-28T14:52:19.135113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789204942906082:2050] [node 3] ICP30 unregister session Session# [5:7577789209237873467:2050] VirtualId# [5:7577789209237873462:0]
2025-11-28T14:52:19.135146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789209237873467:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.135211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:19.135256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:19.135438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:19.135649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789210342648622:2050] [node 3] ICS07 socket disconnect 482 reason# EndOfStream
2025-11-28T14:52:19.135679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210342648622:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.135755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:19.135810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789210342648622:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:19.136499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789206047681242:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63490) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.136549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210342648622:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.136580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789206047681242:2050] [node 3] ICP30 unregister session Session# [6:7577789210342648622:2050] VirtualId# [6:7577789210342648617:0]
2025-11-28T14:52:19.136635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:19.136671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789206047681242:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.136686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.146437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789206047681242:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:19.160379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:19.160622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789209237873478:2108] [node 4] ICS01 socket: 448 reason# EndOfStream
2025-11-28T14:52:19.160671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789204942906145:2108] [node 4] ICP30 unregister session Session# [5:7577789209237873478:2108] VirtualId# [5:7577789209237873474:0]
2025-11-28T14:52:19.160710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789209237873478:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.160789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:19.160843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:19.161483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:19.161725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789210342648635:2107] [node 4] ICS07 socket disconnect 484 reason# EndOfStream
2025-11-28T14:52:19.161757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210342648635:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.161823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:19.161876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789210342648635:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:19.162794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789206047681305:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17658) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.162850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210342648635:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.162885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789206047681305:2107] [node 4] ICP30 unregister session Session# [6:7577789210342648635:2107] VirtualId# [6:7577789210342648627:0]
2025-11-28T14:52:19.162925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:19.162962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789206047681305:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.162980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17658) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.173051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789206047681305:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:19.188533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:19.188739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210342648639:2106] [node 5] ICS01 socket: 449 reason# EndOfStream
2025-11-28T14:52:19.188776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789206047681303:2106] [node 5] ICP30 unregister session Session# [6:7577789210342648639:2106] VirtualId# [6:7577789210342648637:0]
2025-11-28T14:52:19.188798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210342648639:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.188856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:19.188890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:19.240695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789213054239949:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:19.240768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789213054239950:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:19.258177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789212097230319:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:19.258283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789212097230320:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:19.274939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789211164127240:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:19.274984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789211164127241:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:19.295641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789211946823996:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:19.295678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789211946823998:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:19.295700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789211946823997:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:19.315807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789209981061744:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:19.315863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789209981061746:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:19.315937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789209981061745:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:19.335332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789210670538401:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:19.335334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789210670538402:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:19.335367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789210670538403:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:19.432122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789210670538462:2104] [node 5] ICP01 ready to work
2025-11-28T14:52:19.433567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789211946824073:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:19.433618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789211164127304:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:19.433746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789209981061807:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:19.433837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789211164127303:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:19.434448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789210670538464:2105] [node 4] ICP01 ready to work
2025-11-28T14:52:19.435348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789209981061808:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:19.435388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789211946824074:2115] [node 6] ICP01 ready to work
2025-11-28T14:52:19.435434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789211164127305:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:19.436905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789211946823997:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49570581ns) {Configure:104(49566581ns) {ProcessConfigured:135(146455ns) {StartInitialHandshake:157(144173ns) {DropHandshakes:470(2447ns) {DropIncomingHandshake:442(514ns) DropOutgoingHandshake:457(854ns)} GenerateSessionVirtualId:988(414ns) PrepareNewSessionHandshake:477(1090ns) {SwitchToState:179(273ns)}}}}}
2025-11-28T14:52:19.437348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789209981061744:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43787169ns) {Configure:104(43777750ns) {ProcessConfigured:135(242012ns) {StartInitialHandshake:157(239335ns) {DropHandshakes:470(1557ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(518ns) PrepareNewSessionHandshake:477(1075ns) {SwitchToState:179(488ns)}}}}}
2025-11-28T14:52:19.437634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789213054239950:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(54630702ns) {Configure:104(54625078ns) {ProcessConfigured:135(247764ns) {StartInitialHandshake:157(245722ns) {DropHandshakes:470(1261ns) {DropIncomingHandshake:442(448ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:477(844ns) {SwitchToState:179(339ns)}}}}}
2025-11-28T14:52:19.437788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789213054239949:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44796710ns) {Configure:104(44787813ns) {ProcessConfigured:135(83582ns) {StartInitialHandshake:157(81625ns) {DropHandshakes:470(5274ns) {DropIncomingHandshake:442(381ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(739ns) {SwitchToState:179(151ns)}}}}}
2025-11-28T14:52:19.437836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789210670538401:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47509750ns) {Configure:104(47505081ns) {ProcessConfigured:135(75988ns) {StartInitialHandshake:157(74665ns) {DropHandshakes:470(1191ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(320ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:477(6402ns) {SwitchToState:179(550ns)}}}}}
2025-11-28T14:52:19.438081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789213054240026:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:19.438411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789213054240027:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:19.438636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789213054240028:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:19.438656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789212097230320:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45620518ns) {Configure:104(45605141ns) {ProcessConfigured:135(40942ns) {StartInitialHandshake:157(39223ns) {DropHandshakes:470(818ns) {DropIncomingHandshake:442(233ns) DropOutgoingHandshake:457(161ns)} GenerateSessionVirtualId:988(291ns) PrepareNewSessionHandshake:477(705ns) {SwitchToState:179(242ns)}}}}}
2025-11-28T14:52:19.439215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789212097230319:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53596220ns) {Configure:104(53586859ns) {ProcessConfigured:135(331394ns) {StartInitialHandshake:157(329369ns) {DropHandshakes:470(1238ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(321ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:477(1856ns) {SwitchToState:179(1079ns)}}}}}
2025-11-28T14:52:19.439395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789212097230412:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:19.439769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789212097230413:2113] [node 5] ICP01 ready to work
2025-11-28T14:52:19.439938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789212097230414:2134] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.448420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209981061822:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14924
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.455204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789211164127314:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-28T14:52:19.455219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209981061822:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.456498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789211164127315:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.456989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789211164127316:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-28T14:52:19.457014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789213054240035:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.457686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789211164127314:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.457380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789213054240036:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.457985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789211164127315:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.458112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789213054240037:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19245
2025-11-28T14:52:19.458174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789211946824089:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25722
2025-11-28T14:52:19.458251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789211946824088:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25722
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.458651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789211164127322:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.459220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789212097230422:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5383
2025-11-28T14:52:19.459202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240036:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.459344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240035:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.459299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789212097230423:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5383
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.459486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240037:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.459509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789212097230421:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5383
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.459535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789212097230424:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5383
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.459796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789212097230422:2141] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.459903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789213054240041:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19245
2025-11-28T14:52:19.459884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789212097230423:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.460449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789212097230427:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5383
2025-11-28T14:52:19.465024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789212097230421:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.465519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209981061824:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14924
2025-11-28T14:52:19.466118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789211946824089:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.466356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789211946823996:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.466076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789213054240042:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19245
2025-11-28T14:52:19.467041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789211164127316:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.466320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240041:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.477173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789211164127240:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.479244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789212097230424:2143] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.477184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240030:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.479553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789212097230427:2145] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.477260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789213054240030:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.481099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240023:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.481134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789213054240023:2120] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.484317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789213054240030:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:19.491564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789213054240023:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.500567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789213054240026:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.506856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.514158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789211164127326:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-28T14:52:19.506928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789211946824091:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25722
2025-11-28T14:52:19.506929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789213054240026:2107] [node 4] ICP22 created new session: [1:7577789213054240049:2107]
2025-11-28T14:52:19.511822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789213054240049:2107] [node 4] ICS09 handshake done sender: [1:7577789213054240030:2122] self: [1:7577789213054240029:0] peer: [4:7577789211946824090:0] socket: 461 qp: -1
2025-11-28T14:52:19.514462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789213054239950:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.514535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:19.514592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789213054239950:2049] [node 3] ICP22 created new session: [1:7577789213054240050:2049]
2025-11-28T14:52:19.514634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789213054240050:2049] [node 3] ICS09 handshake done sender: [1:7577789213054240023:2120] self: [1:7577789213054240022:0] peer: [3:7577789211164127325:0] socket: 440 qp: -1
2025-11-28T14:52:19.515333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789211946824089:2130] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:19.514662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789213054240050:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:19.514724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.516517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789211164127316:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:19.514722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789213054240049:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:19.517750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789211946823996:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.517826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.517891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789211946823996:2048] [node 1] ICP22 created new session: [4:7577789211946824092:2048]
2025-11-28T14:52:19.517950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789211946824092:2048] [node 1] ICS09 handshake done sender: [4:7577789211946824089:2130] self: [4:7577789211946824090:0] peer: [1:7577789213054240029:0] socket: 464 qp: -1
2025-11-28T14:52:19.517995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789211946824092:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:19.518029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:19.514777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.522183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789211164127240:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.522278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.522342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789211164127240:2048] [node 1] ICP22 created new session: [3:7577789211164127327:2048]
2025-11-28T14:52:19.522414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789211164127327:2048] [node 1] ICS09 handshake done sender: [3:7577789211164127316:2116] self: [3:7577789211164127325:0] peer: [1:7577789213054240022:0] socket: 452 qp: -1
2025-11-28T14:52:19.522637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789211164127327:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:19.522696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:19.522272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789213054240026:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17229333ns) {HandleHandshakeStatus:329(17214691ns) {DropIncomingHandshake:442(23157ns) DropOutgoingHandshake:457(881ns) SwitchToState:179(787ns) ProcessPendingSessionEvents:495(1033732ns) {ForwardSessionEventToSession:612(503286ns) ForwardSessionEventToSession:612(5935ns) ForwardSessionEventToSession:612(16590ns) ForwardSessionEventToSession:612(16958ns) ForwardSessionEventToSession:612(45809ns) ForwardSessionEventToSession:612(262471ns) ForwardSessionEventToSession:612(7373ns) ForwardSessionEventToSession:612(3170ns) ForwardSessionEventToSession:612(15408ns) ForwardSessionEventToSession:612(2837ns)}}}
2025-11-28T14:52:19.528816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789212097230319:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.530063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209981061825:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14924
2025-11-28T14:52:19.530358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209981061824:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.530715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789209981061745:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.530957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538479:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.531055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789211946824088:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.531004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789212097230416:2138] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.533107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789211946823998:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.531038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789212097230416:2138] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.533082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789211164127322:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.533121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209981061826:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14924
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.546927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209981061828:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14924
2025-11-28T14:52:19.534975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789212097230416:2138] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:19.547728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209981061826:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.546255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240025:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.546309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789213054240025:2121] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.547966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789209981061744:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.547905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789211164127311:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.548606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210670538479:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.547944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789211164127311:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.548644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789211164127241:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.549634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789210670538401:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.549094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789212097230413:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.549948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240032:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.549993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789213054240032:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.549167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.549731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209981061824:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:19.549346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789212097230413:2113] [node 5] ICP22 created new session: [2:7577789212097230435:2113]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.549620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789212097230435:2113] [node 5] ICS09 handshake done sender: [2:7577789212097230416:2138] self: [2:7577789212097230415:0] peer: [5:7577789209981061827:0] socket: 471 qp: -1
2025-11-28T14:52:19.550604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240039:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.549661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789212097230435:2113] [node 5] ICS10 traffic start
2025-11-28T14:52:19.550891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789211946824096:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25722
2025-11-28T14:52:19.549755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.550633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789213054240039:2127] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.551452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789209981061745:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.551625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.551686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789209981061745:2049] [node 2] ICP22 created new session: [5:7577789209981061831:2049]
2025-11-28T14:52:19.551816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209981061831:2049] [node 2] ICS09 handshake done sender: [5:7577789209981061824:2116] self: [5:7577789209981061827:0] peer: [2:7577789212097230415:0] socket: 472 qp: -1
2025-11-28T14:52:19.551891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209981061831:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:19.551955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.553033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789211946824088:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:19.553351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789211164127311:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:19.553218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789211946823998:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.552233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789213054240032:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:19.553264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:19.552488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789213054240025:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:19.553462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789211946823998:2050] [node 3] ICP22 created new session: [4:7577789211946824097:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.557154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209981061830:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14924
2025-11-28T14:52:19.554121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789211946824097:2050] [node 3] ICS09 handshake done sender: [4:7577789211946824088:2129] self: [4:7577789211946824095:0] peer: [3:7577789211164127310:0] socket: 462 qp: -1
2025-11-28T14:52:19.558869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789213054240027:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.557722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209981061833:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14924
2025-11-28T14:52:19.558388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789211164127304:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.554175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789211946824097:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:19.559643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.562405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789212097230409:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.558938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.560488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209981061826:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:19.560268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789211164127304:2107] [node 4] ICP22 created new session: [3:7577789211164127332:2107]
2025-11-28T14:52:19.554208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.559008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789213054239949:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.559039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.562061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789211164127332:2107] [node 4] ICS09 handshake done sender: [3:7577789211164127311:2112] self: [3:7577789211164127310:0] peer: [4:7577789211946824095:0] socket: 459 qp: -1
2025-11-28T14:52:19.559487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789213054239949:2048] [node 2] ICP22 created new session: [1:7577789213054240059:2048]
2025-11-28T14:52:19.563350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789211164127332:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:19.559764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789213054240027:2108] [node 5] ICP22 created new session: [1:7577789213054240058:2108]
2025-11-28T14:52:19.563404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.560302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789213054240059:2048] [node 2] ICS09 handshake done sender: [1:7577789213054240025:2121] self: [1:7577789213054240024:0] peer: [2:7577789212097230432:0] socket: 443 qp: -1
2025-11-28T14:52:19.563621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789213054240059:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:19.563685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.562444Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789212097230409:2136] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.564157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209981061830:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.564677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789213054240058:2108] [node 5] ICS09 handshake done sender: [1:7577789213054240032:2123] self: [1:7577789213054240031:0] peer: [5:7577789209981061829:0] socket: 460 qp: -1
2025-11-28T14:52:19.564719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789213054240058:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:19.564867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.564210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789209981061744:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.564285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.564323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789209981061744:2048] [node 1] ICP22 created new session: [5:7577789209981061835:2048]
2025-11-28T14:52:19.569699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789211946824099:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25722
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.571388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538481:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.571509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538482:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.564411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209981061835:2048] [node 1] ICS09 handshake done sender: [5:7577789209981061826:2118] self: [5:7577789209981061829:0] peer: [1:7577789213054240031:0] socket: 478 qp: -1
2025-11-28T14:52:19.565030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209981061835:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:19.565068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:19.572579Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789213054240039:2127] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:19.572379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210670538482:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.572863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789210670538403:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.573063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538483:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.574194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789213054240028:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.575057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.574061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789211164127321:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.574098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789209981061746:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.575215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789212097230437:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5383
2025-11-28T14:52:19.580175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789211946824099:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.575278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789213054240028:2109] [node 6] ICP22 created new session: [1:7577789213054240062:2109]
2025-11-28T14:52:19.574106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789211164127321:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.580658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209981061825:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.575430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789213054240062:2109] [node 6] ICS09 handshake done sender: [1:7577789213054240039:2127] self: [1:7577789213054240038:0] peer: [6:7577789210670538480:0] socket: 479 qp: -1
2025-11-28T14:52:19.575457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210670538479:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:19.581602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789212097230409:2136] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:19.581157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789210670538401:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.580444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789211164127313:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.575617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789213054240062:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.581309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.580546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:19.580492Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789211164127313:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.581836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789210670538401:2048] [node 1] ICP22 created new session: [6:7577789210670538487:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.583505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789209981061807:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.582710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789211164127335:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-28T14:52:19.583688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789211164127321:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:19.582725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210670538487:2048] [node 1] ICS09 handshake done sender: [6:7577789210670538479:2115] self: [6:7577789210670538480:0] peer: [1:7577789213054240038:0] socket: 480 qp: -1
2025-11-28T14:52:19.583813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789211946824101:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25722
2025-11-28T14:52:19.583799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789212097230320:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.583464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210670538487:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:19.583880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:19.583581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:19.584185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789211946824082:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.584268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789212097230320:2049] [node 3] ICP22 created new session: [2:7577789212097230440:2049]
2025-11-28T14:52:19.584215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789211946824082:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.584385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789212097230440:2049] [node 3] ICS09 handshake done sender: [2:7577789212097230409:2136] self: [2:7577789212097230408:0] peer: [3:7577789211164127330:0] socket: 463 qp: -1
2025-11-28T14:52:19.584700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789211164127305:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.584747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.584580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789212097230440:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:19.584828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789211164127322:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:19.584680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.584941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789211164127305:2109] [node 6] ICP22 created new session: [3:7577789211164127338:2109]
2025-11-28T14:52:19.584999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789211164127338:2109] [node 6] ICS09 handshake done sender: [3:7577789211164127321:2117] self: [3:7577789211164127320:0] peer: [6:7577789210670538484:0] socket: 489 qp: -1
2025-11-28T14:52:19.585036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789211164127338:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:19.585081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:19.586238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789211164127241:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.586301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.586343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789211164127241:2049] [node 2] ICP22 created new session: [3:7577789211164127340:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.586994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210670538481:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.587104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789212097230424:2143] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:19.587122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538485:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.587091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209981061840:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14924
2025-11-28T14:52:19.587124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789211946824082:2127] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.587318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789211164127340:2049] [node 2] ICS09 handshake done sender: [3:7577789211164127322:2118] self: [3:7577789211164127330:0] peer: [2:7577789212097230408:0] socket: 466 qp: -1
2025-11-28T14:52:19.587462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538486:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.587441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789211164127313:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:19.587479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789211164127340:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:19.587640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789211946824073:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.587746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.587697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.588980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789213054240042:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.598273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789213054239949:2048] [node 2] ICP18 handshake (actor [1:7577789213054240042:2129]) is too old
2025-11-28T14:52:19.588472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210670538482:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:19.587740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789211946824073:2108] [node 5] ICP22 created new session: [4:7577789211946824104:2108]
2025-11-28T14:52:19.588029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789211946824104:2108] [node 5] ICS09 handshake done sender: [4:7577789211946824082:2127] self: [4:7577789211946824081:0] peer: [5:7577789209981061839:0] socket: 433 qp: -1
2025-11-28T14:52:19.588276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789211946824104:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:19.599372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789211164127303:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.597767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789210670538464:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.599456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.589654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.599440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209981061830:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:19.597849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789210670538403:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.599308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789212097230319:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.600110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789211946824087:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.599886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789211164127303:2108] [node 5] ICP22 created new session: [3:7577789211164127342:2108]
2025-11-28T14:52:19.598115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:19.600141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789211946824087:2128] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.599609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:19.598265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789210670538403:2050] [node 3] ICP22 created new session: [6:7577789210670538490:2050]
2025-11-28T14:52:19.600093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789211164127342:2108] [node 5] ICS09 handshake done sender: [3:7577789211164127313:2113] self: [3:7577789211164127312:0] peer: [5:7577789209981061837:0] socket: 457 qp: -1
2025-11-28T14:52:19.599884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789212097230319:2048] [node 1] ICP22 created new session: [2:7577789212097230442:2048]
2025-11-28T14:52:19.599498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210670538490:2050] [node 3] ICS09 handshake done sender: [6:7577789210670538482:2117] self: [6:7577789210670538484:0] peer: [3:7577789211164127320:0] socket: 491 qp: -1
2025-11-28T14:52:19.600291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789211164127342:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:19.602681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789209981061807:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31332) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789209981061825:2117] held: no
2025-11-28T14:52:19.600330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.599645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210670538490:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:19.600185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789212097230442:2048] [node 1] ICS09 handshake done sender: [2:7577789212097230424:2143] self: [2:7577789212097230432:0] peer: [1:7577789213054240024:0] socket: 456 qp: -1
2025-11-28T14:52:19.602916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789209981061746:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.599689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.600308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789212097230442:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:19.602962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210670538486:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.603140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:19.600921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:19.603310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789209981061746:2050] [node 3] ICP22 created new session: [5:7577789209981061842:2050]
2025-11-28T14:52:19.603355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209981061842:2050] [node 3] ICS09 handshake done sender: [5:7577789209981061830:2120] self: [5:7577789209981061837:0] peer: [3:7577789211164127312:0] socket: 483 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:19.603429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209981061842:2050] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:19.603472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:19.603546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789209981061841:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14924
2025-11-28T14:52:19.603676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789210670538462:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.603758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789211946824087:2128] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:19.605845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789211946824074:2115] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.605888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.606214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789211946824074:2115] [node 6] ICP22 created new session: [4:7577789211946824108:2115]
2025-11-28T14:52:19.606294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789211946824108:2115] [node 6] ICS09 handshake done sender: [4:7577789211946824087:2128] self: [4:7577789211946824083:0] peer: [6:7577789210670538489:0] socket: 454 qp: -1
2025-11-28T14:52:19.606196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789212097230319:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.606335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789211946824107:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25722
2025-11-28T14:52:19.606340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789211946824108:2115] [node 6] ICS10 traffic start
2025-11-28T14:52:19.606374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:19.606432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538495:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.606742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789209981061746:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:19.607685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789211946824101:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.608003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789211946823997:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.608528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789209981061818:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.608566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789209981061818:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.609304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789212097230418:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.609319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209981061825:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:19.609338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789212097230418:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.609424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210670538481:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:19.609997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789209981061818:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:19.610023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789211946824107:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.610184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789209981061807:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.610225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.610243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789211946824074:2115] [node 6] ICP18 handshake (actor [4:7577789211946824107:2135]) is too old
2025-11-28T14:52:19.610349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789209981061807:2108] [node 4] ICP22 created new session: [5:7577789209981061845:2108]
2025-11-28T14:52:19.610379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789210670538464:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.610459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.610836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789212097230418:2139] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:19.610555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789209981061808:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.610671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209981061845:2108] [node 4] ICS09 handshake done sender: [5:7577789209981061825:2117] self: [5:7577789209981061839:0] peer: [4:7577789211946824081:0] socket: 445 qp: -1
2025-11-28T14:52:19.610691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789210670538464:2105] [node 4] ICP22 created new session: [6:7577789210670538498:2105]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.610824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210670538498:2105] [node 4] ICS09 handshake done sender: [6:7577789210670538481:2116] self: [6:7577789210670538489:0] peer: [4:7577789211946824083:0] socket: 490 qp: -1
2025-11-28T14:52:19.610915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210670538498:2105] [node 4] ICS10 traffic start
2025-11-28T14:52:19.611672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789211946824111:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25722
2025-11-28T14:52:19.610979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.611699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789212097230412:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.610758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209981061845:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:19.611633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538496:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.611789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:19.610797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.611833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789212097230412:2107] [node 4] ICP22 created new session: [2:7577789212097230446:2107]
2025-11-28T14:52:19.610949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.611882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789212097230446:2107] [node 4] ICS09 handshake done sender: [2:7577789212097230418:2139] self: [2:7577789212097230417:0] peer: [4:7577789211946824110:0] socket: 493 qp: -1
2025-11-28T14:52:19.611913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789212097230446:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:19.611190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789209981061808:2109] [node 6] ICP22 created new session: [5:7577789209981061847:2109]
2025-11-28T14:52:19.611985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:19.611407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789209981061847:2109] [node 6] ICS09 handshake done sender: [5:7577789209981061818:2114] self: [5:7577789209981061817:0] peer: [6:7577789210670538494:0] socket: 496 qp: -1
2025-11-28T14:52:19.611985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789211946824101:2134] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:19.611727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789209981061847:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.611777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:19.614925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538497:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.615478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789211946823997:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.615547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.615681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789211946823997:2049] [node 2] ICP22 created new session: [4:7577789211946824112:2049]
2025-11-28T14:52:19.615782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789211946824112:2049] [node 2] ICS09 handshake done sender: [4:7577789211946824101:2134] self: [4:7577789211946824110:0] peer: [2:7577789212097230417:0] socket: 497 qp: -1
2025-11-28T14:52:19.615814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789211946824112:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:19.615971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.616820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789210670538496:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.616750Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789211946823997:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:19.616955Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789210670538462:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19514) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789210670538486:2120] held: no
2025-11-28T14:52:19.617081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210670538486:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:19.617782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789210670538462:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.617871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:19.617915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789210670538462:2104] [node 5] ICP22 created new session: [6:7577789210670538500:2104]
2025-11-28T14:52:19.617977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210670538500:2104] [node 5] ICS09 handshake done sender: [6:7577789210670538486:2120] self: [6:7577789210670538494:0] peer: [5:7577789209981061817:0] socket: 499 qp: -1
2025-11-28T14:52:19.618006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210670538500:2104] [node 5] ICS10 traffic start
2025-11-28T14:52:19.618041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:19.618390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789210670538402:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:19.618658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789212097230426:2144] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:19.618693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789212097230426:2144] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:19.619698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789212097230426:2144] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:19.619892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789212097230414:2134] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:19.619986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:19.620032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789212097230414:2134] [node 6] ICP22 created new session: [2:7577789212097230449:2134]
2025-11-28T14:52:19.620075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789212097230449:2134] [node 6] ICS09 handshake done sender: [2:7577789212097230426:2144] self: [2:7577789212097230425:0] peer: [6:7577789210670538502:0] socket: 465 qp: -1
2025-11-28T14:52:19.620116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789212097230449:2134] [node 6] ICS10 traffic start
2025-11-28T14:52:19.619958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789210670538503:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1869
2025-11-28T14:52:19.620154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:19.620226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789210670538496:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:19.620710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789210670538402:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:19.620761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:19.620807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789210670538402:2049] [node 2] ICP22 created new session: [6:7577789210670538504:2049]
2025-11-28T14:52:19.620857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789210670538504:2049] [node 2] ICS09 handshake done sender: [6:7577789210670538496:2122] self: [6:7577789210670538502:0] peer: [2:7577789212097230425:0] socket: 468 qp: -1
2025-11-28T14:52:19.620891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789210670538504:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:19.620926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:19.621541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789210670538402:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:19.658009Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:19.658256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789212097230442:2048] [node 1] ICS07 socket disconnect 456 reason# EndOfStream
2025-11-28T14:52:19.658315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789212097230442:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.658419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.658478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789212097230442:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:19.659281Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:19.659509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789211164127327:2048] [node 1] ICS01 socket: 452 reason# EndOfStream
2025-11-28T14:52:19.659556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789211164127240:2048] [node 1] ICP30 unregister session Session# [3:7577789211164127327:2048] VirtualId# [3:7577789211164127325:0]
2025-11-28T14:52:19.659587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789211164127327:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.659673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.659739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.660919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789212097230319:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11595) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:19.660958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789212097230442:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:19.662449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:19.662650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789212097230319:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11595) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.662705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789212097230442:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.662745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789212097230319:2048] [node 1] ICP30 unregister session Session# [2:7577789212097230442:2048] VirtualId# [2:7577789212097230432:0]
2025-11-28T14:52:19.662794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.662835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789212097230319:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11595) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.662635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789211946824092:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-11-28T14:52:19.662849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11595) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.662671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789211946823996:2048] [node 1] ICP30 unregister session Session# [4:7577789211946824092:2048] VirtualId# [4:7577789211946824090:0]
2025-11-28T14:52:19.662696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789211946824092:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.662757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.662794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.663737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:19.663945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789209981061835:2048] [node 1] ICS01 socket: 478 reason# EndOfStream
2025-11-28T14:52:19.663990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789209981061744:2048] [node 1] ICP30 unregister session Session# [5:7577789209981061835:2048] VirtualId# [5:7577789209981061829:0]
2025-11-28T14:52:19.664020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789209981061835:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.664096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.664156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.664675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:19.664892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789210670538487:2048] [node 1] ICS07 socket disconnect 480 reason# EndOfStream
2025-11-28T14:52:19.664926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210670538487:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.665001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:19.665056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789210670538487:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:19.665791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789210670538401:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11595) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.665840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210670538487:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.665901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789210670538401:2048] [node 1] ICP30 unregister session Session# [6:7577789210670538487:2048] VirtualId# [6:7577789210670538480:0]
2025-11-28T14:52:19.665952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:19.665983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789210670538401:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11595) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.666002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11595) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.675836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789210670538401:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:19.695958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:19.696186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789211164127340:2049] [node 2] ICS07 socket disconnect 466 reason# EndOfStream
2025-11-28T14:52:19.696221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789211164127340:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.696342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:19.696427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789211164127340:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:19.697146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789211164127241:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.697247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789211164127340:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.697282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789211164127241:2049] [node 2] ICP30 unregister session Session# [3:7577789211164127340:2049] VirtualId# [3:7577789211164127330:0]
2025-11-28T14:52:19.697354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:19.697404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789211164127241:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.697433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.698353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:19.698573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789211946824112:2049] [node 2] ICS07 socket disconnect 497 reason# EndOfStream
2025-11-28T14:52:19.698611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789211946824112:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.698670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:19.698714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789211946824112:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:19.699733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789211946823997:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.699802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789211946824112:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.699840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789211946823997:2049] [node 2] ICP30 unregister session Session# [4:7577789211946824112:2049] VirtualId# [4:7577789211946824110:0]
2025-11-28T14:52:19.699779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:19.699946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:19.699973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789209981061831:2049] [node 2] ICS07 socket disconnect 472 reason# EndOfStream
2025-11-28T14:52:19.699984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789211946823997:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.700015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789209981061831:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.700000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.700088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:19.700126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789209981061831:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:19.700595Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:19.700711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789209981061745:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.700806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789210670538504:2049] [node 2] ICS07 socket disconnect 468 reason# EndOfStream
2025-11-28T14:52:19.700766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789209981061831:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.700837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210670538504:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.700788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789209981061745:2049] [node 2] ICP30 unregister session Session# [5:7577789209981061831:2049] VirtualId# [5:7577789209981061827:0]
2025-11-28T14:52:19.700854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:19.700909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:19.701048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789209981061745:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.700981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789210670538504:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:19.701089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.701805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789210670538402:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.701949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210670538504:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.701987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789210670538402:2049] [node 2] ICP30 unregister session Session# [6:7577789210670538504:2049] VirtualId# [6:7577789210670538502:0]
2025-11-28T14:52:19.702031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:19.702071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789210670538402:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.702089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.709642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789211946823997:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:19.710610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789209981061745:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:19.711656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789210670538402:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:19.735142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:19.735434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789211946824097:2050] [node 3] ICS01 socket: 462 reason# EndOfStream
2025-11-28T14:52:19.735490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789211946823998:2050] [node 3] ICP30 unregister session Session# [4:7577789211946824097:2050] VirtualId# [4:7577789211946824095:0]
2025-11-28T14:52:19.735526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789211946824097:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.735616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:19.735683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:19.735866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:19.736039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789209981061842:2050] [node 3] ICS07 socket disconnect 483 reason# EndOfStream
2025-11-28T14:52:19.736067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789209981061842:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.736150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:19.736213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789209981061842:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:19.736726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:19.736990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210670538490:2050] [node 3] ICS01 socket: 491 reason# EndOfStream
2025-11-28T14:52:19.737034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789210670538403:2050] [node 3] ICP30 unregister session Session# [6:7577789210670538490:2050] VirtualId# [6:7577789210670538484:0]
2025-11-28T14:52:19.737094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210670538490:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.737183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:19.737041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789209981061746:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12885) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.737235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:19.737151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789209981061842:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.737247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789209981061746:2050] [node 3] ICP30 unregister session Session# [5:7577789209981061842:2050] VirtualId# [5:7577789209981061837:0]
2025-11-28T14:52:19.737306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:19.737383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789209981061746:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12885) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.737398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12885) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.747519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789209981061746:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:19.784105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:19.784311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789209981061845:2108] [node 4] ICS01 socket: 445 reason# EndOfStream
2025-11-28T14:52:19.784348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789209981061807:2108] [node 4] ICP30 unregister session Session# [5:7577789209981061845:2108] VirtualId# [5:7577789209981061839:0]
2025-11-28T14:52:19.784417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789209981061845:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.784498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:19.784557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:19.785333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:19.785561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789210670538498:2105] [node 4] ICS07 socket disconnect 490 reason# EndOfStream
2025-11-28T14:52:19.785599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210670538498:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.785681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:19.785739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789210670538498:2105] [node 4] ICS15 start handshake
2025-11-28T14:52:19.786433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789210670538464:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31332) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:19.786557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210670538498:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:19.786615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789210670538464:2105] [node 4] ICP30 unregister session Session# [6:7577789210670538498:2105] VirtualId# [6:7577789210670538489:0]
2025-11-28T14:52:19.786666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:19.786706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789210670538464:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31332) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.786719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31332) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:19.796735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789210670538464:2105] [node 4] ICP33 wake up from error state
2025-11-28T14:52:19.806879Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:19.807152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789210670538500:2104] [node 5] ICS01 socket: 499 reason# EndOfStream
2025-11-28T14:52:19.807219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789210670538462:2104] [node 5] ICP30 unregister session Session# [6:7577789210670538500:2104] VirtualId# [6:7577789210670538494:0]
2025-11-28T14:52:19.807257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789210670538500:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:19.807358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:19.807416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:19.929593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789209802539869:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:19.929792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789209802539870:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:19.951446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789210671225216:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:19.951507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789210671225217:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:19.972441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789212955663785:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:19.972562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789212955663786:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:19.993875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789212231175496:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:19.993972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789212231175497:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:19.993997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789212231175498:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:20.014087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215507736132:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:20.014126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215507736133:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:20.014149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215507736134:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:20.037572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215701149475:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:20.037611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215701149477:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:20.037702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215701149476:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:20.129821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215701149536:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:20.140224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789214097507229:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:20.140697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215507736196:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:20.140653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789214966192593:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:20.140764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789216526142855:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:20.140864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789214097507230:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:20.141158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789217250631145:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:20.142081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789214966192594:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:20.141890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215701149539:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:20.142126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789217250631146:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:20.142507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215507736197:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:20.142571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789214097507231:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:20.142613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789217250631147:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:20.142770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789216526142856:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:20.143614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789214966192595:2116] [node 6] ICP01 ready to work
2025-11-28T14:52:20.144919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789212231175497:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51430699ns) {Configure:104(51426746ns) {ProcessConfigured:135(200084ns) {StartInitialHandshake:157(197536ns) {DropHandshakes:470(1708ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(393ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:477(1274ns) {SwitchToState:179(512ns)}}}}}
2025-11-28T14:52:20.144970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789212231175496:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47703308ns) {Configure:104(47688811ns) {ProcessConfigured:135(255125ns) {StartInitialHandshake:157(252945ns) {DropHandshakes:470(2159ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(539ns)} GenerateSessionVirtualId:988(506ns) PrepareNewSessionHandshake:477(940ns) {SwitchToState:179(325ns)}}}}}
2025-11-28T14:52:20.146350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789212955663785:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(55027953ns) {Configure:104(55020947ns) {ProcessConfigured:135(274227ns) {StartInitialHandshake:157(272669ns) {DropHandshakes:470(1229ns) {DropIncomingHandshake:442(286ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(604ns) PrepareNewSessionHandshake:477(1583ns) {SwitchToState:179(880ns)}}}}}
2025-11-28T14:52:20.147270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789210671225216:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50043406ns) {Configure:104(50035278ns) {ProcessConfigured:135(422120ns) {StartInitialHandshake:157(420485ns) {DropHandshakes:470(1230ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(609ns) PrepareNewSessionHandshake:477(1455ns) {SwitchToState:179(520ns)}}}}}
2025-11-28T14:52:20.147442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789210671225217:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(57138161ns) {Configure:104(50045394ns) {ProcessConfigured:135(395026ns) {StartInitialHandshake:157(392885ns) {DropHandshakes:470(1348ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:477(1609ns) {SwitchToState:179(965ns)}}}}}
2025-11-28T14:52:20.147836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789215701149475:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31592869ns) {Configure:104(31583176ns) {ProcessConfigured:135(198484ns) {StartInitialHandshake:157(195877ns) {DropHandshakes:470(2027ns) {DropIncomingHandshake:442(607ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(614ns) PrepareNewSessionHandshake:477(1840ns) {SwitchToState:179(888ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.150634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789216526142870:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20853
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.153701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215507736204:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65065
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.154487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789214097507238:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48707
2025-11-28T14:52:20.154600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789214097507239:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48707
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.154762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215507736204:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.156200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789215507736196:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.156195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789214966192603:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29521
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.156626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507238:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.156618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789216526142862:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.156654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789216526142862:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.157318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789214966192602:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29521
2025-11-28T14:52:20.157291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789214097507247:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48707
2025-11-28T14:52:20.158922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789214966192611:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29521
2025-11-28T14:52:20.158903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789217250631161:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48756
No RDMA context found for GID: 2025-11-28T14:52:20.159692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789214097507248:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48707
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.162683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789214097507249:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48707
2025-11-28T14:52:20.159560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789214966192612:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.162868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789217250631161:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.162794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507248:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.161588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789214966192603:2132] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.162802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507239:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.163012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789216526142871:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20853
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.163138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789216526142862:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:20.163305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789217250631162:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48756
2025-11-28T14:52:20.162028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789214966192611:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.162829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789214966192602:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.163179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789216526142870:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.164023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789217250631163:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48756
2025-11-28T14:52:20.163413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789212231175498:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.162983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789214966192612:2136] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.163377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789214966192613:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29521
2025-11-28T14:52:20.164143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789217250631164:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48756
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.163475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789216526142872:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20853
2025-11-28T14:52:20.164643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789217250631162:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.164264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789216526142855:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.164685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789217250631153:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.164708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789217250631153:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.166893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507247:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.164869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:20.167006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789217250631153:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:20.166572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789216526142855:2108] [node 5] ICP22 created new session: [4:7577789216526142876:2108]
2025-11-28T14:52:20.166915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215507736213:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65065
2025-11-28T14:52:20.166897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789216526142876:2108] [node 5] ICS09 handshake done sender: [4:7577789216526142862:2112] self: [4:7577789216526142861:0] peer: [5:7577789215507736212:0] socket: 440 qp: -1
2025-11-28T14:52:20.167334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789217250631164:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.167352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789214966192613:2137] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.167380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507249:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.167052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789216526142876:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:20.167869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215507736204:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:20.167645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789212955663786:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.167904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789210671225216:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.167189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:20.168493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789215507736196:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.168738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507234:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.168054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789217250631163:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.168558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149553:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
2025-11-28T14:52:20.167338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789216526142872:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.168138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789217250631145:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.168775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214097507234:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.168454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789214966192598:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.169816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.168180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.167573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789216526142871:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.168480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789214966192598:2128] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.169848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215507736196:2108] [node 4] ICP22 created new session: [5:7577789215507736214:2108]
2025-11-28T14:52:20.169887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789215507736214:2108] [node 4] ICS09 handshake done sender: [5:7577789215507736204:2114] self: [5:7577789215507736212:0] peer: [4:7577789216526142861:0] socket: 442 qp: -1
2025-11-28T14:52:20.168513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789217250631145:2107] [node 4] ICP22 created new session: [3:7577789217250631169:2107]
2025-11-28T14:52:20.170143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789215507736214:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:20.170176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.170687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789212231175497:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.168627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789217250631169:2107] [node 4] ICS09 handshake done sender: [3:7577789217250631153:2113] self: [3:7577789217250631151:0] peer: [4:7577789216526142874:0] socket: 443 qp: -1
2025-11-28T14:52:20.170647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507236:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.170739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789214966192598:2128] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:20.177731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789212231175496:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.170690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214097507236:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.169537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789217250631169:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:20.186661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789216526142878:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20853
2025-11-28T14:52:20.169548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789212955663785:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.171200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214097507234:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:20.169830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.190927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789210671225217:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.190975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214097507236:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:20.191609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789216526142870:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:20.191556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149554:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
2025-11-28T14:52:20.191188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:20.191314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789217250631173:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48756
2025-11-28T14:52:20.191516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789209802539869:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.192881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215507736217:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65065
2025-11-28T14:52:20.191811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789217250631145:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22355133ns) {HandleHandshakeStatus:329(22352533ns) {DropIncomingHandshake:442(426ns) DropOutgoingHandshake:457(517ns) SwitchToState:179(275ns) ProcessPendingSessionEvents:495(20051572ns) {ForwardSessionEventToSession:612(7334838ns) ForwardSessionEventToSession:612(6903ns) ForwardSessionEventToSession:612(8834025ns) ForwardSessionEventToSession:612(8389ns) ForwardSessionEventToSession:612(2743ns) ForwardSessionEventToSession:612(2495060ns) ForwardSessionEventToSession:612(16972ns) ForwardSessionEventToSession:612(3790ns) ForwardSessionEventToSession:612(1314071ns) ForwardSessionEventToSession:612(3689ns)}}}
2025-11-28T14:52:20.192957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789215701149554:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:20.191347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789210671225217:2049] [node 3] ICP22 created new session: [2:7577789214966192621:2049]
2025-11-28T14:52:20.193360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789212231175498:2050] [node 3] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:20.191585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507237:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.193006Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789215507736196:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(22160629ns) {HandleHandshakeStatus:329(22158333ns) {DropIncomingHandshake:442(485ns) DropOutgoingHandshake:457(9472ns) SwitchToState:179(353ns) ProcessPendingSessionEvents:495(13170779ns) {ForwardSessionEventToSession:612(12202919ns) ForwardSessionEventToSession:612(8860ns) ForwardSessionEventToSession:612(889711ns) ForwardSessionEventToSession:612(6846ns) ForwardSessionEventToSession:612(3491ns) ForwardSessionEventToSession:612(20080ns) ForwardSessionEventToSession:612(4249ns) ForwardSessionEventToSession:612(3117ns) ForwardSessionEventToSession:612(12719ns) ForwardSessionEventToSession:612(3153ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.191566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789214966192601:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.191636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214097507237:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.191594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789214966192601:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.193407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:20.193911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215701149536:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.191664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789214966192621:2049] [node 3] ICS09 handshake done sender: [2:7577789214966192598:2128] self: [2:7577789214966192597:0] peer: [3:7577789217250631168:0] socket: 465 qp: -1
2025-11-28T14:52:20.193952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789217250631164:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:20.193906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789212231175498:2050] [node 3] ICP22 created new session: [4:7577789216526142881:2050]
2025-11-28T14:52:20.192904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789214966192621:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:20.191793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:20.192964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:20.193262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789209802539869:2048] [node 2] ICP22 created new session: [1:7577789214097507257:2048]
2025-11-28T14:52:20.194358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789217250631174:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48756
2025-11-28T14:52:20.194823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215507736217:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.194770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789212955663786:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.193855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214097507257:2048] [node 2] ICS09 handshake done sender: [1:7577789214097507234:2111] self: [1:7577789214097507232:0] peer: [2:7577789214966192618:0] socket: 467 qp: -1
2025-11-28T14:52:20.195174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215507736211:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.193894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214097507257:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:20.195397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:20.194285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789214966192620:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29521
2025-11-28T14:52:20.194176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789216526142881:2050] [node 3] ICS09 handshake done sender: [4:7577789216526142870:2115] self: [4:7577789216526142874:0] peer: [3:7577789217250631151:0] socket: 444 qp: -1
2025-11-28T14:52:20.195200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789215507736211:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.194336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789216526142881:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:20.193950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789209802539870:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.197311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789212955663786:2049] [node 2] ICP22 created new session: [3:7577789217250631175:2049]
2025-11-28T14:52:20.197508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789214966192613:2137] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.194363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:20.197351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789217250631175:2049] [node 2] ICS09 handshake done sender: [3:7577789217250631164:2119] self: [3:7577789217250631168:0] peer: [2:7577789214966192597:0] socket: 478 qp: -1
2025-11-28T14:52:20.197535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789215507736134:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.194132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:20.197375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789217250631175:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:20.198033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215507736211:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:20.194441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:20.202005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789214966192601:2130] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:20.197402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:20.202159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789215507736197:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.195200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789209802539870:2049] [node 3] ICP22 created new session: [1:7577789214097507259:2049]
2025-11-28T14:52:20.200974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789217250631163:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.202360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789210671225216:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17342) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789214966192613:2137] held: no
2025-11-28T14:52:20.202600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789217250631158:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.197418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214097507259:2049] [node 3] ICS09 handshake done sender: [1:7577789214097507236:2112] self: [1:7577789214097507233:0] peer: [3:7577789217250631170:0] socket: 466 qp: -1
2025-11-28T14:52:20.203385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.202621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789217250631158:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.202440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789210671225216:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.197462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214097507259:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:20.202686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789212955663786:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:20.204501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149556:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
2025-11-28T14:52:20.202809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:20.203634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215507736197:2109] [node 6] ICP22 created new session: [5:7577789215507736221:2109]
2025-11-28T14:52:20.197513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: 2025-11-28T14:52:20.204036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789212955663785:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:20.203490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789210671225216:2048] [node 1] ICP22 created new session: [2:7577789214966192625:2048]
2025-11-28T14:52:20.204161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:20.204864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789214966192593:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.204211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789212955663785:2048] [node 1] ICP22 created new session: [3:7577789217250631177:2048]
2025-11-28T14:52:20.205006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789214966192625:2048] [node 1] ICS09 handshake done sender: [2:7577789214966192613:2137] self: [2:7577789214966192618:0] peer: [1:7577789214097507232:0] socket: 476 qp: -1
2025-11-28T14:52:20.205155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214097507237:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.205144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215507736220:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65065
2025-11-28T14:52:20.205502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149557:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
2025-11-28T14:52:20.205029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.204895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789217250631177:2048] [node 1] ICS09 handshake done sender: [3:7577789217250631163:2118] self: [3:7577789217250631170:0] peer: [1:7577789214097507233:0] socket: 475 qp: -1
2025-11-28T14:52:20.205510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789215507736221:2109] [node 6] ICS09 handshake done sender: [5:7577789215507736211:2116] self: [5:7577789215507736209:0] peer: [6:7577789215701149555:0] socket: 487 qp: -1
2025-11-28T14:52:20.205984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789214097507229:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.205891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789215701149554:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:20.204941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789217250631177:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.206197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789216526142883:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20853
2025-11-28T14:52:20.205039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789214966192625:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.205081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:20.205754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789215507736221:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:20.207399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215701149536:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.205075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789214966192593:2107] [node 4] ICP22 created new session: [2:7577789214966192626:2107]
2025-11-28T14:52:20.206041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.205359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:20.206016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:20.205771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789214966192626:2107] [node 4] ICS09 handshake done sender: [2:7577789214966192601:2130] self: [2:7577789214966192600:0] peer: [4:7577789216526142879:0] socket: 471 qp: -1
2025-11-28T14:52:20.206262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789214097507229:2107] [node 4] ICP22 created new session: [1:7577789214097507264:2107]
2025-11-28T14:52:20.208355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:20.206802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215507736222:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65065
2025-11-28T14:52:20.208747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789216526142872:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:20.205820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789214966192626:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.207145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214097507264:2107] [node 4] ICS09 handshake done sender: [1:7577789214097507237:2113] self: [1:7577789214097507235:0] peer: [4:7577789216526142880:0] socket: 470 qp: -1
2025-11-28T14:52:20.208431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215701149536:2106] [node 5] ICP22 created new session: [6:7577789215701149558:2106]
2025-11-28T14:52:20.209908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789212955663785:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:20.205867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:20.209073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789215701149558:2106] [node 5] ICS09 handshake done sender: [6:7577789215701149554:2116] self: [6:7577789215701149555:0] peer: [5:7577789215507736209:0] socket: 488 qp: -1
2025-11-28T14:52:20.209700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789215701149558:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:20.207419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214097507264:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:20.210461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789217250631158:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.210888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789216526142884:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20853
2025-11-28T14:52:20.209937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:20.208326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:20.210901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789212231175497:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20851) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789216526142872:2117] held: no
2025-11-28T14:52:20.211552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215507736222:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.211116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789212231175497:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.212431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789217250631146:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.212486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:20.212263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789215701149557:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.211149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:20.211558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215507736224:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65065
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.211437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789212231175497:2049] [node 2] ICP22 created new session: [4:7577789216526142885:2049]
2025-11-28T14:52:20.212083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789215507736132:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.213568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507243:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.213640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789217250631146:2108] [node 5] ICP22 created new session: [3:7577789217250631181:2108]
2025-11-28T14:52:20.211601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789216526142885:2049] [node 2] ICS09 handshake done sender: [4:7577789216526142872:2117] self: [4:7577789216526142879:0] peer: [2:7577789214966192600:0] socket: 474 qp: -1
2025-11-28T14:52:20.213609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214097507243:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.213334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215507736217:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:20.215511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149560:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
2025-11-28T14:52:20.211829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789216526142885:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:20.215273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789215507736134:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.215026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789217250631181:2108] [node 5] ICS09 handshake done sender: [3:7577789217250631158:2114] self: [3:7577789217250631157:0] peer: [5:7577789215507736219:0] socket: 486 qp: -1
2025-11-28T14:52:20.216928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215701149475:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.216251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: 2025-11-28T14:52:20.215237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789217250631181:2108] [node 5] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:20.212068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:20.216253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215507736220:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.216246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214097507243:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:20.217028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789215701149560:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.215287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:20.216986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215507736134:2050] [node 3] ICP22 created new session: [5:7577789215507736226:2050]
2025-11-28T14:52:20.217223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215701149477:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.217049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789215507736226:2050] [node 3] ICS09 handshake done sender: [5:7577789215507736217:2118] self: [5:7577789215507736219:0] peer: [3:7577789217250631157:0] socket: 489 qp: -1
2025-11-28T14:52:20.217149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789214097507230:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.219061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789217250631160:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.217620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149562:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
No RDMA context found for GID: 2025-11-28T14:52:20.217100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789215507736226:2050] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:20.219204Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789217250631160:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.217137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:20.218809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789215701149562:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.217668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:20.220235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789214966192607:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.220024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215701149476:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.220155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789215507736134:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:20.218527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789214097507230:2109] [node 5] ICP22 created new session: [1:7577789214097507267:2109]
2025-11-28T14:52:20.220710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215507736227:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65065
2025-11-28T14:52:20.220266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789214966192607:2134] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.218868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214097507241:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.218897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214097507241:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.219976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214097507267:2109] [node 5] ICS09 handshake done sender: [1:7577789214097507243:2117] self: [1:7577789214097507242:0] peer: [5:7577789215507736225:0] socket: 500 qp: -1
2025-11-28T14:52:20.220025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214097507267:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.220325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:20.220842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214097507241:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:20.222377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789216526142871:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.222612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789217250631160:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:20.221680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789214097507231:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.222441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789215507736133:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.222474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149566:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
2025-11-28T14:52:20.222695Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789212231175496:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17342) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789216526142871:2116] held: no
2025-11-28T14:52:20.221729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:20.223053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215507736222:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.222824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789212231175496:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.224439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789214966192605:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.222288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789214097507231:2110] [node 6] ICP22 created new session: [1:7577789214097507270:2110]
2025-11-28T14:52:20.224461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789215507736132:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17342) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789215507736222:2120] held: no
2025-11-28T14:52:20.226323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789217250631147:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.224478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789214966192605:2133] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.222359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214097507270:2110] [node 6] ICS09 handshake done sender: [1:7577789214097507241:2116] self: [1:7577789214097507240:0] peer: [6:7577789215701149563:0] socket: 499 qp: -1
2025-11-28T14:52:20.226553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789215507736132:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.226535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:20.226853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789214966192607:2134] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:20.224118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:20.226652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789217250631147:2110] [node 6] ICP22 created new session: [3:7577789217250631184:2110]
2025-11-28T14:52:20.222399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214097507270:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:20.226593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: 2025-11-28T14:52:20.226879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789217250631184:2110] [node 6] ICS09 handshake done sender: [3:7577789217250631160:2115] self: [3:7577789217250631159:0] peer: [6:7577789215701149564:0] socket: 447 qp: -1
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.224230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789212231175496:2048] [node 1] ICP22 created new session: [4:7577789216526142888:2048]
2025-11-28T14:52:20.227756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215701149475:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17342) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789215701149557:2118] held: no
2025-11-28T14:52:20.222443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:20.225193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789216526142888:2048] [node 1] ICS09 handshake done sender: [4:7577789216526142871:2116] self: [4:7577789216526142880:0] peer: [1:7577789214097507235:0] socket: 472 qp: -1
2025-11-28T14:52:20.228323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789214966192595:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.227747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789217250631184:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:20.228460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149567:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
2025-11-28T14:52:20.227758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215507736132:2048] [node 1] ICP22 created new session: [5:7577789215507736230:2048]
2025-11-28T14:52:20.226339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789216526142888:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.228332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789214966192605:2133] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:20.227796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:20.226501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:20.228023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789215507736230:2048] [node 1] ICS09 handshake done sender: [5:7577789215507736222:2120] self: [5:7577789215507736225:0] peer: [1:7577789214097507242:0] socket: 501 qp: -1
2025-11-28T14:52:20.228518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:20.228939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789215701149557:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.228388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789215507736230:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.228560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789214966192595:2116] [node 6] ICP22 created new session: [2:7577789214966192632:2116]
2025-11-28T14:52:20.228466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789216526142887:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20853
2025-11-28T14:52:20.228975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789214966192632:2116] [node 6] ICS09 handshake done sender: [2:7577789214966192607:2134] self: [2:7577789214966192606:0] peer: [6:7577789215701149565:0] socket: 438 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.228531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:20.229452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789214966192594:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.229458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789214966192632:2116] [node 6] ICS10 traffic start
2025-11-28T14:52:20.229605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:20.230531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215507736232:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65065
2025-11-28T14:52:20.230360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:20.230604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789214966192594:2108] [node 5] ICP22 created new session: [2:7577789214966192634:2108]
2025-11-28T14:52:20.230582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215701149477:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29886) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789215701149560:2119] held: no
2025-11-28T14:52:20.230698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789214966192634:2108] [node 5] ICS09 handshake done sender: [2:7577789214966192605:2133] self: [2:7577789214966192604:0] peer: [5:7577789215507736229:0] socket: 497 qp: -1
2025-11-28T14:52:20.230586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789216526142887:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.230633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215701149475:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.230736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789214966192634:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:20.231050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215507736220:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:20.230958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:20.230759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:20.231128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215701149475:2048] [node 1] ICP22 created new session: [6:7577789215701149569:2048]
2025-11-28T14:52:20.231551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789215507736133:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20851) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789215507736220:2119] held: no
2025-11-28T14:52:20.231398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789215701149569:2048] [node 1] ICS09 handshake done sender: [6:7577789215701149557:2118] self: [6:7577789215701149563:0] peer: [1:7577789214097507240:0] socket: 468 qp: -1
2025-11-28T14:52:20.231433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789215701149569:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.231475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:20.232494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789215701149560:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:20.232545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789215507736133:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.232640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:20.233106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215507736133:2049] [node 2] ICP22 created new session: [5:7577789215507736233:2049]
2025-11-28T14:52:20.233340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789215507736233:2049] [node 2] ICS09 handshake done sender: [5:7577789215507736220:2119] self: [5:7577789215507736229:0] peer: [2:7577789214966192604:0] socket: 498 qp: -1
2025-11-28T14:52:20.233410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215701149476:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20851) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789215701149562:2120] held: no
No RDMA context found for GID: 2025-11-28T14:52:20.233441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789215507736233:2049] [node 2] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:20.233473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:20.233626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149568:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
2025-11-28T14:52:20.233655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215701149477:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.233783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:20.233842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215701149477:2050] [node 3] ICP22 created new session: [6:7577789215701149571:2050]
2025-11-28T14:52:20.233893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789215701149571:2050] [node 3] ICS09 handshake done sender: [6:7577789215701149560:2119] self: [6:7577789215701149564:0] peer: [3:7577789217250631159:0] socket: 504 qp: -1
2025-11-28T14:52:20.233924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789215701149562:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:20.233928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789215701149571:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:20.233953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:20.234351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215701149476:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.234354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789215701149553:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.234422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:20.234464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215701149476:2049] [node 2] ICP22 created new session: [6:7577789215701149573:2049]
2025-11-28T14:52:20.234509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215701149539:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.234511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789215701149573:2049] [node 2] ICS09 handshake done sender: [6:7577789215701149562:2120] self: [6:7577789215701149565:0] peer: [2:7577789214966192606:0] socket: 505 qp: -1
2025-11-28T14:52:20.234536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789215701149573:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:20.234569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:20.234813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789216526142867:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.234846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789216526142867:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.235553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789216526142867:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:20.235712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789216526142856:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.235779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:20.235819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789216526142856:2109] [node 6] ICP22 created new session: [4:7577789216526142892:2109]
2025-11-28T14:52:20.235852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789216526142892:2109] [node 6] ICS09 handshake done sender: [4:7577789216526142867:2113] self: [4:7577789216526142866:0] peer: [6:7577789215701149574:0] socket: 482 qp: -1
2025-11-28T14:52:20.235842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215701149576:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62753
2025-11-28T14:52:20.235873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789216526142892:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:20.235912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:20.236090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789215701149553:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:20.236410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215701149539:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789215701149553:2115] held: no
2025-11-28T14:52:20.236575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215701149539:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.236647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.236697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215701149539:2107] [node 4] ICP22 created new session: [6:7577789215701149577:2107]
2025-11-28T14:52:20.236769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789215701149577:2107] [node 4] ICS09 handshake done sender: [6:7577789215701149553:2115] self: [6:7577789215701149574:0] peer: [4:7577789216526142866:0] socket: 483 qp: -1
2025-11-28T14:52:20.236823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789215701149577:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:20.236864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:20.267762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:20.267941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789214966192625:2048] [node 1] ICS01 socket: 476 reason# EndOfStream
2025-11-28T14:52:20.267982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789210671225216:2048] [node 1] ICP30 unregister session Session# [2:7577789214966192625:2048] VirtualId# [2:7577789214966192618:0]
2025-11-28T14:52:20.268014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789214966192625:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.268089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:20.268151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:20.268854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:20.269066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789217250631177:2048] [node 1] ICS07 socket disconnect 475 reason# EndOfStream
2025-11-28T14:52:20.269104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789217250631177:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.269193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:20.269242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789217250631177:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:20.269913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789212955663785:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17342) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:20.269966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789217250631177:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:20.269990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789212955663785:2048] [node 1] ICP30 unregister session Session# [3:7577789217250631177:2048] VirtualId# [3:7577789217250631170:0]
2025-11-28T14:52:20.270032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:20.270059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789212955663785:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.270068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.270723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:20.270920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789216526142888:2048] [node 1] ICS01 socket: 472 reason# EndOfStream
2025-11-28T14:52:20.270959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789212231175496:2048] [node 1] ICP30 unregister session Session# [4:7577789216526142888:2048] VirtualId# [4:7577789216526142880:0]
2025-11-28T14:52:20.270998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789216526142888:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.271077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:20.271122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:20.271377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:20.271599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789215507736230:2048] [node 1] ICS01 socket: 501 reason# EndOfStream
2025-11-28T14:52:20.271652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789215507736132:2048] [node 1] ICP30 unregister session Session# [5:7577789215507736230:2048] VirtualId# [5:7577789215507736225:0]
2025-11-28T14:52:20.271682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789215507736230:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.271758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:20.271818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:20.272207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:20.272423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789215701149569:2048] [node 1] ICS01 socket: 468 reason# EndOfStream
2025-11-28T14:52:20.272467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215701149475:2048] [node 1] ICP30 unregister session Session# [6:7577789215701149569:2048] VirtualId# [6:7577789215701149563:0]
2025-11-28T14:52:20.272492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789215701149569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.272569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:20.272633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:20.279993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789212955663785:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:20.330102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:20.330385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789217250631175:2049] [node 2] ICS07 socket disconnect 478 reason# EndOfStream
2025-11-28T14:52:20.330432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789217250631175:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.330511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:20.330575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789217250631175:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:20.331305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789212955663786:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20851) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:20.331414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789217250631175:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:20.331469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789212955663786:2049] [node 2] ICP30 unregister session Session# [3:7577789217250631175:2049] VirtualId# [3:7577789217250631168:0]
2025-11-28T14:52:20.331518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:20.331583Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789212955663786:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20851) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.331609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20851) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.332255Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:20.332450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789216526142885:2049] [node 2] ICS01 socket: 474 reason# EndOfStream
2025-11-28T14:52:20.332479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789212231175497:2049] [node 2] ICP30 unregister session Session# [4:7577789216526142885:2049] VirtualId# [4:7577789216526142879:0]
2025-11-28T14:52:20.332506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789216526142885:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.332593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:20.332674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:20.332943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:20.333158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789215507736233:2049] [node 2] ICS01 socket: 498 reason# EndOfStream
2025-11-28T14:52:20.333201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789215507736133:2049] [node 2] ICP30 unregister session Session# [5:7577789215507736233:2049] VirtualId# [5:7577789215507736229:0]
2025-11-28T14:52:20.333244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789215507736233:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.333318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:20.333389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:20.333722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:20.333937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789215701149573:2049] [node 2] ICS01 socket: 505 reason# EndOfStream
2025-11-28T14:52:20.333977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215701149476:2049] [node 2] ICP30 unregister session Session# [6:7577789215701149573:2049] VirtualId# [6:7577789215701149565:0]
2025-11-28T14:52:20.334001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789215701149573:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.334062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:20.334127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:20.380813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:20.380981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789216526142881:2050] [node 3] ICS07 socket disconnect 444 reason# EndOfStream
2025-11-28T14:52:20.381009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789216526142881:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.381070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:20.381134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789216526142881:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:20.381520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:20.381745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789215507736226:2050] [node 3] ICS07 socket disconnect 489 reason# EndOfStream
2025-11-28T14:52:20.381762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789212231175498:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29886) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:20.381777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789215507736226:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.381839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789216526142881:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:20.381848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:20.381869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789212231175498:2050] [node 3] ICP30 unregister session Session# [4:7577789216526142881:2050] VirtualId# [4:7577789216526142874:0]
2025-11-28T14:52:20.381905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:20.381893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789215507736226:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:20.381947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789212231175498:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29886) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.381999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29886) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.382467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:20.382655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789215701149571:2050] [node 3] ICS01 socket: 504 reason# EndOfStream
2025-11-28T14:52:20.382693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215701149477:2050] [node 3] ICP30 unregister session Session# [6:7577789215701149571:2050] VirtualId# [6:7577789215701149564:0]
2025-11-28T14:52:20.382707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789215507736134:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29886) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:20.382720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789215701149571:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.382799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:20.382745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789215507736226:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:20.382853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:20.382769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789215507736134:2050] [node 3] ICP30 unregister session Session# [5:7577789215507736226:2050] VirtualId# [5:7577789215507736219:0]
2025-11-28T14:52:20.382812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:20.382849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789215507736134:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29886) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.382857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29886) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.391645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789212231175498:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:20.392605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789215507736134:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:20.433968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:20.434228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789215507736214:2108] [node 4] ICS07 socket disconnect 442 reason# EndOfStream
2025-11-28T14:52:20.434271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789215507736214:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.434379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:20.434429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789215507736214:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:20.434535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:20.434814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789215701149577:2107] [node 4] ICS01 socket: 483 reason# EndOfStream
2025-11-28T14:52:20.434870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215701149539:2107] [node 4] ICP30 unregister session Session# [6:7577789215701149577:2107] VirtualId# [6:7577789215701149574:0]
2025-11-28T14:52:20.434900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789215701149577:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.434969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:20.435019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:20.435279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789215507736196:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30033) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:20.435375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789215507736214:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:20.435410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789215507736196:2108] [node 4] ICP30 unregister session Session# [5:7577789215507736214:2108] VirtualId# [5:7577789215507736212:0]
2025-11-28T14:52:20.435450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:20.435488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789215507736196:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30033) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.435498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30033) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.475936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:20.476173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789215701149558:2106] [node 5] ICS07 socket disconnect 488 reason# EndOfStream
2025-11-28T14:52:20.476234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789215701149558:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:20.476311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:20.476358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789215701149558:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:20.477139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215701149536:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10750) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:20.477221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789215701149558:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:20.477272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215701149536:2106] [node 5] ICP30 unregister session Session# [6:7577789215701149558:2106] VirtualId# [6:7577789215701149555:0]
2025-11-28T14:52:20.477324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:20.477355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789215701149536:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.477367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:20.486838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789215701149536:2106] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:20.586698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789214500745624:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:20.586711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789214500745625:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:20.605904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789217199542428:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:20.605935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789217199542429:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:20.625855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789216992303421:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:20.626280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789216992303422:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:20.646879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789214938238838:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:20.646930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789214938238840:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:20.646997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789214938238839:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:20.668191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215699515338:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:20.668251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215699515339:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:20.668358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215699515340:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:20.690984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215694509014:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:20.691067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215694509013:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:20.691189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215694509015:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:20.770767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215694509072:2104] [node 5] ICP01 ready to work
2025-11-28T14:52:20.783797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789214938238902:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:20.784423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215699515402:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:20.784460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789215694509076:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:20.784418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789217199542492:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:20.785329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789214500745688:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:20.785297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789217199542493:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:20.785789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789214500745689:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:20.785707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789215699515403:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:20.785944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789214938238903:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:20.785957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789216992303484:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:20.785991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789214500745690:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:20.786200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789216992303485:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:20.786442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789217199542494:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:20.786609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789216992303486:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:20.788406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789216992303422:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(35045321ns) {Configure:104(35036987ns) {ProcessConfigured:135(99054ns) {StartInitialHandshake:157(93667ns) {DropHandshakes:470(2184ns) {DropIncomingHandshake:442(778ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(847ns) PrepareNewSessionHandshake:477(1434ns) {SwitchToState:179(699ns)}}}}}
2025-11-28T14:52:20.789140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789214938238839:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30055503ns) {Configure:104(30046306ns) {ProcessConfigured:135(169438ns) {StartInitialHandshake:157(165129ns) {DropHandshakes:470(3548ns) {DropIncomingHandshake:442(2730ns) DropOutgoingHandshake:457(160ns)} GenerateSessionVirtualId:988(1099ns) PrepareNewSessionHandshake:477(818ns) {SwitchToState:179(304ns)}}}}}
2025-11-28T14:52:20.789301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789214938238838:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40083866ns) {Configure:104(40075145ns) {ProcessConfigured:135(326731ns) {StartInitialHandshake:157(324403ns) {DropHandshakes:470(1016ns) {DropIncomingHandshake:442(332ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:477(1311ns) {SwitchToState:179(473ns)}}}}}
2025-11-28T14:52:20.790215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789214500745624:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44017429ns) {Configure:104(44009780ns) {ProcessConfigured:135(258544ns) {StartInitialHandshake:157(256055ns) {DropHandshakes:470(1502ns) {DropIncomingHandshake:442(503ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(982ns) PrepareNewSessionHandshake:477(1294ns) {SwitchToState:179(563ns)}}}}}
2025-11-28T14:52:20.791701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789215699515338:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32324823ns) {Configure:104(32317774ns) {ProcessConfigured:135(530794ns) {StartInitialHandshake:157(529289ns) {DropHandshakes:470(4454ns) {DropIncomingHandshake:442(372ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(671ns) {SwitchToState:179(267ns)}}}}}
2025-11-28T14:52:20.790790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789214500745625:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(46615449ns) {Configure:104(46604671ns) {ProcessConfigured:135(1025769ns) {StartInitialHandshake:157(1023592ns) {DropHandshakes:470(1551ns) {DropIncomingHandshake:442(445ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(509ns) PrepareNewSessionHandshake:477(761ns) {SwitchToState:179(278ns)}}}}}
2025-11-28T14:52:20.792444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789215694509013:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38892121ns) {Configure:104(38886557ns) {ProcessConfigured:135(578845ns) {StartInitialHandshake:157(577617ns) {DropHandshakes:470(999ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:477(714ns) {SwitchToState:179(305ns)}}}}}
2025-11-28T14:52:20.792329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789215699515339:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(37319024ns) {Configure:104(37314241ns) {ProcessConfigured:135(638293ns) {StartInitialHandshake:157(636235ns) {DropHandshakes:470(1361ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(4558ns) {SwitchToState:179(3897ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.796183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789214500745701:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21070
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.796735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789214938238915:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61993
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.797675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215699515412:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.800314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789217199542503:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12829
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:20.803131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214500745701:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.803537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789217199542504:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12829
2025-11-28T14:52:20.802295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789216992303499:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-28T14:52:20.804211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215699515416:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.804198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789217199542505:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12829
2025-11-28T14:52:20.805229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215699515412:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.805174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789214500745705:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21070
2025-11-28T14:52:20.805428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789214938238918:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61993
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.805576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789215699515340:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.803699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789216992303500:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 2025-11-28T14:52:20.805568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789214938238915:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:20.804627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789217199542503:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.805239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789216992303500:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.805909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214500745705:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.805572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789217199542504:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.02025-11-28T14:52:20.805684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789217199542505:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.805573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789216992303499:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.805984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215699515418:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-28T14:52:20.805935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789217199542428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.805977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789214938238840:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.805886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789216992303502:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-28T14:52:20.805944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789216992303421:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.805939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789217199542512:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12829
2025-11-28T14:52:20.806075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789214938238919:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61993
2025-11-28T14:52:20.806002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789217199542511:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12829
2025-11-28T14:52:20.806224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215699515416:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.806141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789214938238918:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.806470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214500745694:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.806603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789214938238838:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.806366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789217199542512:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.806052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789216992303503:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-28T14:52:20.806504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214500745694:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.806555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789217199542511:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.806514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214500745693:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.806083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789216992303494:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.806105Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789216992303494:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.806544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214500745693:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.806110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789216992303504:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-28T14:52:20.806339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789216992303502:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.809470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214500745697:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.806862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789216992303490:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.809489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214500745697:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.806886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789216992303490:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.817196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214500745697:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.817746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214500745694:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:20.817115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789216992303494:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.820522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789214938238922:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61993
2025-11-28T14:52:20.817759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789216992303490:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:20.819917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789214500745688:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.839246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215699515422:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-28T14:52:20.819835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789216992303484:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.847947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789214938238923:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61993
2025-11-28T14:52:20.820475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789214500745625:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.847717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789217199542518:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12829
2025-11-28T14:52:20.820474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789216992303486:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.820483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214500745693:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:20.831516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:20.820563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.831560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.848426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789214938238915:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:20.820570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:20.831728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789216992303486:2108] [node 5] ICP22 created new session: [3:7577789216992303511:2108]
2025-11-28T14:52:20.820626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789214500745625:2049] [node 3] ICP22 created new session: [1:7577789214500745712:2049]
2025-11-28T14:52:20.850456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789214938238840:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789214938238915:2115] held: no
2025-11-28T14:52:20.849377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215699515412:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:20.831597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789214500745688:2107] [node 4] ICP22 created new session: [1:7577789214500745711:2107]
2025-11-28T14:52:20.831734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789216992303484:2107] [node 4] ICP22 created new session: [3:7577789216992303512:2107]
2025-11-28T14:52:20.872421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789217199542505:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.859189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789215699515340:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.839270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214500745712:2049] [node 3] ICS09 handshake done sender: [1:7577789214500745694:2112] self: [1:7577789214500745691:0] peer: [3:7577789216992303505:0] socket: 457 qp: -1
2025-11-28T14:52:20.866543Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789214938238838:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789214938238918:2117] held: no
2025-11-28T14:52:20.846815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789216992303512:2107] [node 4] ICS09 handshake done sender: [3:7577789216992303490:2111] self: [3:7577789216992303489:0] peer: [4:7577789214938238920:0] socket: 446 qp: -1
2025-11-28T14:52:20.846532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214500745711:2107] [node 4] ICS09 handshake done sender: [1:7577789214500745697:2113] self: [1:7577789214500745696:0] peer: [4:7577789214938238921:0] socket: 464 qp: -1
2025-11-28T14:52:20.872704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789214938238918:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.874548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:20.874936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789217199542428:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.847747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789216992303512:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:20.874649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215699515340:2050] [node 3] ICP22 created new session: [5:7577789215699515423:2050]
2025-11-28T14:52:20.874985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:20.874761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789214938238838:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.846819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214500745711:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:20.847850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789216992303511:2108] [node 5] ICS09 handshake done sender: [3:7577789216992303494:2113] self: [3:7577789216992303492:0] peer: [5:7577789215699515420:0] socket: 445 qp: -1
2025-11-28T14:52:20.875052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789214938238840:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.875065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789215699515423:2050] [node 3] ICS09 handshake done sender: [5:7577789215699515412:2115] self: [5:7577789215699515420:0] peer: [3:7577789216992303492:0] socket: 452 qp: -1
2025-11-28T14:52:20.846842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214500745712:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.882798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789217199542428:2048] [node 1] ICP22 created new session: [2:7577789217199542519:2048]
2025-11-28T14:52:20.847898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:20.875093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:20.890569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789217199542519:2048] [node 1] ICS09 handshake done sender: [2:7577789217199542505:2117] self: [2:7577789217199542515:0] peer: [1:7577789214500745692:0] socket: 465 qp: -1
2025-11-28T14:52:20.847763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:20.847976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789216992303511:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:20.875123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789215699515423:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:20.875231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789214938238840:2050] [node 3] ICP22 created new session: [4:7577789214938238927:2050]
2025-11-28T14:52:20.847883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:20.875431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:20.890598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789217199542519:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.848017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:20.848305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789214500745624:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.875248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: 2025-11-28T14:52:20.897458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
::ffff:127.1.0.0
2025-11-28T14:52:20.882758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789214938238927:2050] [node 3] ICS09 handshake done sender: [4:7577789214938238915:2115] self: [4:7577789214938238920:0] peer: [3:7577789216992303489:0] socket: 451 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.848421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789216992303510:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.866332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789216992303500:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.882816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789214938238838:2048] [node 1] ICP22 created new session: [4:7577789214938238926:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.883135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789216992303484:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(30625328ns) {HandleHandshakeStatus:329(30622492ns) {DropIncomingHandshake:442(15548ns) DropOutgoingHandshake:457(790ns) SwitchToState:179(753ns) ProcessPendingSessionEvents:495(2336013ns) {ForwardSessionEventToSession:612(381222ns) ForwardSessionEventToSession:612(75860ns) ForwardSessionEventToSession:612(2775ns) ForwardSessionEventToSession:612(633447ns) ForwardSessionEventToSession:612(4141ns) ForwardSessionEventToSession:612(3029ns) ForwardSessionEventToSession:612(147016ns) ForwardSessionEventToSession:612(2986ns) ForwardSessionEventToSession:612(152355ns) ForwardSessionEventToSession:612(920519ns)}}}
2025-11-28T14:52:20.848349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:20.901545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215699515424:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-28T14:52:20.899981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215694509091:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:20.883160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789214938238926:2048] [node 1] ICS09 handshake done sender: [4:7577789214938238918:2117] self: [4:7577789214938238921:0] peer: [1:7577789214500745696:0] socket: 466 qp: -1
2025-11-28T14:52:20.848415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789214500745624:2048] [node 2] ICP22 created new session: [1:7577789214500745715:2048]
2025-11-28T14:52:20.883143Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789216992303486:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(46043267ns) {HandleHandshakeStatus:329(46041232ns) {DropIncomingHandshake:442(376ns) DropOutgoingHandshake:457(385ns) SwitchToState:179(1155ns) ProcessPendingSessionEvents:495(17118347ns) {ForwardSessionEventToSession:612(9233105ns) ForwardSessionEventToSession:612(6004ns) ForwardSessionEventToSession:612(2515ns) ForwardSessionEventToSession:612(498051ns) ForwardSessionEventToSession:612(5665ns) ForwardSessionEventToSession:612(3113ns) ForwardSessionEventToSession:612(7167546ns) ForwardSessionEventToSession:612(2432ns) ForwardSessionEventToSession:612(178190ns) ForwardSessionEventToSession:612(3676ns)}}}
2025-11-28T14:52:20.883189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789214938238926:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.901791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215694509092:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:20.901810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215699515426:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-28T14:52:20.873633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214500745715:2048] [node 2] ICS09 handshake done sender: [1:7577789214500745693:2111] self: [1:7577789214500745692:0] peer: [2:7577789217199542515:0] socket: 458 qp: -1
2025-11-28T14:52:20.890138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789214938238927:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:20.890037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789216992303421:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.901804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215694509093:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:20.902466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789215699515340:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(40789553ns) {HandleHandshakeStatus:329(40786182ns) {DropIncomingHandshake:442(907ns) DropOutgoingHandshake:457(4900ns) SwitchToState:179(672ns) ProcessPendingSessionEvents:495(2741821ns) {ForwardSessionEventToSession:612(1798626ns) ForwardSessionEventToSession:612(161741ns) ForwardSessionEventToSession:612(4567ns) ForwardSessionEventToSession:612(6969ns) ForwardSessionEventToSession:612(5589ns) ForwardSessionEventToSession:612(3262ns) ForwardSessionEventToSession:612(3581ns) ForwardSessionEventToSession:612(155406ns) ForwardSessionEventToSession:612(4452ns) ForwardSessionEventToSession:612(580854ns)}}}
2025-11-28T14:52:20.873728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789214500745625:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(45978956ns) {HandleHandshakeStatus:329(45977027ns) {DropIncomingHandshake:442(602ns) DropOutgoingHandshake:457(744ns) SwitchToState:179(182ns) ProcessPendingSessionEvents:495(17851179ns) {ForwardSessionEventToSession:612(804444ns) ForwardSessionEventToSession:612(117434ns) ForwardSessionEventToSession:612(1067779ns) ForwardSessionEventToSession:612(4225ns) ForwardSessionEventToSession:612(2227ns) ForwardSessionEventToSession:612(8374967ns) ForwardSessionEventToSession:612(2295ns) ForwardSessionEventToSession:612(255628ns) ForwardSessionEventToSession:612(3659ns) ForwardSessionEventToSession:612(7176273ns)}}}
2025-11-28T14:52:20.903019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215699515426:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.898842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:20.874469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214500745715:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:20.903367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789215699515402:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.890203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:20.901987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789215694509091:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.899175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789216992303421:2048] [node 1] ICP22 created new session: [3:7577789216992303517:2048]
2025-11-28T14:52:20.903977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789217199542428:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27677098ns) {HandleHandshakeStatus:329(27675281ns) {DropIncomingHandshake:442(362ns) DropOutgoingHandshake:457(3371ns) SwitchToState:179(566ns) ProcessPendingSessionEvents:495(2526798ns) {ForwardSessionEventToSession:612(1262068ns) ForwardSessionEventToSession:612(292710ns) ForwardSessionEventToSession:612(3899ns) ForwardSessionEventToSession:612(239590ns) ForwardSessionEventToSession:612(3879ns) ForwardSessionEventToSession:612(3415ns) ForwardSessionEventToSession:612(267719ns) ForwardSessionEventToSession:612(6563ns) ForwardSessionEventToSession:612(2613ns) ForwardSessionEventToSession:612(256015ns)}}}
2025-11-28T14:52:20.890225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:20.875199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:20.902399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789215694509093:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.900010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789216992303517:2048] [node 1] ICS09 handshake done sender: [3:7577789216992303500:2115] self: [3:7577789216992303505:0] peer: [1:7577789214500745691:0] socket: 459 qp: -1
2025-11-28T14:52:20.904785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215699515424:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.899331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789214938238838:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24167973ns) {HandleHandshakeStatus:329(24165800ns) {DropIncomingHandshake:442(765ns) DropOutgoingHandshake:457(312ns) SwitchToState:179(498ns) ProcessPendingSessionEvents:495(1072ns)}}
2025-11-28T14:52:20.902687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215694509015:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.906188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789215699515338:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.875276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789214500745688:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(46413561ns) {HandleHandshakeStatus:329(46410549ns) {DropIncomingHandshake:442(15272ns) DropOutgoingHandshake:457(604ns) SwitchToState:179(618ns) ProcessPendingSessionEvents:495(17876925ns) {ForwardSessionEventToSession:612(102371ns) ForwardSessionEventToSession:612(812315ns) ForwardSessionEventToSession:612(3037ns) ForwardSessionEventToSession:612(1837ns) ForwardSessionEventToSession:612(153168ns) ForwardSessionEventToSession:612(918386ns) ForwardSessionEventToSession:612(8395908ns) ForwardSessionEventToSession:612(254938ns) ForwardSessionEventToSession:612(2196ns) ForwardSessionEventToSession:612(7213604ns)}}}
2025-11-28T14:52:20.900051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789216992303517:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.903445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215694509076:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.900665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:20.899982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789214938238840:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(22930814ns) {HandleHandshakeStatus:329(22928620ns) {DropIncomingHandshake:442(1040ns) DropOutgoingHandshake:457(178ns) SwitchToState:179(2927ns) ProcessPendingSessionEvents:495(1334ns)}}
2025-11-28T14:52:20.903572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789215694509092:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.902379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789214500745718:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21070
2025-11-28T14:52:20.903696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789216992303501:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.903953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215694509013:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.902623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789214500745624:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(51668713ns) {HandleHandshakeStatus:329(51665690ns) {DropIncomingHandshake:442(518ns) DropOutgoingHandshake:457(981ns) SwitchToState:179(4380ns) ProcessPendingSessionEvents:495(9377072ns) {ForwardSessionEventToSession:612(8370366ns) ForwardSessionEventToSession:612(178203ns) ForwardSessionEventToSession:612(3549ns) ForwardSessionEventToSession:612(4555ns) ForwardSessionEventToSession:612(140322ns) ForwardSessionEventToSession:612(565605ns) ForwardSessionEventToSession:612(4868ns) ForwardSessionEventToSession:612(2060ns) ForwardSessionEventToSession:612(1749ns) ForwardSessionEventToSession:612(3913ns)}}}
2025-11-28T14:52:20.904092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789214938238917:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.903721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789216992303501:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.903499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789214500745625:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:20.904109Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789214938238917:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.904266Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789216992303421:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20330489ns) {HandleHandshakeStatus:329(20327759ns) {DropIncomingHandshake:442(2669ns) DropOutgoingHandshake:457(9955ns) SwitchToState:179(644ns) ProcessPendingSessionEvents:495(998198ns) {ForwardSessionEventToSession:612(441963ns) ForwardSessionEventToSession:612(3650ns) ForwardSessionEventToSession:612(23606ns) ForwardSessionEventToSession:612(175329ns) ForwardSessionEventToSession:612(2776ns) ForwardSessionEventToSession:612(90836ns) ForwardSessionEventToSession:612(2747ns) ForwardSessionEventToSession:612(107105ns) ForwardSessionEventToSession:612(134863ns) ForwardSessionEventToSession:612(6029ns)}}}
2025-11-28T14:52:20.905900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214500745704:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.909565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215694509097:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:20.909522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215699515429:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-28T14:52:20.904201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789214938238911:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.907756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789216992303501:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:20.905933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214500745704:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.910737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215699515426:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:20.904223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789214938238911:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.908043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789214500745698:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.908691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789216992303485:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.908743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: 2025-11-28T14:52:20.908052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214500745704:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:20.906870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789214938238917:2116] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:20.909127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789216992303485:2109] [node 6] ICP22 created new session: [3:7577789216992303520:2109]
2025-11-28T14:52:20.912212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215694509098:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:20.907555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789214938238911:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:20.908070Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789214500745698:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.909584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789216992303520:2109] [node 6] ICS09 handshake done sender: [3:7577789216992303501:2116] self: [3:7577789216992303498:0] peer: [6:7577789215694509094:0] socket: 448 qp: -1
2025-11-28T14:52:20.908167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789214938238903:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.909813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789216992303520:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:20.910077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789214500745690:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.909087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789214938238902:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.910108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:20.912778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789215699515402:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.909753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:20.910340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:20.913149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215694509099:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:20.910648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789214500745690:2110] [node 6] ICP22 created new session: [1:7577789214500745722:2110]
2025-11-28T14:52:20.909966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789214938238903:2109] [node 6] ICP22 created new session: [4:7577789214938238932:2109]
2025-11-28T14:52:20.913597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.911383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214500745722:2110] [node 6] ICS09 handshake done sender: [1:7577789214500745704:2116] self: [1:7577789214500745703:0] peer: [6:7577789215694509096:0] socket: 498 qp: -1
2025-11-28T14:52:20.914161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215699515402:2108] [node 4] ICP22 created new session: [5:7577789215699515431:2108]
2025-11-28T14:52:20.909969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:20.911389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789214500745698:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:20.914700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789215699515431:2108] [node 4] ICS09 handshake done sender: [5:7577789215699515426:2121] self: [5:7577789215699515427:0] peer: [4:7577789214938238910:0] socket: 502 qp: -1
2025-11-28T14:52:20.913466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215694509015:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789215694509091:2115] held: no
2025-11-28T14:52:20.910158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789214938238902:2108] [node 5] ICP22 created new session: [4:7577789214938238933:2108]
2025-11-28T14:52:20.911589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214500745722:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:20.914892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789215699515431:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:20.914945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:20.913811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789215694509093:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:20.911619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:20.910711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789214938238933:2108] [node 5] ICS09 handshake done sender: [4:7577789214938238911:2114] self: [4:7577789214938238910:0] peer: [5:7577789215699515427:0] socket: 500 qp: -1
2025-11-28T14:52:20.910861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789214938238933:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:20.914802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215694509076:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.912787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789214500745689:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.914850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.912868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:20.910933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789214938238932:2109] [node 6] ICS09 handshake done sender: [4:7577789214938238917:2116] self: [4:7577789214938238916:0] peer: [6:7577789215694509095:0] socket: 501 qp: -1
2025-11-28T14:52:20.914861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789215694509091:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:20.914878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789215694509092:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.913472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789214500745689:2109] [node 5] ICP22 created new session: [1:7577789214500745724:2109]
2025-11-28T14:52:20.911275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789214938238932:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:20.914892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215694509076:2106] [node 4] ICP22 created new session: [6:7577789215694509100:2106]
2025-11-28T14:52:20.914936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789215694509100:2106] [node 4] ICS09 handshake done sender: [6:7577789215694509093:2117] self: [6:7577789215694509095:0] peer: [4:7577789214938238916:0] socket: 503 qp: -1
2025-11-28T14:52:20.914290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789214500745724:2109] [node 5] ICS09 handshake done sender: [1:7577789214500745698:2114] self: [1:7577789214500745695:0] peer: [5:7577789215699515428:0] socket: 453 qp: -1
2025-11-28T14:52:20.911293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:20.911836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:20.914529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789214500745724:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:20.915044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789215694509100:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:20.915062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: 2025-11-28T14:52:20.914589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.0
2025-11-28T14:52:20.916252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215694509015:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.916395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:20.916405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215694509013:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.918167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789215699515430:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-28T14:52:20.916438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215694509015:2050] [node 3] ICP22 created new session: [6:7577789215694509103:2050]
2025-11-28T14:52:20.916929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:20.916936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789215694509103:2050] [node 3] ICS09 handshake done sender: [6:7577789215694509091:2115] self: [6:7577789215694509094:0] peer: [3:7577789216992303498:0] socket: 496 qp: -1
2025-11-28T14:52:20.917084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789215694509103:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:20.917132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.917152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215694509013:2048] [node 1] ICP22 created new session: [6:7577789215694509105:2048]
2025-11-28T14:52:20.917434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789215694509105:2048] [node 1] ICS09 handshake done sender: [6:7577789215694509092:2116] self: [6:7577789215694509096:0] peer: [1:7577789214500745703:0] socket: 499 qp: -1
2025-11-28T14:52:20.917474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789215694509105:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.918924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789215699515338:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789215699515424:2120] held: no
2025-11-28T14:52:20.917671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:20.919032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789214500745726:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21070
2025-11-28T14:52:20.918896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789215694509104:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:20.919257Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215699515424:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:20.919435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789215699515338:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.919575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:20.919712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215699515338:2048] [node 1] ICP22 created new session: [5:7577789215699515434:2048]
2025-11-28T14:52:20.919946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789215699515434:2048] [node 1] ICS09 handshake done sender: [5:7577789215699515424:2120] self: [5:7577789215699515428:0] peer: [1:7577789214500745695:0] socket: 497 qp: -1
2025-11-28T14:52:20.919978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789215699515434:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:20.920004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:20.920451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789216992303503:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.938995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789216992303486:2108] [node 5] ICP18 handshake (actor [3:7577789216992303503:2118]) is too old
2025-11-28T14:52:20.942029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [3:7577789216992303486:2108] [node 5] ICP35 event processing took too much time StateWork:294(18447425ns) {IncomingHandshake:261(8515349ns)}
2025-11-28T14:52:20.948585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215699515418:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.948863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789215699515339:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.948892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789214938238919:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.949110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789214938238839:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.949143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789217199542499:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.949296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789216992303504:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.949183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789217199542499:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.949407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789217199542501:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.949423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789217199542501:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.949823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789216992303422:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.950037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789217199542500:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.950072Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789217199542500:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.950500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789217199542499:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:20.950613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789215694509104:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.950737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789217199542501:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:20.950925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789217199542493:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.951024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215694509014:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:20.951054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:20.951278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789217199542493:2108] [node 5] ICP22 created new session: [2:7577789217199542524:2108]
2025-11-28T14:52:20.951301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789217199542492:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.951349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:20.951350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789217199542524:2108] [node 5] ICS09 handshake done sender: [2:7577789217199542499:2111] self: [2:7577789217199542498:0] peer: [5:7577789215699515436:0] socket: 479 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.951462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789217199542524:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:20.951489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789215699515339:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789215699515418:2117] held: no
2025-11-28T14:52:20.951489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:20.951491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789217199542492:2107] [node 4] ICP22 created new session: [2:7577789217199542525:2107]
2025-11-28T14:52:20.951549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789217199542507:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:20.951564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789217199542507:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:20.951611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789214938238937:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61993
2025-11-28T14:52:20.951569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789217199542525:2107] [node 4] ICS09 handshake done sender: [2:7577789217199542501:2113] self: [2:7577789217199542497:0] peer: [4:7577789214938238936:0] socket: 481 qp: -1
2025-11-28T14:52:20.951598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789217199542525:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:20.952312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:20.952891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789217199542507:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:20.952892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789214938238839:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789214938238919:2118] held: no
2025-11-28T14:52:20.953309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789217199542500:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:20.954015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789217199542494:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.954426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789216992303523:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63813
2025-11-28T14:52:20.954526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789217199542429:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:20.954640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789214938238919:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:20.954576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:20.954614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:20.954674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789217199542429:2049] [node 3] ICP22 created new session: [2:7577789217199542530:2049]
2025-11-28T14:52:20.954793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789217199542530:2049] [node 3] ICS09 handshake done sender: [2:7577789217199542500:2112] self: [2:7577789217199542496:0] peer: [3:7577789216992303522:0] socket: 483 qp: -1
2025-11-28T14:52:20.955020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789216992303504:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:20.954803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789217199542494:2109] [node 6] ICP22 created new session: [2:7577789217199542531:2109]
2025-11-28T14:52:20.955218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215694509014:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789215694509104:2121] held: no
2025-11-28T14:52:20.954995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789214938238839:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.956133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:20.954818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789217199542530:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:20.956095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789216992303422:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789216992303504:2119] held: no
2025-11-28T14:52:20.956560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789214938238839:2049] [node 2] ICP22 created new session: [4:7577789214938238938:2049]
2025-11-28T14:52:20.954828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789217199542531:2109] [node 6] ICS09 handshake done sender: [2:7577789217199542507:2118] self: [2:7577789217199542506:0] peer: [6:7577789215694509108:0] socket: 462 qp: -1
2025-11-28T14:52:20.956753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789214938238938:2049] [node 2] ICS09 handshake done sender: [4:7577789214938238919:2118] self: [4:7577789214938238936:0] peer: [2:7577789217199542497:0] socket: 482 qp: -1
2025-11-28T14:52:20.956759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789216992303422:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:20.956796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789214938238938:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:20.954847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789217199542531:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:20.954849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:20.956838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:20.956851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:20.954949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:20.956885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789216992303422:2049] [node 2] ICP22 created new session: [3:7577789216992303524:2049]
2025-11-28T14:52:20.956919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789216992303524:2049] [node 2] ICS09 handshake done sender: [3:7577789216992303504:2119] self: [3:7577789216992303522:0] peer: [2:7577789217199542496:0] socket: 484 qp: -1
2025-11-28T14:52:20.956943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789216992303524:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:20.956976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.020875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789219989476406:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:21.020905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789219989476405:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:21.020901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789219994482733:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-28T14:52:21.021062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789215694509104:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:21.021136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215699515418:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:21.021307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215694509014:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.021461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789215699515339:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.021610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:21.021566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789219989476405:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.021575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:21.021639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215694509014:2049] [node 2] ICP22 created new session: [6:7577789219989476407:2049]
2025-11-28T14:52:21.021690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215699515339:2049] [node 2] ICP22 created new session: [5:7577789219994482734:2049]
2025-11-28T14:52:21.021709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789219989476407:2049] [node 2] ICS09 handshake done sender: [6:7577789215694509104:2121] self: [6:7577789215694509108:0] peer: [2:7577789217199542506:0] socket: 472 qp: -1
2025-11-28T14:52:21.021749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789215694509072:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.021771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789219994482734:2049] [node 2] ICS09 handshake done sender: [5:7577789215699515418:2117] self: [5:7577789215699515436:0] peer: [2:7577789217199542498:0] socket: 480 qp: -1
2025-11-28T14:52:21.021758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789219989476407:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:21.021809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789219994482734:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:21.021802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:21.021877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:21.021920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789215699515419:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.021938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789215699515419:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.022811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789215699515419:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.022952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789215699515403:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.023090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789219989476410:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17492
2025-11-28T14:52:21.023046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:21.023115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789215699515403:2110] [node 6] ICP22 created new session: [5:7577789219994482737:2110]
2025-11-28T14:52:21.023183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789219994482737:2110] [node 6] ICS09 handshake done sender: [5:7577789215699515419:2118] self: [5:7577789215699515417:0] peer: [6:7577789219989476408:0] socket: 475 qp: -1
2025-11-28T14:52:21.023264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789219994482737:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:21.023293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:21.023325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215694509072:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789219989476405:2122] held: no
2025-11-28T14:52:21.023447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789219989476405:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:21.023700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789215694509072:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.024278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:21.024338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789215694509072:2104] [node 5] ICP22 created new session: [6:7577789219989476411:2104]
2025-11-28T14:52:21.024433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789219989476411:2104] [node 5] ICS09 handshake done sender: [6:7577789219989476405:2122] self: [6:7577789219989476408:0] peer: [5:7577789215699515417:0] socket: 447 qp: -1
2025-11-28T14:52:21.024464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789219989476411:2104] [node 5] ICS10 traffic start
2025-11-28T14:52:21.024514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:21.048209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:21.048407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789217199542519:2048] [node 1] ICS07 socket disconnect 465 reason# EndOfStream
2025-11-28T14:52:21.048431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789217199542519:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.048517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.048571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789217199542519:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:21.049207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:21.049423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789216992303517:2048] [node 1] ICS07 socket disconnect 459 reason# EndOfStream
2025-11-28T14:52:21.049462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789216992303517:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.049550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.049611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789216992303517:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:21.050483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789217199542428:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:21.050604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789216992303421:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.050523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789217199542519:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:21.051033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789216992303517:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.051063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789216992303421:2048] [node 1] ICP30 unregister session Session# [3:7577789216992303517:2048] VirtualId# [3:7577789216992303505:0]
2025-11-28T14:52:21.051111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.052074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789216992303421:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.052085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.052303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:21.052248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789217199542428:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.052283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789217199542519:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.052302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789217199542428:2048] [node 1] ICP30 unregister session Session# [2:7577789217199542519:2048] VirtualId# [2:7577789217199542515:0]
2025-11-28T14:52:21.052345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.052387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789217199542428:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.052650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789214938238926:2048] [node 1] ICS01 socket: 466 reason# EndOfStream
2025-11-28T14:52:21.052402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.052725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789214938238838:2048] [node 1] ICP30 unregister session Session# [4:7577789214938238926:2048] VirtualId# [4:7577789214938238921:0]
2025-11-28T14:52:21.052753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789214938238926:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.052811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.052848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.053338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:21.053525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789215699515434:2048] [node 1] ICS01 socket: 497 reason# EndOfStream
2025-11-28T14:52:21.053607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789215699515338:2048] [node 1] ICP30 unregister session Session# [5:7577789215699515434:2048] VirtualId# [5:7577789215699515428:0]
2025-11-28T14:52:21.053637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789215699515434:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.053711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.053789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.053917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:21.054131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789215694509105:2048] [node 1] ICS07 socket disconnect 499 reason# EndOfStream
2025-11-28T14:52:21.054173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789215694509105:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.054244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.054297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789215694509105:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:21.054935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215694509013:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.055004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789215694509105:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.055047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215694509013:2048] [node 1] ICP30 unregister session Session# [6:7577789215694509105:2048] VirtualId# [6:7577789215694509096:0]
2025-11-28T14:52:21.055104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.055138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789215694509013:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.055151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20050) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.062279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789216992303421:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:21.062728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789217199542428:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:21.065382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789215694509013:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:21.119686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:21.119923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789216992303524:2049] [node 2] ICS01 socket: 484 reason# EndOfStream
2025-11-28T14:52:21.119969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789216992303422:2049] [node 2] ICP30 unregister session Session# [3:7577789216992303524:2049] VirtualId# [3:7577789216992303522:0]
2025-11-28T14:52:21.120023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789216992303524:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.120124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:21.120192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:21.121836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:21.122060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789214938238938:2049] [node 2] ICS01 socket: 482 reason# EndOfStream
2025-11-28T14:52:21.122135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789214938238839:2049] [node 2] ICP30 unregister session Session# [4:7577789214938238938:2049] VirtualId# [4:7577789214938238936:0]
2025-11-28T14:52:21.122173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789214938238938:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.122245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:21.122296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:21.122478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:21.122606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789219994482734:2049] [node 2] ICS01 socket: 480 reason# EndOfStream
2025-11-28T14:52:21.122637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789215699515339:2049] [node 2] ICP30 unregister session Session# [5:7577789219994482734:2049] VirtualId# [5:7577789215699515436:0]
2025-11-28T14:52:21.122654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789219994482734:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.122697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:21.122727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:21.123260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:21.123399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789219989476407:2049] [node 2] ICS01 socket: 472 reason# EndOfStream
2025-11-28T14:52:21.123435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215694509014:2049] [node 2] ICP30 unregister session Session# [6:7577789219989476407:2049] VirtualId# [6:7577789215694509108:0]
2025-11-28T14:52:21.123465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789219989476407:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.123547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:21.123587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:21.171720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:21.171880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789214938238927:2050] [node 3] ICS01 socket: 451 reason# EndOfStream
2025-11-28T14:52:21.171913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789214938238840:2050] [node 3] ICP30 unregister session Session# [4:7577789214938238927:2050] VirtualId# [4:7577789214938238920:0]
2025-11-28T14:52:21.171934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789214938238927:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.171987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:21.172043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:21.172408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:21.172606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789215699515423:2050] [node 3] ICS07 socket disconnect 452 reason# EndOfStream
2025-11-28T14:52:21.172637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789215699515423:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.172728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:21.172772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789215699515423:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:21.173147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:21.173297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789215694509103:2050] [node 3] ICS01 socket: 496 reason# EndOfStream
2025-11-28T14:52:21.173325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215694509015:2050] [node 3] ICP30 unregister session Session# [6:7577789215694509103:2050] VirtualId# [6:7577789215694509094:0]
2025-11-28T14:52:21.173348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789215694509103:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.173429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:21.173475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:21.173371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789215699515340:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.173493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789215699515423:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.173578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789215699515340:2050] [node 3] ICP30 unregister session Session# [5:7577789215699515423:2050] VirtualId# [5:7577789215699515420:0]
2025-11-28T14:52:21.173619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:21.173662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789215699515340:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.173676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.183152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789215699515340:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:21.209396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:21.209615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789215699515431:2108] [node 4] ICS07 socket disconnect 502 reason# EndOfStream
2025-11-28T14:52:21.209649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789215699515431:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.209732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:21.209774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789215699515431:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:21.210537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789215699515402:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.210601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789215699515431:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.210639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789215699515402:2108] [node 4] ICP30 unregister session Session# [5:7577789215699515431:2108] VirtualId# [5:7577789215699515427:0]
2025-11-28T14:52:21.210692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:21.210748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789215699515402:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.210768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.210849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:21.211037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789215694509100:2106] [node 4] ICS07 socket disconnect 503 reason# EndOfStream
2025-11-28T14:52:21.211079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789215694509100:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.211142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:21.211188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789215694509100:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:21.211821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789215694509076:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.211913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789215694509100:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.211952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215694509076:2106] [node 4] ICP30 unregister session Session# [6:7577789215694509100:2106] VirtualId# [6:7577789215694509095:0]
2025-11-28T14:52:21.211997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:21.212028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789215694509076:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.212044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.221083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789215699515402:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:21.222048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789215694509076:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:21.281351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:21.281519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789219989476411:2104] [node 5] ICS01 socket: 447 reason# EndOfStream
2025-11-28T14:52:21.281563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789215694509072:2104] [node 5] ICP30 unregister session Session# [6:7577789219989476411:2104] VirtualId# [6:7577789219989476408:0]
2025-11-28T14:52:21.281593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789219989476411:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.281658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:21.281700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:21.412165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789220450164917:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:21.412277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789220450164918:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:21.433199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789221047182333:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:21.433280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789221047182334:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:21.452691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789220538656290:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:21.452704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789220538656291:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:21.478014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789217989041735:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:21.478074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789217989041736:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:21.478597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789217989041737:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:21.505476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789221672848019:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:21.505479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789221672848018:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:21.505529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789221672848020:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:21.531698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789220200392863:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:21.531751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789220200392864:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:21.531788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789220200392865:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:21.595035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789220200392920:2103] [node 5] ICP01 ready to work
2025-11-28T14:52:21.621627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789220450164980:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:21.627855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789221047182396:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:21.633158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789220450164981:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:21.633628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789220538656353:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:21.633905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789217989041798:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:21.634249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789221672848082:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:21.634416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789220538656354:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:21.634162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789220450164982:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:21.634412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789221672848081:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:21.635286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789217989041799:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:21.635108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789220538656355:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:21.636546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789221047182397:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:21.636554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789221047182398:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:21.636567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789220200392936:2105] [node 4] ICP01 ready to work
2025-11-28T14:52:21.639462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789220200392920:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(44056089ns) {Configure:104(44040887ns) {ProcessConfigured:135(111387ns) {StartInitialHandshake:157(108148ns) {DropHandshakes:470(2842ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(520ns)} GenerateSessionVirtualId:988(823ns) PrepareNewSessionHandshake:477(1230ns) {SwitchToState:179(426ns)}}}}}
2025-11-28T14:52:21.640635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789220200392863:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27285824ns) {Configure:104(27277582ns) {ProcessConfigured:135(260435ns) {StartInitialHandshake:157(258555ns) {DropHandshakes:470(1218ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:477(1043ns) {SwitchToState:179(375ns)}}}}}
2025-11-28T14:52:21.641916Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789217989041736:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33807721ns) {Configure:104(33802774ns) {ProcessConfigured:135(255974ns) {StartInitialHandshake:157(253997ns) {DropHandshakes:470(947ns) {DropIncomingHandshake:442(332ns) DropOutgoingHandshake:457(151ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(1268ns) {SwitchToState:179(540ns)}}}}}
2025-11-28T14:52:21.642166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789217989041735:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39926415ns) {Configure:104(27969999ns) {ProcessConfigured:135(235880ns) {StartInitialHandshake:157(234584ns) {DropHandshakes:470(1095ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(309ns) PrepareNewSessionHandshake:477(150342ns) {SwitchToState:179(831ns)}}}}}
2025-11-28T14:52:21.642935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789217989041737:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19024896ns) {Configure:104(13069253ns) {ProcessConfigured:135(91686ns) {StartInitialHandshake:157(89547ns) {DropHandshakes:470(1122ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(341ns) PrepareNewSessionHandshake:477(1402ns) {SwitchToState:179(621ns)}}}}}
2025-11-28T14:52:21.643460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789220538656290:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43251990ns) {Configure:104(35308379ns) {ProcessConfigured:135(501147ns) {StartInitialHandshake:157(500153ns) {DropHandshakes:470(764ns) {DropIncomingHandshake:442(201ns) DropOutgoingHandshake:457(165ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(626ns) {SwitchToState:179(78ns)}}}}}
2025-11-28T14:52:21.643557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789221047182333:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21090028ns) {Configure:104(15098438ns) {ProcessConfigured:135(1037123ns) {StartInitialHandshake:157(1035979ns) {DropHandshakes:470(1249ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(509ns)} GenerateSessionVirtualId:988(336ns) PrepareNewSessionHandshake:477(1361ns) {SwitchToState:179(194ns)}}}}}
2025-11-28T14:52:21.643571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789221672848019:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34609881ns) {Configure:104(34604924ns) {ProcessConfigured:135(859744ns) {StartInitialHandshake:157(855903ns) {DropHandshakes:470(1075ns) {DropIncomingHandshake:442(277ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(653ns) PrepareNewSessionHandshake:477(1688ns) {SwitchToState:179(543ns)}}}}}
2025-11-28T14:52:21.649040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789220538656291:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44129560ns) {Configure:104(30403239ns) {ProcessConfigured:135(1144110ns) {StartInitialHandshake:157(1142309ns) {DropHandshakes:470(725ns) {DropIncomingHandshake:442(183ns) DropOutgoingHandshake:457(100ns)} GenerateSessionVirtualId:988(166ns) PrepareNewSessionHandshake:477(1505ns) {SwitchToState:179(135ns)}}}}}
2025-11-28T14:52:21.643622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789221672848018:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34957275ns) {Configure:104(34951241ns) {ProcessConfigured:135(991230ns) {StartInitialHandshake:157(988981ns) {DropHandshakes:470(1237ns) {DropIncomingHandshake:442(688ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(1512ns) {SwitchToState:179(651ns)}}}}}
2025-11-28T14:52:21.643842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789221047182334:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45465719ns) {Configure:104(45459480ns) {ProcessConfigured:135(1528626ns) {StartInitialHandshake:157(1526879ns) {DropHandshakes:470(1037ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(533ns) PrepareNewSessionHandshake:477(1634ns) {SwitchToState:179(776ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.651148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789220538656366:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42579
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.650837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789220450164918:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(53267186ns) {Configure:104(53263336ns) {ProcessConfigured:135(5506186ns) {StartInitialHandshake:157(5503333ns) {DropHandshakes:470(173972ns) {DropIncomingHandshake:442(573ns) DropOutgoingHandshake:457(175ns)} GenerateSessionVirtualId:988(617ns) PrepareNewSessionHandshake:477(3581ns) {SwitchToState:179(1360ns)}}}}}
2025-11-28T14:52:21.652838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392950:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.653891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789221672848093:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15994
2025-11-28T14:52:21.653913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789217989041810:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35192
2025-11-28T14:52:21.655112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789221672848097:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15994
2025-11-28T14:52:21.655127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789220538656366:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.651017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789220450164917:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(53476042ns) {Configure:104(53471264ns) {ProcessConfigured:135(4521583ns) {StartInitialHandshake:157(4520117ns) {DropHandshakes:470(820ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(213ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:477(1762ns) {SwitchToState:179(135ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:21.655265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789221672848093:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:21.655234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789217989041810:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:21.655328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789220538656371:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42579
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.655370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789220538656369:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42579
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.655797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789217989041814:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35192
2025-11-28T14:52:21.655605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789221672848098:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15994
2025-11-28T14:52:21.656285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789220450164997:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-11-28T14:52:21.658742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392953:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
2025-11-28T14:52:21.657183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789217989041815:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35192
2025-11-28T14:52:21.657174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789220450164995:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-11-28T14:52:21.655723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789221672848100:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15994
2025-11-28T14:52:21.658774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392951:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
2025-11-28T14:52:21.656876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789221672848099:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15994
2025-11-28T14:52:21.659208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392952:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
2025-11-28T14:52:21.660780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789217989041816:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35192
2025-11-28T14:52:21.661321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789220538656371:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.661673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789217989041814:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.661593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789221672848097:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.661588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789220200392950:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.661756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789220450164995:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.661652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789220538656369:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.662247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789217989041815:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.662593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789220200392864:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.663073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789221672848081:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.663844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789220450164997:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.663088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789220200392951:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.663387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789217989041736:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.664208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789221672848098:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.664620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789220200392920:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.663799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789217989041805:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.665896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789220200392953:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.663467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789221047182405:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.665112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789221672848088:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.666710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789220450164990:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.666669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789220538656363:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.663824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789217989041805:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.663505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789221047182405:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.666711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789220200392863:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.665131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789221672848100:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.665987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789217989041816:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.665180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789221047182405:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:21.666861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789220200392952:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.665138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789221672848088:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.666748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789220450164990:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.666321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789217989041805:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:21.666706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789220538656363:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.668522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789220200392865:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.665271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789221047182406:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.665322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789221672848099:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.669280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789220538656363:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:21.666896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789220450164988:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.665303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789221047182406:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.666496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789217989041735:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.665865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789221672848019:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.669986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789217989041799:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.668529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789221047182398:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.665938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789221672848018:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.666916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789220450164988:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.668693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789220450164989:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.666068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789221672848020:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:52:21.672855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392957:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
::ffff:127.1.0.0
2025-11-28T14:52:21.668541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789221047182406:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:21.668708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789220450164989:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.669859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789220538656364:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.669487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789221672848088:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:21.669914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789220450164990:2112] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:21.670986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:21.669891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789220538656364:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.668581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.668752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789221047182398:2109] [node 6] ICP22 created new session: [2:7577789221047182414:2109]
2025-11-28T14:52:21.675182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392959:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
2025-11-28T14:52:21.671893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789217989041799:2108] [node 5] ICP22 created new session: [4:7577789217989041823:2108]
2025-11-28T14:52:21.673089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789221672848106:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15994
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.669852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789221047182404:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.675262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392961:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
2025-11-28T14:52:21.673731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789221672848082:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.673157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789220538656354:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.672224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789217989041823:2108] [node 5] ICS09 handshake done sender: [4:7577789217989041805:2110] self: [4:7577789217989041800:0] peer: [5:7577789221672848102:0] socket: 461 qp: -1
2025-11-28T14:52:21.670032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789220450164988:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:21.669872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789221047182404:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.672561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789217989041823:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:21.671323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789221047182414:2109] [node 6] ICS09 handshake done sender: [2:7577789221047182405:2111] self: [2:7577789221047182401:0] peer: [6:7577789220200392954:0] socket: 457 qp: -1
2025-11-28T14:52:21.673209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:21.672587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:21.676918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392960:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
2025-11-28T14:52:21.673227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789220538656364:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:21.675810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789221672848081:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30857) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789221672848097:2116] held: no
2025-11-28T14:52:21.676357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789217989041822:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35192
2025-11-28T14:52:21.671292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789221047182396:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.673168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789220450164989:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.676107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:21.675201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789220538656354:2108] [node 5] ICP22 created new session: [3:7577789220538656377:2108]
2025-11-28T14:52:21.677532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789220200392951:2125] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:21.671884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:21.676916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789217989041826:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35192
2025-11-28T14:52:21.673725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789220450164981:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.676904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789221672848109:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15994
2025-11-28T14:52:21.676407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789220538656377:2108] [node 5] ICS09 handshake done sender: [3:7577789220538656363:2112] self: [3:7577789220538656356:0] peer: [5:7577789221672848105:0] socket: 471 qp: -1
2025-11-28T14:52:21.675243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:21.678537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789217989041814:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:21.677879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789220200392950:2124] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:21.671914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789221047182414:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:21.676921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789221672848082:2109] [node 6] ICP22 created new session: [5:7577789221672848111:2109]
2025-11-28T14:52:21.676447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789220538656377:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.678636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789220200392953:2127] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:21.679822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789217989041816:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:21.675692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789220450164982:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.671946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.676966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789221672848108:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15994
2025-11-28T14:52:21.676478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:21.678924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392962:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.682189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789217989041736:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.672135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789221047182396:2107] [node 4] ICP22 created new session: [2:7577789221047182417:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.675759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789220450164981:2108] [node 5] ICP22 created new session: [1:7577789220450165003:2108]
2025-11-28T14:52:21.676980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789221672848111:2109] [node 6] ICS09 handshake done sender: [5:7577789221672848088:2110] self: [5:7577789221672848084:0] peer: [6:7577789220200392955:0] socket: 467 qp: -1
2025-11-28T14:52:21.682238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789220200392961:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.676483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789220538656355:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.683623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789217989041735:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.672498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789221047182417:2107] [node 4] ICS09 handshake done sender: [2:7577789221047182406:2112] self: [2:7577789221047182399:0] peer: [4:7577789217989041818:0] socket: 460 qp: -1
2025-11-28T14:52:21.675802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789220450165003:2108] [node 5] ICS09 handshake done sender: [1:7577789220450164990:2112] self: [1:7577789220450164987:0] peer: [5:7577789221672848104:0] socket: 488 qp: -1
2025-11-28T14:52:21.682282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789220200392864:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.677022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:21.683650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:21.672766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789221047182417:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:21.677014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789221672848111:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:21.677846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789220538656355:2109] [node 6] ICP22 created new session: [3:7577789220538656382:2109]
2025-11-28T14:52:21.676047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:21.683209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789220200392865:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21414) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789220200392952:2126] held: no
2025-11-28T14:52:21.683860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789217989041735:2048] [node 1] ICP22 created new session: [4:7577789217989041828:2048]
2025-11-28T14:52:21.677487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.672811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:21.684009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:21.678934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789220538656382:2109] [node 6] ICS09 handshake done sender: [3:7577789220538656364:2113] self: [3:7577789220538656358:0] peer: [6:7577789220200392958:0] socket: 475 qp: -1
2025-11-28T14:52:21.676425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789220450164982:2109] [node 6] ICP22 created new session: [1:7577789220450165004:2109]
2025-11-28T14:52:21.672818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789221047182404:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:21.683267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789220200392920:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.684162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789217989041736:2049] [node 2] ICP22 created new session: [4:7577789217989041829:2049]
2025-11-28T14:52:21.679778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789220538656382:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:21.683464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789220200392863:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.676454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789220450165003:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:21.680998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789220538656378:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42579
2025-11-28T14:52:21.683237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789221672848097:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:21.683503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:21.682185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:21.674425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789221047182397:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.675259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:21.676963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789220450164980:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.684187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789217989041828:2048] [node 1] ICS09 handshake done sender: [4:7577789217989041816:2118] self: [4:7577789217989041821:0] peer: [1:7577789220450164986:0] socket: 485 qp: -1
2025-11-28T14:52:21.682225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789220538656378:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.683834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789221672848020:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21414) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789221672848099:2118] held: no
2025-11-28T14:52:21.683652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:21.684149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789220538656291:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.677018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:21.683841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789220200392863:2048] [node 1] ICP22 created new session: [6:7577789220200392964:2048]
2025-11-28T14:52:21.676459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789221047182397:2108] [node 5] ICP22 created new session: [2:7577789221047182421:2108]
2025-11-28T14:52:21.684770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789217989041829:2049] [node 2] ICS09 handshake done sender: [4:7577789217989041814:2116] self: [4:7577789217989041818:0] peer: [2:7577789221047182399:0] socket: 463 qp: -1
2025-11-28T14:52:21.683871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789220200392920:2103] [node 5] ICP22 created new session: [6:7577789220200392963:2103]
2025-11-28T14:52:21.677032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789220450165004:2109] [node 6] ICS09 handshake done sender: [1:7577789220450164988:2110] self: [1:7577789220450164983:0] peer: [6:7577789220200392956:0] socket: 483 qp: -1
2025-11-28T14:52:21.684098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789221672848100:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.690355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789220538656361:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.684804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789217989041828:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:21.684331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789221672848099:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:21.677542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789221047182421:2108] [node 5] ICS09 handshake done sender: [2:7577789221047182404:2110] self: [2:7577789221047182402:0] peer: [5:7577789221672848103:0] socket: 464 qp: -1
2025-11-28T14:52:21.690381Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789220538656361:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.677854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789220450165004:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:21.684187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789220200392964:2048] [node 1] ICS09 handshake done sender: [6:7577789220200392953:2127] self: [6:7577789220200392956:0] peer: [1:7577789220450164983:0] socket: 487 qp: -1
2025-11-28T14:52:21.685311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:21.684811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:21.685461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789221672848018:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.677123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:21.677899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789221047182421:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:21.690450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789220538656385:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42579
2025-11-28T14:52:21.678612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:21.685466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789217989041829:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:21.684837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789220200392964:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:21.691679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789220538656385:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.685812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:21.684881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:21.677890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:21.691979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789220538656361:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:21.686672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789217989041827:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35192
2025-11-28T14:52:21.685531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789221672848081:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.686129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789221047182407:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.685511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789220200392864:2049] [node 2] ICP22 created new session: [6:7577789220200392965:2049]
2025-11-28T14:52:21.678587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789220450164980:2107] [node 4] ICP22 created new session: [1:7577789220450165007:2107]
2025-11-28T14:52:21.685814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:21.692429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789220538656290:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.686084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789220200392952:2126] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.686162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789221047182407:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.688456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789217989041827:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.681056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789220450165007:2107] [node 4] ICS09 handshake done sender: [1:7577789220450164989:2111] self: [1:7577789220450164986:0] peer: [4:7577789217989041821:0] socket: 484 qp: -1
2025-11-28T14:52:21.685846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:21.686650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789220200392965:2049] [node 2] ICS09 handshake done sender: [6:7577789220200392950:2124] self: [6:7577789220200392954:0] peer: [2:7577789221047182401:0] socket: 458 qp: -1
2025-11-28T14:52:21.688878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789217989041737:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.693001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789220538656386:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42579
2025-11-28T14:52:21.686672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789221047182424:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21495
2025-11-28T14:52:21.687039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789220200392965:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:21.681710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789220450165007:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:21.683275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:21.686456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789221672848110:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15994
2025-11-28T14:52:21.689297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789217989041809:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.694582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789220538656291:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789220538656378:2118] held: no
2025-11-28T14:52:21.687482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789220200392936:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.687905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789221047182423:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21495
2025-11-28T14:52:21.689322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789217989041809:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.687484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:21.689314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789221047182423:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.695089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789220538656353:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.690240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789217989041735:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:21.687956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789220200392865:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.690017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789221047182407:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:21.696123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:21.686504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789221672848081:2108] [node 4] ICP22 created new session: [5:7577789221672848116:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.688650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789220200392963:2103] [node 5] ICS09 handshake done sender: [6:7577789220200392951:2125] self: [6:7577789220200392955:0] peer: [5:7577789221672848084:0] socket: 468 qp: -1
2025-11-28T14:52:21.690883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789221047182334:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.685853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789220450165005:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-11-28T14:52:21.690666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789217989041809:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:21.696160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789220538656353:2107] [node 4] ICP22 created new session: [3:7577789220538656389:2107]
2025-11-28T14:52:21.688918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789220200392963:2103] [node 5] ICS10 traffic start
2025-11-28T14:52:21.691907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789217989041798:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.690928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:21.686681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789221672848018:2048] [node 1] ICP22 created new session: [5:7577789221672848115:2048]
2025-11-28T14:52:21.690750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789220450165005:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.691563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789220450165012:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-11-28T14:52:21.697666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789220538656378:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:21.687067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789221672848116:2108] [node 4] ICS09 handshake done sender: [5:7577789221672848097:2116] self: [5:7577789221672848102:0] peer: [4:7577789217989041800:0] socket: 466 qp: -1
2025-11-28T14:52:21.689144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:21.692443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:21.691423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789221047182334:2049] [node 3] ICP22 created new session: [2:7577789221047182428:2049]
2025-11-28T14:52:21.689277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:21.689300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789220200392865:2050] [node 3] ICP22 created new session: [6:7577789220200392969:2050]
2025-11-28T14:52:21.694531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789220450164992:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.693367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789217989041798:2109] [node 6] ICP22 created new session: [4:7577789217989041834:2109]
2025-11-28T14:52:21.697933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789220538656389:2107] [node 4] ICS09 handshake done sender: [3:7577789220538656361:2110] self: [3:7577789220538656357:0] peer: [4:7577789217989041832:0] socket: 472 qp: -1
2025-11-28T14:52:21.691933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789221047182428:2049] [node 3] ICS09 handshake done sender: [2:7577789221047182407:2113] self: [2:7577789221047182400:0] peer: [3:7577789220538656384:0] socket: 505 qp: -1
2025-11-28T14:52:21.687494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789221672848116:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:21.690353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789220200392969:2050] [node 3] ICS09 handshake done sender: [6:7577789220200392952:2126] self: [6:7577789220200392958:0] peer: [3:7577789220538656358:0] socket: 479 qp: -1
2025-11-28T14:52:21.687496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789221672848115:2048] [node 1] ICS09 handshake done sender: [5:7577789221672848100:2119] self: [5:7577789221672848104:0] peer: [1:7577789220450164987:0] socket: 489 qp: -1
2025-11-28T14:52:21.694549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789220450164992:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.699190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789220538656389:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:21.694481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789217989041834:2109] [node 6] ICS09 handshake done sender: [4:7577789217989041809:2114] self: [4:7577789217989041801:0] peer: [6:7577789220200392967:0] socket: 508 qp: -1
2025-11-28T14:52:21.687939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:21.690740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789220200392969:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:21.692112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789221047182428:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:21.694630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789217989041834:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:21.695068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789220450164991:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.691412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:21.692137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:21.692516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789221047182424:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:21.691881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789220200392920:2103] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:21.695121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:21.695087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789220450164991:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:21.687948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789221672848115:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:21.693222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789221047182425:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21495
2025-11-28T14:52:21.699391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789220538656291:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.693300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789221047182333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:21.687987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:21.697949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789217989041835:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35192
2025-11-28T14:52:21.696455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789220450164992:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:21.697383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789220200392936:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30857) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789220200392961:2131] held: no
2025-11-28T14:52:21.701082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789221047182397:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:21.688967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789221672848020:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.699429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:21.689175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:21.698046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789220450164918:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.703588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789221047182431:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21495
2025-11-28T14:52:21.689337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789221672848020:2050] [node 3] ICP22 created new session: [5:7577789221672848119:2050]
2025-11-28T14:52:21.699785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789220450164991:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:21.690356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789221672848119:2050] [node 3] ICS09 handshake done sender: [5:7577789221672848099:2118] self: [5:7577789221672848105:0] peer: [3:7577789220538656356:0] socket: 480 qp: -1
2025-11-28T14:52:21.699441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:21.704732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789221047182424:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:21.699391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789217989041737:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21414) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789217989041827:2121] held: no
2025-11-28T14:52:21.700517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789220450164917:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:21.699612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789220538656291:2049] [node 2] ICP22 created new session: [3:7577789220538656391:2049]
2025-11-28T14:52:21.701090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789217989041827:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:21.700125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789220200392972:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59771
2025-11-28T14:52:21.690504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789221672848098:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:21.701867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789217989041737:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.701610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789220200392961:2131] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:21.690726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789221672848119:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:21.703411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:21.700931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:21.704839Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789221047182333:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789221047182424:2116] held: no
2025-11-28T14:52:21.702226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789220200392936:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.703460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789217989041737:2050] [node 3] ICP22 created new session: [4:7577789217989041837:2050]
2025-11-28T14:52:21.704179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:21.705268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789221047182333:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.691281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:21.699657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789220538656391:2049] [node 2] ICS09 handshake done sender: [3:7577789220538656378:2118] self: [3:7577789220538656384:0] peer: [2:7577789221047182400:0] socket: 512 qp: -1
2025-11-28T14:52:21.703622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789217989041837:2050] [node 3] ICS09 handshake done sender: [4:7577789217989041827:2121] self: [4:7577789217989041832:0] peer: [3:7577789220538656357:0] socket: 477 qp: -1
2025-11-28T14:52:21.706368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:21.705259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789220200392936:2105] [node 4] ICP22 created new session: [6:7577789220200392973:2105]
2025-11-28T14:52:21.701266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789220450164917:2048] [node 2] ICP22 created new session: [1:7577789220450165016:2048]
2025-11-28T14:52:21.693058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789221672848019:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.703764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789217989041837:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:21.699863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789220538656391:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:21.706824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789221047182333:2048] [node 1] ICP22 created new session: [2:7577789221047182432:2048]
2025-11-28T14:52:21.703957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:21.701738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:21.707052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789220200392973:2105] [node 4] ICS09 handshake done sender: [6:7577789220200392961:2131] self: [6:7577789220200392967:0] peer: [4:7577789217989041801:0] socket: 509 qp: -1
2025-11-28T14:52:21.693266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:21.700177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:21.707061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789221047182432:2048] [node 1] ICS09 handshake done sender: [2:7577789221047182424:2116] self: [2:7577789221047182430:0] peer: [1:7577789220450164984:0] socket: 453 qp: -1
2025-11-28T14:52:21.694416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789221672848019:2049] [node 2] ICP22 created new session: [5:7577789221672848121:2049]
2025-11-28T14:52:21.707101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789220200392973:2105] [node 4] ICS10 traffic start
2025-11-28T14:52:21.701783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789220450164918:2049] [node 3] ICP22 created new session: [1:7577789220450165017:2049]
2025-11-28T14:52:21.707096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789221047182432:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:21.701526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789220538656290:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789220538656385:2119] held: no
2025-11-28T14:52:21.701800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789220450165016:2048] [node 2] ICS09 handshake done sender: [1:7577789220450164991:2113] self: [1:7577789220450164984:0] peer: [2:7577789221047182430:0] socket: 452 qp: -1
2025-11-28T14:52:21.707852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:21.695115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789221672848121:2049] [node 2] ICS09 handshake done sender: [5:7577789221672848098:2117] self: [5:7577789221672848103:0] peer: [2:7577789221047182402:0] socket: 469 qp: -1
2025-11-28T14:52:21.707236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:21.702045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789220450165016:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:21.703344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789220538656390:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42579
2025-11-28T14:52:21.695134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789221672848121:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:21.702080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:21.704499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789220538656385:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:21.697440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:21.702190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789220450165017:2049] [node 3] ICS09 handshake done sender: [1:7577789220450164992:2114] self: [1:7577789220450164985:0] peer: [3:7577789220538656388:0] socket: 456 qp: -1
2025-11-28T14:52:21.705052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789220538656290:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:21.702212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789220450165017:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:21.706688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:21.706747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789220538656290:2048] [node 1] ICP22 created new session: [3:7577789220538656394:2048]
2025-11-28T14:52:21.702240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:21.706781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789220538656394:2048] [node 1] ICS09 handshake done sender: [3:7577789220538656385:2119] self: [3:7577789220538656388:0] peer: [1:7577789220450164985:0] socket: 473 qp: -1
2025-11-28T14:52:21.706815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789220538656394:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:21.706853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:21.753538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:21.753699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789221047182432:2048] [node 1] ICS01 socket: 453 reason# EndOfStream
2025-11-28T14:52:21.753733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789221047182333:2048] [node 1] ICP30 unregister session Session# [2:7577789221047182432:2048] VirtualId# [2:7577789221047182430:0]
2025-11-28T14:52:21.753754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789221047182432:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.753808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.753846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.754335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:21.754507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789220538656394:2048] [node 1] ICS01 socket: 473 reason# EndOfStream
2025-11-28T14:52:21.754543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789220538656290:2048] [node 1] ICP30 unregister session Session# [3:7577789220538656394:2048] VirtualId# [3:7577789220538656388:0]
2025-11-28T14:52:21.754570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789220538656394:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.754652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.754697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.756142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:21.756333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789217989041828:2048] [node 1] ICS07 socket disconnect 485 reason# EndOfStream
2025-11-28T14:52:21.756384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789217989041828:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.756474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.756524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789217989041828:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:21.756834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:21.757011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789221672848115:2048] [node 1] ICS07 socket disconnect 489 reason# EndOfStream
2025-11-28T14:52:21.757037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789221672848115:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.757088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.757126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789221672848115:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:21.757267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789217989041735:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.757375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789217989041828:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.757412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789217989041735:2048] [node 1] ICP30 unregister session Session# [4:7577789217989041828:2048] VirtualId# [4:7577789217989041821:0]
2025-11-28T14:52:21.757465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.757499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789217989041735:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.757516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.757497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:21.757617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789221672848018:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.757684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789221672848115:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.757705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789221672848018:2048] [node 1] ICP30 unregister session Session# [5:7577789221672848115:2048] VirtualId# [5:7577789221672848104:0]
2025-11-28T14:52:21.757685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789220200392964:2048] [node 1] ICS07 socket disconnect 487 reason# EndOfStream
2025-11-28T14:52:21.757740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.757710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789220200392964:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.757761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789221672848018:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.757777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:21.757769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.757822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789220200392964:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:21.758465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789220200392863:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.758526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789220200392964:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.758561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789220200392863:2048] [node 1] ICP30 unregister session Session# [6:7577789220200392964:2048] VirtualId# [6:7577789220200392956:0]
2025-11-28T14:52:21.758605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:21.758638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789220200392863:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.758655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26671) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.767840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789221672848018:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:21.767868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789217989041735:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:21.768828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789220200392863:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:21.806043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:21.806202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789220538656391:2049] [node 2] ICS01 socket: 512 reason# EndOfStream
2025-11-28T14:52:21.806234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789220538656291:2049] [node 2] ICP30 unregister session Session# [3:7577789220538656391:2049] VirtualId# [3:7577789220538656384:0]
2025-11-28T14:52:21.806257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789220538656391:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.806334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:21.806383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:21.808348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:21.808554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789217989041829:2049] [node 2] ICS07 socket disconnect 463 reason# EndOfStream
2025-11-28T14:52:21.808588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789217989041829:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.808677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:21.808738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789217989041829:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:21.809051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:21.809195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789221672848121:2049] [node 2] ICS07 socket disconnect 469 reason# EndOfStream
2025-11-28T14:52:21.809220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789221672848121:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.809273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:21.809312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789221672848121:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:21.809484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789217989041736:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.809547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789217989041829:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.809612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789217989041736:2049] [node 2] ICP30 unregister session Session# [4:7577789217989041829:2049] VirtualId# [4:7577789217989041818:0]
2025-11-28T14:52:21.809675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:21.809704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789217989041736:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.809676Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:21.809719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.809870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789220200392965:2049] [node 2] ICS07 socket disconnect 458 reason# EndOfStream
2025-11-28T14:52:21.809900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789220200392965:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.809967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:21.809977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789221672848019:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.810018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789220200392965:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:21.810066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789221672848121:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.810108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789221672848019:2049] [node 2] ICP30 unregister session Session# [5:7577789221672848121:2049] VirtualId# [5:7577789221672848103:0]
2025-11-28T14:52:21.810147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:21.810182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789221672848019:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.810195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.810569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789220200392864:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.810630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789220200392965:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.810648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789220200392864:2049] [node 2] ICP30 unregister session Session# [6:7577789220200392965:2049] VirtualId# [6:7577789220200392954:0]
2025-11-28T14:52:21.810681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:21.810706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789220200392864:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.810726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63315) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.820044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789220200392864:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:21.820061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789221672848019:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:21.820061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789217989041736:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:21.847462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:21.847673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789217989041837:2050] [node 3] ICS01 socket: 477 reason# EndOfStream
2025-11-28T14:52:21.847734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789217989041737:2050] [node 3] ICP30 unregister session Session# [4:7577789217989041837:2050] VirtualId# [4:7577789217989041832:0]
2025-11-28T14:52:21.847780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789217989041837:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.847858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:21.847904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:21.848092Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:21.848265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789221672848119:2050] [node 3] ICS01 socket: 480 reason# EndOfStream
2025-11-28T14:52:21.848307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789221672848020:2050] [node 3] ICP30 unregister session Session# [5:7577789221672848119:2050] VirtualId# [5:7577789221672848105:0]
2025-11-28T14:52:21.848332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789221672848119:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.848459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:21.848508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:21.848774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:21.848895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789220200392969:2050] [node 3] ICS01 socket: 479 reason# EndOfStream
2025-11-28T14:52:21.848924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789220200392865:2050] [node 3] ICP30 unregister session Session# [6:7577789220200392969:2050] VirtualId# [6:7577789220200392958:0]
2025-11-28T14:52:21.848944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789220200392969:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.849000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:21.849048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:21.906034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:21.906257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789221672848116:2108] [node 4] ICS01 socket: 466 reason# EndOfStream
2025-11-28T14:52:21.906304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789221672848081:2108] [node 4] ICP30 unregister session Session# [5:7577789221672848116:2108] VirtualId# [5:7577789221672848102:0]
2025-11-28T14:52:21.906338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789221672848116:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.906410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:21.906465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:21.907117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:21.907267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789220200392973:2105] [node 4] ICS01 socket: 509 reason# EndOfStream
2025-11-28T14:52:21.907317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789220200392936:2105] [node 4] ICP30 unregister session Session# [6:7577789220200392973:2105] VirtualId# [6:7577789220200392967:0]
2025-11-28T14:52:21.907335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789220200392973:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.907398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:21.907446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:21.958032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:21.958187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789220200392963:2103] [node 5] ICS07 socket disconnect 468 reason# EndOfStream
2025-11-28T14:52:21.958211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789220200392963:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:21.958261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:21.958297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789220200392963:2103] [node 5] ICS15 start handshake
2025-11-28T14:52:21.958913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789220200392920:2103] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31294) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:21.959022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789220200392963:2103] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:21.959069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789220200392920:2103] [node 5] ICP30 unregister session Session# [6:7577789220200392963:2103] VirtualId# [6:7577789220200392955:0]
2025-11-28T14:52:21.959121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:21.959168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789220200392920:2103] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31294) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.959192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31294) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:21.968554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789220200392920:2103] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:22.050510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789224902246142:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:22.050548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789224902246143:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:22.070955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225475504520:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.070991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225475504521:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:22.092302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789224002547569:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.092333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789224002547570:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:22.114452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789222700321172:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.114484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789222700321173:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:22.114506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789222700321174:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:22.132567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789222977094965:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.132628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789222977094966:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:22.132652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789222977094967:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:22.149440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789225478008049:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.149473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789225478008050:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:22.149503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789225478008051:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:22.232656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789225478008108:2104] [node 5] ICP01 ready to work
2025-11-28T14:52:22.235214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789222977095037:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:22.235698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789222700321235:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:22.235544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225475504583:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:22.235875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789224902246223:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:22.236192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225475504584:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:22.235949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789224002547634:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:22.236352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789225478008112:2105] [node 4] ICP01 ready to work
2025-11-28T14:52:22.236579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789224902246224:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:22.236705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789224002547635:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:22.236948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789222700321236:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:22.237145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225475504585:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:22.237459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789224002547636:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:22.237510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789224902246225:2117] [node 6] ICP01 ready to work
2025-11-28T14:52:22.238492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789222977095040:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.246103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789222700321247:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9078
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.248093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225475504596:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8822
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.248504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225475504597:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8822
2025-11-28T14:52:22.248643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225475504598:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8822
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.252007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789224902246236:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50222
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.252174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789224902246237:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50222
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.252293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789224902246238:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50222
2025-11-28T14:52:22.252273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789222977095052:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26864
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.254037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789224902246239:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50222
2025-11-28T14:52:22.254269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789224002547650:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25337
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.254510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789224002547651:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25337
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.254555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789222700321247:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.254628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789222700321251:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9078
2025-11-28T14:52:22.254665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789222700321252:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9078
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.254663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789224002547650:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:22.254716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246237:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.254716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789224002547652:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25337
2025-11-28T14:52:22.254824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246236:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.254828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789222700321174:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.254828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789224002547653:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25337
2025-11-28T14:52:22.254928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789224002547651:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.255021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246238:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.255014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789224002547642:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.255031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789224002547642:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.255782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789224902246243:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50222
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.256189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789222700321254:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9078
2025-11-28T14:52:22.256483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789224002547642:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:22.257017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789222700321247:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:22.257357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789224002547634:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.257547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:22.257592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789224002547634:2107] [node 4] ICP22 created new session: [3:7577789224002547657:2107]
2025-11-28T14:52:22.257861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789222700321174:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.257907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.257976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789222700321174:2050] [node 3] ICP22 created new session: [4:7577789222700321255:2050]
2025-11-28T14:52:22.257791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789224002547657:2107] [node 4] ICS09 handshake done sender: [3:7577789224002547642:2114] self: [3:7577789224002547641:0] peer: [4:7577789222700321253:0] socket: 458 qp: -1
2025-11-28T14:52:22.258007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789222700321255:2050] [node 3] ICS09 handshake done sender: [4:7577789222700321247:2115] self: [4:7577789222700321253:0] peer: [3:7577789224002547641:0] socket: 463 qp: -1
2025-11-28T14:52:22.257864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789224002547657:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:22.258063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789222700321255:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:22.257899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:22.258110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.258129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225475504597:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.258231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225475504596:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.258296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225475504598:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.259413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225475504602:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8822
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.259590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789222977095052:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.259692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225475504603:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8822
2025-11-28T14:52:22.259731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789222700321251:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.259715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789224002547652:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.259868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225475504602:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.259957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789222700321173:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.259954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789224002547570:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.260132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225475504591:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.260156Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789225475504591:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.260792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225475504589:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.260826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789225475504589:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.261676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789222977095054:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26864
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.261873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789222977095055:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26864
2025-11-28T14:52:22.261927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225475504591:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.262314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789222977095055:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.262294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789222700321259:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9078
2025-11-28T14:52:22.262354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789224002547661:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25337
2025-11-28T14:52:22.262425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789222977095054:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.262385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225475504589:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:22.262961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789224002547652:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.263002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789222977095037:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.263063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789222977094967:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.263107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789222700321251:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.263099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246239:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.263089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789225475504583:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.263209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:22.263286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225475504583:2107] [node 4] ICP22 created new session: [2:7577789225475504610:2107]
2025-11-28T14:52:22.263271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246243:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.263322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225475504610:2107] [node 4] ICS09 handshake done sender: [2:7577789225475504591:2112] self: [2:7577789225475504590:0] peer: [4:7577789222700321258:0] socket: 478 qp: -1
2025-11-28T14:52:22.263307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789224002547570:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.263361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.263396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225475504610:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:22.263401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789224002547570:2049] [node 2] ICP22 created new session: [3:7577789224002547662:2049]
2025-11-28T14:52:22.263441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:22.263482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789224002547662:2049] [node 2] ICS09 handshake done sender: [3:7577789224002547652:2119] self: [3:7577789224002547660:0] peer: [2:7577789225475504588:0] socket: 489 qp: -1
2025-11-28T14:52:22.263521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789224002547662:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:22.263550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:22.263594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789222700321244:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.263618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789222700321244:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.263844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789225475504521:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.263927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789224002547644:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.263955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789224002547644:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.264505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789222700321173:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.264511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.264590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.264667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789222700321173:2049] [node 2] ICP22 created new session: [4:7577789222700321260:2049]
2025-11-28T14:52:22.264589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225475504521:2049] [node 3] ICP22 created new session: [2:7577789225475504612:2049]
2025-11-28T14:52:22.264715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789222700321260:2049] [node 2] ICS09 handshake done sender: [4:7577789222700321251:2116] self: [4:7577789222700321258:0] peer: [2:7577789225475504590:0] socket: 494 qp: -1
2025-11-28T14:52:22.264650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225475504612:2049] [node 3] ICS09 handshake done sender: [2:7577789225475504589:2111] self: [2:7577789225475504588:0] peer: [3:7577789224002547660:0] socket: 477 qp: -1
2025-11-28T14:52:22.264679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225475504612:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:22.264757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789222700321260:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:22.264724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.264813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:22.265331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789222700321252:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.266452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789224002547653:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.266543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225475504603:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.266789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789222700321172:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.267083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789222700321244:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:22.267109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789224002547569:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.267111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789222977095058:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26864
2025-11-28T14:52:22.267124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246231:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.267153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789224902246231:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.267409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789224002547644:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:22.267776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246229:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.267888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789222700321235:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.268059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789225475504520:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.268150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789224002547635:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.267815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789224902246229:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.268007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.268384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.268432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789224002547635:2108] [node 5] ICP22 created new session: [3:7577789224002547667:2108]
2025-11-28T14:52:22.268158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789222700321235:2108] [node 5] ICP22 created new session: [4:7577789222700321265:2108]
2025-11-28T14:52:22.268650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789224002547667:2108] [node 5] ICS09 handshake done sender: [3:7577789224002547644:2115] self: [3:7577789224002547643:0] peer: [5:7577789222977095057:0] socket: 469 qp: -1
2025-11-28T14:52:22.268330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789222700321265:2108] [node 5] ICS09 handshake done sender: [4:7577789222700321244:2113] self: [4:7577789222700321243:0] peer: [5:7577789222977095056:0] socket: 472 qp: -1
2025-11-28T14:52:22.268687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789224002547667:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:22.268350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789222700321265:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:22.268630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:22.268709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789222977095055:2126] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:22.268713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.268817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246227:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.268867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789224902246227:2128] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.268943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008127:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
2025-11-28T14:52:22.269114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008128:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.269274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789222977095059:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26864
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.270012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008129:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
2025-11-28T14:52:22.270048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789222977095037:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.270170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:22.270223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789222977095037:2108] [node 4] ICP22 created new session: [5:7577789222977095061:2108]
2025-11-28T14:52:22.270270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789224902246231:2130] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:22.270309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789222977095061:2108] [node 4] ICS09 handshake done sender: [5:7577789222977095055:2126] self: [5:7577789222977095056:0] peer: [4:7577789222700321243:0] socket: 501 qp: -1
2025-11-28T14:52:22.270352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789222977095061:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:22.270400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:22.271108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789224902246229:2129] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.271664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789224002547669:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25337
2025-11-28T14:52:22.271710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789224902246223:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.271763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:22.271815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789224902246223:2107] [node 4] ICP22 created new session: [1:7577789224902246252:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:22.271888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789224902246252:2107] [node 4] ICS09 handshake done sender: [1:7577789224902246231:2130] self: [1:7577789224902246230:0] peer: [4:7577789222700321263:0] socket: 491 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:22.271880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225475504617:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8822
2025-11-28T14:52:22.271870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789225478008129:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.271977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789224902246252:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:22.272052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789225478008127:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.272112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:22.272151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789225478008112:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.272127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789222700321267:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9078
2025-11-28T14:52:22.272197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789225478008050:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.272191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789224002547653:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.272251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789225478008128:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.272573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789224002547569:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.272659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789225478008051:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.272721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789224902246143:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.272779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225475504595:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.272661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.272783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.272758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789222700321246:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.272806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789225475504595:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.272756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789224002547569:2048] [node 1] ICP22 created new session: [3:7577789224002547670:2048]
2025-11-28T14:52:22.272823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789224902246143:2049] [node 3] ICP22 created new session: [1:7577789224902246254:2049]
2025-11-28T14:52:22.272841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789224002547670:2048] [node 1] ICS09 handshake done sender: [3:7577789224002547653:2120] self: [3:7577789224002547665:0] peer: [1:7577789224902246228:0] socket: 493 qp: -1
2025-11-28T14:52:22.272778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789222700321246:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.273630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789224902246254:2049] [node 3] ICS09 handshake done sender: [1:7577789224902246229:2129] self: [1:7577789224902246228:0] peer: [3:7577789224002547665:0] socket: 490 qp: -1
2025-11-28T14:52:22.272872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789224002547670:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.273683Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789222700321252:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.272896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:22.273673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789224902246254:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:22.273721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.273837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225475504595:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:22.274160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789222700321172:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.274271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.274298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789222700321172:2048] [node 1] ICP22 created new session: [4:7577789222700321269:2048]
2025-11-28T14:52:22.274319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789222977095060:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26864
2025-11-28T14:52:22.274345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789222700321269:2048] [node 1] ICS09 handshake done sender: [4:7577789222700321252:2117] self: [4:7577789222700321263:0] peer: [1:7577789224902246230:0] socket: 495 qp: -1
2025-11-28T14:52:22.274348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008133:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
2025-11-28T14:52:22.274366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789222700321269:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.274421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789225475504585:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.274486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.274392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:22.274515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225475504585:2109] [node 6] ICP22 created new session: [2:7577789225475504619:2109]
2025-11-28T14:52:22.274578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789222977095054:2125] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:22.274961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789225478008127:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.274839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225475504619:2109] [node 6] ICS09 handshake done sender: [2:7577789225475504595:2114] self: [2:7577789225475504594:0] peer: [6:7577789225478008131:0] socket: 507 qp: -1
2025-11-28T14:52:22.274879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225475504619:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:22.274906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:22.275480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789222977094967:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63842) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789222977095054:2125] held: no
2025-11-28T14:52:22.275529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789224902246227:2128] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.275599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789222977094967:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.275662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789224002547646:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.275686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789224002547646:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.275689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.275732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789225478008050:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.275735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789222977094967:2050] [node 3] ICP22 created new session: [5:7577789222977095063:2050]
2025-11-28T14:52:22.275939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789222977095063:2050] [node 3] ICS09 handshake done sender: [5:7577789222977095054:2125] self: [5:7577789222977095057:0] peer: [3:7577789224002547643:0] socket: 500 qp: -1
2025-11-28T14:52:22.275939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.275992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789225478008050:2049] [node 2] ICP22 created new session: [6:7577789225478008134:2049]
2025-11-28T14:52:22.276418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789222977095063:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:22.277057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789225478008134:2049] [node 2] ICS09 handshake done sender: [6:7577789225478008127:2115] self: [6:7577789225478008131:0] peer: [2:7577789225475504594:0] socket: 509 qp: -1
2025-11-28T14:52:22.276476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.277145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789225478008134:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:22.277193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:22.277682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789222700321246:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:22.277873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789224002547646:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:22.277905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789222977095060:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.278194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789222700321236:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.278263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789224002547636:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.278253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.278298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.278284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789222700321236:2109] [node 6] ICP22 created new session: [4:7577789222700321271:2109]
2025-11-28T14:52:22.278334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789222700321271:2109] [node 6] ICS09 handshake done sender: [4:7577789222700321246:2114] self: [4:7577789222700321245:0] peer: [6:7577789225478008130:0] socket: 512 qp: -1
2025-11-28T14:52:22.278380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789224002547636:2109] [node 6] ICP22 created new session: [3:7577789224002547673:2109]
2025-11-28T14:52:22.278417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789224002547673:2109] [node 6] ICS09 handshake done sender: [3:7577789224002547646:2116] self: [3:7577789224002547645:0] peer: [6:7577789225478008132:0] socket: 508 qp: -1
2025-11-28T14:52:22.278376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789222700321271:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:22.278480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789224002547673:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:22.278453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789222977094966:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.278487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225475504603:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.278402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:22.278536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:22.278659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789224902246142:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.279136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.279194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789224902246142:2048] [node 2] ICP22 created new session: [1:7577789224902246256:2048]
2025-11-28T14:52:22.279209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789225475504520:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.279292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789224902246256:2048] [node 2] ICS09 handshake done sender: [1:7577789224902246227:2128] self: [1:7577789224902246226:0] peer: [2:7577789225475504616:0] socket: 487 qp: -1
2025-11-28T14:52:22.279323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.279350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225475504520:2048] [node 1] ICP22 created new session: [2:7577789225475504621:2048]
2025-11-28T14:52:22.279336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789224902246256:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:22.279377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225475504621:2048] [node 1] ICS09 handshake done sender: [2:7577789225475504603:2119] self: [2:7577789225475504616:0] peer: [1:7577789224902246226:0] socket: 499 qp: -1
2025-11-28T14:52:22.279375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:22.279400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225475504621:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.279415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:22.280016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225475504593:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.280033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789225475504593:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.280337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008136:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.281125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008137:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.281743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008138:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
2025-11-28T14:52:22.282042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789225478008129:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:22.282208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789225478008128:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.282361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789224002547675:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25337
2025-11-28T14:52:22.282461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789225478008138:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.282470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225475504593:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:22.282503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789225478008112:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.282538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:22.282564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789225478008112:2105] [node 4] ICP22 created new session: [6:7577789225478008140:2105]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.282589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789225478008140:2105] [node 4] ICS09 handshake done sender: [6:7577789225478008129:2117] self: [6:7577789225478008130:0] peer: [4:7577789222700321245:0] socket: 513 qp: -1
2025-11-28T14:52:22.282648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789225478008140:2105] [node 4] ICS10 traffic start
2025-11-28T14:52:22.282673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.282783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789225475504584:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.282710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789222977095067:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26864
2025-11-28T14:52:22.282785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789222977095068:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26864
2025-11-28T14:52:22.282876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.282913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225475504584:2108] [node 5] ICP22 created new session: [2:7577789225475504624:2108]
2025-11-28T14:52:22.282949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225475504624:2108] [node 5] ICS09 handshake done sender: [2:7577789225475504593:2113] self: [2:7577789225475504592:0] peer: [5:7577789222977095066:0] socket: 506 qp: -1
2025-11-28T14:52:22.283010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225475504624:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:22.283039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:22.283128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789222977095060:2129] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.283133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789225478008051:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.283157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.283177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789225478008051:2050] [node 3] ICP22 created new session: [6:7577789225478008142:2050]
2025-11-28T14:52:22.283200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789225478008142:2050] [node 3] ICS09 handshake done sender: [6:7577789225478008128:2116] self: [6:7577789225478008132:0] peer: [3:7577789224002547645:0] socket: 510 qp: -1
2025-11-28T14:52:22.283250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789225478008142:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:22.283345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789222977094966:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.283272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.283396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.283439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789222977094966:2049] [node 2] ICP22 created new session: [5:7577789222977095069:2049]
2025-11-28T14:52:22.283485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789222977095069:2049] [node 2] ICS09 handshake done sender: [5:7577789222977095060:2129] self: [5:7577789222977095066:0] peer: [2:7577789225475504592:0] socket: 511 qp: -1
2025-11-28T14:52:22.283518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789222977095069:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:22.283548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.284760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789225478008108:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.284817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008144:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
2025-11-28T14:52:22.285803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789222977095048:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.285826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789222977095048:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.285890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789225478008144:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.286004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789225478008049:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.286202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246235:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.286234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789224902246235:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.286440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789222977095067:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.286660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789222977094965:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.286809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008147:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
2025-11-28T14:52:22.286840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789222977095048:2123] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:22.286902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789224902246233:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.286928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789224902246233:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.287705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789222977095040:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.287748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.287784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789222977095040:2109] [node 6] ICP22 created new session: [5:7577789222977095073:2109]
2025-11-28T14:52:22.287765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789224902246235:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:22.287818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789222977095073:2109] [node 6] ICS09 handshake done sender: [5:7577789222977095048:2123] self: [5:7577789222977095047:0] peer: [6:7577789225478008145:0] socket: 486 qp: -1
2025-11-28T14:52:22.287841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789222977095073:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:22.287846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789225478008138:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:22.287864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:22.288063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789225478008108:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.288099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.288125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789225478008108:2104] [node 5] ICP22 created new session: [6:7577789225478008149:2104]
2025-11-28T14:52:22.288149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789225478008149:2104] [node 5] ICS09 handshake done sender: [6:7577789225478008138:2121] self: [6:7577789225478008145:0] peer: [5:7577789222977095047:0] socket: 521 qp: -1
2025-11-28T14:52:22.288170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789225478008149:2104] [node 5] ICS10 traffic start
2025-11-28T14:52:22.288170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789224902246225:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.288186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:22.288223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.288289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789224902246225:2117] [node 6] ICP22 created new session: [1:7577789224902246262:2117]
2025-11-28T14:52:22.288345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789224902246262:2117] [node 6] ICS09 handshake done sender: [1:7577789224902246235:2132] self: [1:7577789224902246234:0] peer: [6:7577789225478008146:0] socket: 523 qp: -1
2025-11-28T14:52:22.288402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789224902246262:2117] [node 6] ICS10 traffic start
2025-11-28T14:52:22.288440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.288686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789225478008148:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45425
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.289567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789222977095075:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26864
2025-11-28T14:52:22.289978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789224902246233:2131] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:22.290490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789222977095067:2130] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.290665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789224902246224:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.290767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.290785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789222977094965:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.290824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789224902246224:2108] [node 5] ICP22 created new session: [1:7577789224902246264:2108]
2025-11-28T14:52:22.290825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.290853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789222977094965:2048] [node 1] ICP22 created new session: [5:7577789222977095076:2048]
2025-11-28T14:52:22.290843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789225478008144:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.290880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789222977095076:2048] [node 1] ICS09 handshake done sender: [5:7577789222977095067:2130] self: [5:7577789222977095072:0] peer: [1:7577789224902246232:0] socket: 522 qp: -1
2025-11-28T14:52:22.290911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789222977095076:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.290871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789224902246264:2108] [node 5] ICS09 handshake done sender: [1:7577789224902246233:2131] self: [1:7577789224902246232:0] peer: [5:7577789222977095072:0] socket: 480 qp: -1
2025-11-28T14:52:22.290955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:22.290908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789224902246264:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:22.290948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:22.291057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789225478008049:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789225478008144:2122] held: no
2025-11-28T14:52:22.291117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789225478008049:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.291145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.291202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789225478008049:2048] [node 1] ICP22 created new session: [6:7577789225478008151:2048]
2025-11-28T14:52:22.291232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789225478008151:2048] [node 1] ICS09 handshake done sender: [6:7577789225478008144:2122] self: [6:7577789225478008146:0] peer: [1:7577789224902246234:0] socket: 525 qp: -1
2025-11-28T14:52:22.291252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789225478008151:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.291271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:22.308750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:22.308909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789225475504621:2048] [node 1] ICS01 socket: 499 reason# EndOfStream
2025-11-28T14:52:22.308943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789225475504520:2048] [node 1] ICP30 unregister session Session# [2:7577789225475504621:2048] VirtualId# [2:7577789225475504616:0]
2025-11-28T14:52:22.308989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789225475504621:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.309055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.309102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.309682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:22.309956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789224002547670:2048] [node 1] ICS01 socket: 493 reason# EndOfStream
2025-11-28T14:52:22.310004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789224002547569:2048] [node 1] ICP30 unregister session Session# [3:7577789224002547670:2048] VirtualId# [3:7577789224002547665:0]
2025-11-28T14:52:22.310041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789224002547670:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.310119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.310176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.311674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:22.311890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789222700321269:2048] [node 1] ICS01 socket: 495 reason# EndOfStream
2025-11-28T14:52:22.311960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789222700321172:2048] [node 1] ICP30 unregister session Session# [4:7577789222700321269:2048] VirtualId# [4:7577789222700321263:0]
2025-11-28T14:52:22.312014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789222700321269:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.312109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.312168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.312294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:22.312482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789222977095076:2048] [node 1] ICS01 socket: 522 reason# EndOfStream
2025-11-28T14:52:22.312515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789222977094965:2048] [node 1] ICP30 unregister session Session# [5:7577789222977095076:2048] VirtualId# [5:7577789222977095072:0]
2025-11-28T14:52:22.312540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789222977095076:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.312606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.312657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.313016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:22.313148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789225478008151:2048] [node 1] ICS01 socket: 525 reason# EndOfStream
2025-11-28T14:52:22.313174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789225478008049:2048] [node 1] ICP30 unregister session Session# [6:7577789225478008151:2048] VirtualId# [6:7577789225478008146:0]
2025-11-28T14:52:22.313194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789225478008151:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.313270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.313305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.325989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:22.326155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789224002547662:2049] [node 2] ICS01 socket: 489 reason# EndOfStream
2025-11-28T14:52:22.326193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789224002547570:2049] [node 2] ICP30 unregister session Session# [3:7577789224002547662:2049] VirtualId# [3:7577789224002547660:0]
2025-11-28T14:52:22.326217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789224002547662:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.326284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:22.326337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:22.329266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:22.329507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789222700321260:2049] [node 2] ICS01 socket: 494 reason# EndOfStream
2025-11-28T14:52:22.329554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789222700321173:2049] [node 2] ICP30 unregister session Session# [4:7577789222700321260:2049] VirtualId# [4:7577789222700321258:0]
2025-11-28T14:52:22.329588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789222700321260:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.329665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:22.329723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:22.329849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:22.330017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789222977095069:2049] [node 2] ICS07 socket disconnect 511 reason# EndOfStream
2025-11-28T14:52:22.330041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789222977095069:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.330099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:22.330143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789222977095069:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:22.330499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:22.330630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789225478008134:2049] [node 2] ICS01 socket: 509 reason# EndOfStream
2025-11-28T14:52:22.330658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789222977094966:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30242) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.330676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789225478008050:2049] [node 2] ICP30 unregister session Session# [6:7577789225478008134:2049] VirtualId# [6:7577789225478008131:0]
2025-11-28T14:52:22.330690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789222977095069:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.330706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789225478008134:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.330708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789222977094966:2049] [node 2] ICP30 unregister session Session# [5:7577789222977095069:2049] VirtualId# [5:7577789222977095066:0]
2025-11-28T14:52:22.330743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:22.330763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789222977094966:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30242) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.330783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30242) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.330782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:22.330835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:22.340257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789222977094966:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:22.358613Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:22.358855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789222700321255:2050] [node 3] ICS07 socket disconnect 463 reason# EndOfStream
2025-11-28T14:52:22.358917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789222700321255:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.359009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:22.359057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789222700321255:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:22.359300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:22.359470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789222977095063:2050] [node 3] ICS01 socket: 500 reason# EndOfStream
2025-11-28T14:52:22.359556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789222977094967:2050] [node 3] ICP30 unregister session Session# [5:7577789222977095063:2050] VirtualId# [5:7577789222977095057:0]
2025-11-28T14:52:22.359583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789222977095063:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.359658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:22.359693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:22.359901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789222700321174:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63842) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.359956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789222700321255:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.359990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789222700321174:2050] [node 3] ICP30 unregister session Session# [4:7577789222700321255:2050] VirtualId# [4:7577789222700321253:0]
2025-11-28T14:52:22.360042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:22.360074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789222700321174:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.360022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:22.360088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.360147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789225478008142:2050] [node 3] ICS07 socket disconnect 510 reason# EndOfStream
2025-11-28T14:52:22.360169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789225478008142:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.360213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:22.360259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789225478008142:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:22.360970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789225478008051:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63842) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.361023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789225478008142:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.361044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789225478008051:2050] [node 3] ICP30 unregister session Session# [6:7577789225478008142:2050] VirtualId# [6:7577789225478008132:0]
2025-11-28T14:52:22.361075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:22.361096Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789225478008051:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.361105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.371027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789225478008051:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:22.377965Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:22.378198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789222977095061:2108] [node 4] ICS07 socket disconnect 501 reason# EndOfStream
2025-11-28T14:52:22.378233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789222977095061:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.378319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:22.378366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789222977095061:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:22.378943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:22.379050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789222977095037:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30243) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.379085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789222977095061:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.379095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789225478008140:2105] [node 4] ICS07 socket disconnect 513 reason# EndOfStream
2025-11-28T14:52:22.379118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789225478008140:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.379116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789222977095037:2108] [node 4] ICP30 unregister session Session# [5:7577789222977095061:2108] VirtualId# [5:7577789222977095056:0]
2025-11-28T14:52:22.379163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:22.379146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:22.379188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789225478008140:2105] [node 4] ICS15 start handshake
2025-11-28T14:52:22.379174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789222977095037:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30243) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.379188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30243) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.379706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789225478008112:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30243) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.379740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789225478008140:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.379766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789225478008112:2105] [node 4] ICP30 unregister session Session# [6:7577789225478008140:2105] VirtualId# [6:7577789225478008130:0]
2025-11-28T14:52:22.379796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:22.379813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789225478008112:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30243) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.379836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30243) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.389410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789225478008112:2105] [node 4] ICP33 wake up from error state
2025-11-28T14:52:22.389411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789222977095037:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:22.417365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:22.417563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789225478008149:2104] [node 5] ICS01 socket: 521 reason# EndOfStream
2025-11-28T14:52:22.417597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789225478008108:2104] [node 5] ICP30 unregister session Session# [6:7577789225478008149:2104] VirtualId# [6:7577789225478008145:0]
2025-11-28T14:52:22.417621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789225478008149:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.417677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:22.417722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:22.543981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789223661875174:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:22.544015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789223661875175:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:22.561444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225237897293:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.561478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225237897294:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:22.578160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789223510213532:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.578172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789223510213533:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:22.594983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789224198701284:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.595020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789224198701286:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:22.595233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789224198701285:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:22.614229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789225601053738:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.614279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789225601053740:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:22.614291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789225601053739:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:22.631861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789223338726781:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:22.631908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789223338726782:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:22.631954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789223338726783:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:22.711011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789223338726842:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:22.724569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789223338726844:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:22.724110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789224198701347:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:22.724919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789225601053817:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:22.724479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789224198701348:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:22.724875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789223661875238:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:22.725450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789223661875239:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:22.725677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789223661875240:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:22.725850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789225601053818:2125] [node 6] ICP01 ready to work
2025-11-28T14:52:22.727610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789223510213532:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38377601ns) {Configure:104(38371673ns) {ProcessConfigured:135(109738ns) {StartInitialHandshake:157(106709ns) {DropHandshakes:470(2188ns) {DropIncomingHandshake:442(549ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(626ns) PrepareNewSessionHandshake:477(1684ns) {SwitchToState:179(510ns)}}}}}
2025-11-28T14:52:22.727859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789223510213533:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33476126ns) {Configure:104(33467581ns) {ProcessConfigured:135(66705ns) {StartInitialHandshake:157(64753ns) {DropHandshakes:470(1359ns) {DropIncomingHandshake:442(373ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(584ns) PrepareNewSessionHandshake:477(6428ns) {SwitchToState:179(514ns)}}}}}
2025-11-28T14:52:22.727928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789223510213626:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:22.728240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789225601053739:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23762524ns) {Configure:104(23756053ns) {ProcessConfigured:135(166554ns) {StartInitialHandshake:157(165120ns) {DropHandshakes:470(1157ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(453ns) PrepareNewSessionHandshake:477(3882ns) {SwitchToState:179(377ns)}}}}}
2025-11-28T14:52:22.728328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789223510213627:2135] [node 5] ICP01 ready to work
2025-11-28T14:52:22.728435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789223510213628:2125] [node 6] ICP01 ready to work
2025-11-28T14:52:22.730064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789225237897293:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47815909ns) {Configure:104(47810303ns) {ProcessConfigured:135(147080ns) {StartInitialHandshake:157(138250ns) {DropHandshakes:470(1226ns) {DropIncomingHandshake:442(321ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(2948ns) PrepareNewSessionHandshake:477(843ns) {SwitchToState:179(352ns)}}}}}
2025-11-28T14:52:22.730248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789225237897294:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47503808ns) {Configure:104(47500668ns) {ProcessConfigured:135(240981ns) {StartInitialHandshake:157(238920ns) {DropHandshakes:470(1071ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(707ns) PrepareNewSessionHandshake:477(1027ns) {SwitchToState:179(515ns)}}}}}
2025-11-28T14:52:22.730790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225237897362:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:22.730792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225237897361:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:22.731395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789225237897363:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.734911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789225601053830:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47965
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.736451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789223510213638:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11040
2025-11-28T14:52:22.736782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789223510213637:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11040
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.738411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789225601053833:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47965
2025-11-28T14:52:22.739845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789225601053830:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.739998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789223510213637:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.740652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789224198701362:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5973
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.742708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789225601053833:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.742674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789223661875249:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53356
2025-11-28T14:52:22.743038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789225601053817:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.742809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789223661875250:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53356
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.743760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789223661875254:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53356
2025-11-28T14:52:22.743773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789224198701352:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.743809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789224198701352:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.743773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789223661875253:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53356
2025-11-28T14:52:22.744429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875249:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.744957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875254:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.745065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875253:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.745096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789224198701363:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5973
2025-11-28T14:52:22.745159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789223661875258:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53356
2025-11-28T14:52:22.745157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789223510213638:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.745130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225237897371:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35428
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.745440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875250:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.745451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789224198701352:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.745143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225237897370:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35428
No RDMA context found for GID: 2025-11-28T14:52:22.745522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789223510213640:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11040
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.745595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225237897372:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35428
2025-11-28T14:52:22.745662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789223510213641:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11040
2025-11-28T14:52:22.745685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789224198701362:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.745706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789225601053836:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47965
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.745802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789224198701286:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.745813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225237897370:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.746401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789224198701347:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.746636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.746993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225237897375:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35428
2025-11-28T14:52:22.746662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789224198701347:2108] [node 5] ICP22 created new session: [4:7577789224198701366:2108]
2025-11-28T14:52:22.747070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789224198701366:2108] [node 5] ICS09 handshake done sender: [4:7577789224198701352:2111] self: [4:7577789224198701351:0] peer: [5:7577789225601053834:0] socket: 463 qp: -1
2025-11-28T14:52:22.747095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789224198701366:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:22.747127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:22.747402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789223510213630:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.747209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225237897371:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.747217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789225601053830:2130] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:22.747269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225237897372:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.747430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789223510213630:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.749027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789225601053817:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.749088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:22.749939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789225601053817:2108] [node 4] ICP22 created new session: [5:7577789225601053837:2108]
2025-11-28T14:52:22.750360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789223510213630:2139] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:22.749989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789225601053837:2108] [node 4] ICS09 handshake done sender: [5:7577789225601053830:2130] self: [5:7577789225601053834:0] peer: [4:7577789224198701351:0] socket: 465 qp: -1
2025-11-28T14:52:22.750424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789223510213641:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.750183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789225601053837:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:22.750565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:22.750810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789223510213532:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.751057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789223510213626:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.751116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.751168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789223510213626:2107] [node 4] ICP22 created new session: [3:7577789223510213646:2107]
2025-11-28T14:52:22.751157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875244:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.751210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789223510213646:2107] [node 4] ICS09 handshake done sender: [3:7577789223510213630:2139] self: [3:7577789223510213629:0] peer: [4:7577789224198701365:0] socket: 481 qp: -1
2025-11-28T14:52:22.751185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789223661875244:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.751258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726859:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
2025-11-28T14:52:22.751239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789223510213646:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.751293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:22.751898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875258:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.751323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225237897375:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.752409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789224198701368:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5973
2025-11-28T14:52:22.752552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789223661875244:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:22.752556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789223510213640:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.752835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789225237897293:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.752986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789224198701363:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.752963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225237897380:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35428
2025-11-28T14:52:22.753104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789224198701286:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8235) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789224198701362:2115] held: no
2025-11-28T14:52:22.753352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875242:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:22.753416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789223338726859:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:22.753382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789223661875242:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.753730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789223661875175:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.753802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.753919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789223338726842:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.753894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789224198701370:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5973
2025-11-28T14:52:22.753906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789223510213533:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.753845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789223661875175:2049] [node 3] ICP22 created new session: [1:7577789223661875265:2049]
2025-11-28T14:52:22.754421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789223510213648:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11040
2025-11-28T14:52:22.754404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789225601053832:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.753889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789223661875265:2049] [node 3] ICS09 handshake done sender: [1:7577789223661875244:2112] self: [1:7577789223661875243:0] peer: [3:7577789223510213645:0] socket: 493 qp: -1
2025-11-28T14:52:22.754433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789225601053832:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.754273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789223661875265:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:22.754305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.754812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225237897360:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.755397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789225601053840:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47965
2025-11-28T14:52:22.754843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789225237897360:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.755373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789224198701368:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.756485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225237897360:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:22.756512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789225601053832:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:22.756943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789223510213641:2145] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.756973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789223661875242:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.757052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789225601053818:2125] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.757018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789224198701362:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:22.758581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789223661875174:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.757104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.758728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789225237897294:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.758843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789223510213532:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789223510213641:2145] held: no
2025-11-28T14:52:22.758766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.758966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.758875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726861:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
2025-11-28T14:52:22.759106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789223510213532:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.758811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789223661875174:2048] [node 2] ICP22 created new session: [1:7577789223661875269:2048]
2025-11-28T14:52:22.758873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789224198701284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.757146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789225601053818:2125] [node 6] ICP22 created new session: [5:7577789225601053842:2125]
2025-11-28T14:52:22.759061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789224198701286:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.759016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225237897294:2049] [node 3] ICP22 created new session: [2:7577789225237897384:2049]
2025-11-28T14:52:22.759089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.757637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789225601053842:2125] [node 6] ICS09 handshake done sender: [5:7577789225601053832:2132] self: [5:7577789225601053831:0] peer: [6:7577789223338726860:0] socket: 507 qp: -1
2025-11-28T14:52:22.758919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789223661875269:2048] [node 2] ICS09 handshake done sender: [1:7577789223661875242:2111] self: [1:7577789223661875241:0] peer: [2:7577789225237897381:0] socket: 494 qp: -1
2025-11-28T14:52:22.759689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.757722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789225601053842:2125] [node 6] ICS10 traffic start
2025-11-28T14:52:22.759923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225237897384:2049] [node 3] ICS09 handshake done sender: [2:7577789225237897360:2112] self: [2:7577789225237897357:0] peer: [3:7577789223510213650:0] socket: 491 qp: -1
2025-11-28T14:52:22.759910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789224198701286:2050] [node 3] ICP22 created new session: [4:7577789224198701373:2050]
2025-11-28T14:52:22.759124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789223661875269:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:22.759728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789223510213532:2048] [node 1] ICP22 created new session: [3:7577789223510213652:2048]
2025-11-28T14:52:22.758752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:22.760084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225237897384:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:22.759939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789223510213652:2048] [node 1] ICS09 handshake done sender: [3:7577789223510213641:2145] self: [3:7577789223510213645:0] peer: [1:7577789223661875243:0] socket: 498 qp: -1
2025-11-28T14:52:22.760114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789224198701285:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.760318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.759969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789223510213652:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.760547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789223338726842:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789223338726859:2115] held: no
2025-11-28T14:52:22.759696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:22.760143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789224198701373:2050] [node 3] ICS09 handshake done sender: [4:7577789224198701362:2115] self: [4:7577789224198701365:0] peer: [3:7577789223510213629:0] socket: 487 qp: -1
2025-11-28T14:52:22.760166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: 2025-11-28T14:52:22.760780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789225601053840:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:22.760176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789224198701373:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:22.760216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.760881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789225601053843:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47965
2025-11-28T14:52:22.760893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875247:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.760875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789223338726859:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:22.760927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789223661875247:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.761006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789225601053740:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.763046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789223338726842:2106] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.763327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225237897366:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.763314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.763357Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789225237897366:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.763372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789223338726842:2106] [node 5] ICP22 created new session: [6:7577789223338726864:2106]
2025-11-28T14:52:22.763397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789223510213651:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11040
2025-11-28T14:52:22.763746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789225601053843:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.763730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789223338726864:2106] [node 5] ICS09 handshake done sender: [6:7577789223338726859:2115] self: [6:7577789223338726860:0] peer: [5:7577789225601053831:0] socket: 508 qp: -1
2025-11-28T14:52:22.763774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789223338726864:2106] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.763831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789223510213533:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25738) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789223510213640:2144] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.763813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:22.764812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789225601053738:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.764956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789225237897383:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35428
2025-11-28T14:52:22.764972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225237897380:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.764981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789223661875247:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:22.764996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726862:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
2025-11-28T14:52:22.765107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789225237897293:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789225237897375:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.765162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789223510213632:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.765559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789224198701376:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5973
2025-11-28T14:52:22.765189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789223510213632:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.765724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789223510213640:2144] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.766355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789224198701363:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.766450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789223510213533:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.768629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225237897375:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.766132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789223661875238:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.766476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.766504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789223510213533:2049] [node 2] ICP22 created new session: [3:7577789223510213657:2049]
2025-11-28T14:52:22.766201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:22.768728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789223510213657:2049] [node 2] ICS09 handshake done sender: [3:7577789223510213640:2144] self: [3:7577789223510213650:0] peer: [2:7577789225237897357:0] socket: 497 qp: -1
2025-11-28T14:52:22.768773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789223510213657:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:22.768908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789223338726862:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.768938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789224198701284:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.768810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:22.769027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789225237897293:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.766327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875248:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.769104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.766346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789223661875248:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.769345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.769799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789225601053847:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47965
2025-11-28T14:52:22.769254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789224198701284:2048] [node 1] ICP22 created new session: [4:7577789224198701377:2048]
2025-11-28T14:52:22.769665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225237897293:2048] [node 1] ICP22 created new session: [2:7577789225237897390:2048]
2025-11-28T14:52:22.766454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789223661875238:2107] [node 4] ICP22 created new session: [1:7577789223661875273:2107]
2025-11-28T14:52:22.769805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789224198701377:2048] [node 1] ICS09 handshake done sender: [4:7577789224198701363:2116] self: [4:7577789224198701372:0] peer: [1:7577789223661875245:0] socket: 496 qp: -1
2025-11-28T14:52:22.769765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225237897390:2048] [node 1] ICS09 handshake done sender: [2:7577789225237897375:2119] self: [2:7577789225237897381:0] peer: [1:7577789223661875241:0] socket: 499 qp: -1
2025-11-28T14:52:22.768759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789223661875273:2107] [node 4] ICS09 handshake done sender: [1:7577789223661875247:2113] self: [1:7577789223661875245:0] peer: [4:7577789224198701372:0] socket: 495 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.769848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789224198701377:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.769805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225237897390:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.768803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789223661875273:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:22.769992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.770012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:22.769022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:22.772865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789223510213632:2140] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:22.771976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225237897366:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:22.772774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789225601053848:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47965
::ffff:127.1.0.02025-11-28T14:52:22.772744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726866:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.769905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789223661875248:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:22.773014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789225601053849:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47965
2025-11-28T14:52:22.773116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789223510213655:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11040
2025-11-28T14:52:22.773236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789223661875239:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.773134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789223338726844:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.773153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726867:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
2025-11-28T14:52:22.773744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789225601053840:2134] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:22.773738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.773572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789223510213627:2135] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.773784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789223661875239:2108] [node 5] ICP22 created new session: [1:7577789223661875276:2108]
2025-11-28T14:52:22.773900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789223338726867:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.773788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789224198701361:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.774014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789223338726783:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.773623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.773820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789224198701361:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.773937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789223661875276:2108] [node 5] ICS09 handshake done sender: [1:7577789223661875248:2114] self: [1:7577789223661875246:0] peer: [5:7577789225601053846:0] socket: 517 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:22.774490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789223661875276:2108] [node 5] ICS10 traffic start
::ffff:127.1.0.02025-11-28T14:52:22.773669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789223510213627:2135] [node 5] ICP22 created new session: [3:7577789223510213659:2135]
2025-11-28T14:52:22.774510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789225601053843:2135] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.774518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789225237897361:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.774511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:22.774648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789225601053740:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.773737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789223510213659:2135] [node 5] ICS09 handshake done sender: [3:7577789223510213632:2140] self: [3:7577789223510213631:0] peer: [5:7577789225601053845:0] socket: 462 qp: -1
2025-11-28T14:52:22.774733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:22.774741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.773797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789223510213659:2135] [node 5] ICS10 traffic start
2025-11-28T14:52:22.774816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789224198701378:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5973
2025-11-28T14:52:22.774784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225237897361:2107] [node 4] ICP22 created new session: [2:7577789225237897392:2107]
2025-11-28T14:52:22.774842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789225601053740:2050] [node 3] ICP22 created new session: [5:7577789225601053850:2050]
2025-11-28T14:52:22.773893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:22.775002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225237897392:2107] [node 4] ICS09 handshake done sender: [2:7577789225237897366:2113] self: [2:7577789225237897364:0] peer: [4:7577789224198701374:0] socket: 503 qp: -1
2025-11-28T14:52:22.773891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789223510213655:2148] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.775019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789225601053850:2050] [node 3] ICS09 handshake done sender: [5:7577789225601053840:2134] self: [5:7577789225601053845:0] peer: [3:7577789223510213631:0] socket: 513 qp: -1
2025-11-28T14:52:22.775162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225237897392:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:22.775135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789225601053850:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.775201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:22.775170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.775116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789223338726866:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.775497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789224198701368:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.775231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789223338726781:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.776129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789223510213636:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.775941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726869:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
2025-11-28T14:52:22.776160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789223510213636:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.775725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789224198701361:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:22.776611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789225601053738:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.777598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.777655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789223661875252:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.777641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789225601053738:2048] [node 1] ICP22 created new session: [5:7577789225601053852:2048]
2025-11-28T14:52:22.777677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789223661875252:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.777935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789224198701285:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.777974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.778318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789225601053852:2048] [node 1] ICS09 handshake done sender: [5:7577789225601053843:2135] self: [5:7577789225601053846:0] peer: [1:7577789223661875246:0] socket: 519 qp: -1
2025-11-28T14:52:22.778342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789224198701285:2049] [node 2] ICP22 created new session: [4:7577789224198701381:2049]
2025-11-28T14:52:22.778437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789223510213636:2141] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:22.778383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789224198701348:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.778366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789225601053852:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.778393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:22.778469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.778593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789223661875252:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:22.778484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726872:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
2025-11-28T14:52:22.778475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789224198701381:2049] [node 2] ICS09 handshake done sender: [4:7577789224198701368:2117] self: [4:7577789224198701374:0] peer: [2:7577789225237897364:0] socket: 505 qp: -1
2025-11-28T14:52:22.778670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789223510213628:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.778502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789224198701381:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:22.778858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.778927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789223661875240:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.778514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789224198701348:2109] [node 6] ICP22 created new session: [4:7577789224198701382:2109]
2025-11-28T14:52:22.778673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:22.779079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789223510213628:2125] [node 6] ICP22 created new session: [3:7577789223510213663:2125]
2025-11-28T14:52:22.779089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.779170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789223510213663:2125] [node 6] ICS09 handshake done sender: [3:7577789223510213636:2141] self: [3:7577789223510213635:0] peer: [6:7577789223338726870:0] socket: 521 qp: -1
2025-11-28T14:52:22.778840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789224198701382:2109] [node 6] ICS09 handshake done sender: [4:7577789224198701361:2114] self: [4:7577789224198701360:0] peer: [6:7577789223338726868:0] socket: 479 qp: -1
2025-11-28T14:52:22.779418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789223510213663:2125] [node 6] ICS10 traffic start
2025-11-28T14:52:22.779400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789223661875240:2109] [node 6] ICP22 created new session: [1:7577789223661875279:2109]
2025-11-28T14:52:22.779055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789224198701382:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.779444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:22.779430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789223661875279:2109] [node 6] ICS09 handshake done sender: [1:7577789223661875252:2117] self: [1:7577789223661875251:0] peer: [6:7577789223338726871:0] socket: 470 qp: -1
2025-11-28T14:52:22.779093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:22.779576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789223338726862:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:22.779450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789223661875279:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:22.779604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726873:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
2025-11-28T14:52:22.779468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:22.780489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789225601053738:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:22.782705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789223338726783:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8235) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789223338726867:2119] held: no
2025-11-28T14:52:22.782630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789225601053847:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.782821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789225601053739:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.782850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789223338726844:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.782894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:22.782922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789223338726844:2108] [node 4] ICP22 created new session: [6:7577789223338726875:2108]
2025-11-28T14:52:22.782936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726874:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
2025-11-28T14:52:22.782960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789223338726875:2108] [node 4] ICS09 handshake done sender: [6:7577789223338726862:2117] self: [6:7577789223338726868:0] peer: [4:7577789224198701360:0] socket: 476 qp: -1
2025-11-28T14:52:22.782995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789223338726875:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:22.783053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:22.783038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225237897367:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.783072Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789225237897367:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.784057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789223338726781:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789223338726866:2118] held: no
2025-11-28T14:52:22.784269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225237897367:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:22.784268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789223338726867:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:22.784503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789225601053855:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47965
2025-11-28T14:52:22.784635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789225237897362:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:22.784794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:22.784859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789223338726866:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:22.784863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225237897362:2108] [node 5] ICP22 created new session: [2:7577789225237897395:2108]
2025-11-28T14:52:22.784895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789225601053847:2136] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.784912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789223338726783:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.785171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:22.785281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789223338726783:2050] [node 3] ICP22 created new session: [6:7577789223338726877:2050]
2025-11-28T14:52:22.784907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225237897395:2108] [node 5] ICS09 handshake done sender: [2:7577789225237897367:2114] self: [2:7577789225237897365:0] peer: [5:7577789225601053854:0] socket: 523 qp: -1
2025-11-28T14:52:22.785421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789223338726877:2050] [node 3] ICS09 handshake done sender: [6:7577789223338726867:2119] self: [6:7577789223338726870:0] peer: [3:7577789223510213635:0] socket: 527 qp: -1
2025-11-28T14:52:22.785475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789223338726877:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:22.784935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225237897395:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:22.785491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789225601053739:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.785485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789223338726869:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.785372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:22.785515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:22.785535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.785580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789225601053739:2049] [node 2] ICP22 created new session: [5:7577789225601053856:2049]
2025-11-28T14:52:22.786383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789223338726781:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.786395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789225601053856:2049] [node 2] ICS09 handshake done sender: [5:7577789225601053847:2136] self: [5:7577789225601053854:0] peer: [2:7577789225237897365:0] socket: 525 qp: -1
2025-11-28T14:52:22.786497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:22.786561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789223338726781:2048] [node 1] ICP22 created new session: [6:7577789223338726879:2048]
2025-11-28T14:52:22.786430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789225601053856:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:22.786468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:22.786613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789223338726879:2048] [node 1] ICS09 handshake done sender: [6:7577789223338726866:2118] self: [6:7577789223338726871:0] peer: [1:7577789223661875251:0] socket: 490 qp: -1
2025-11-28T14:52:22.786647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789223338726879:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:22.786680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:22.786719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789225601053739:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:22.787056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789223338726782:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:22.787249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789225237897374:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:22.787276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789225237897374:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:22.787891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789225237897374:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:22.788076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789225237897363:2109] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:22.788118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:22.788144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789225237897363:2109] [node 6] ICP22 created new session: [2:7577789225237897398:2109]
2025-11-28T14:52:22.788196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789223338726882:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1878
2025-11-28T14:52:22.788176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789225237897398:2109] [node 6] ICS09 handshake done sender: [2:7577789225237897374:2118] self: [2:7577789225237897373:0] peer: [6:7577789223338726881:0] socket: 464 qp: -1
2025-11-28T14:52:22.788198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789225237897398:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:22.788219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:22.788738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789223338726782:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25738) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789223338726869:2120] held: no
2025-11-28T14:52:22.788872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789223338726869:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:22.789441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789223338726782:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:22.789490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:22.789541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789223338726782:2049] [node 2] ICP22 created new session: [6:7577789223338726883:2049]
2025-11-28T14:52:22.789588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789223338726883:2049] [node 2] ICS09 handshake done sender: [6:7577789223338726869:2120] self: [6:7577789223338726881:0] peer: [2:7577789225237897373:0] socket: 531 qp: -1
2025-11-28T14:52:22.789624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789223338726883:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:22.789693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:22.815335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:22.815520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789225237897390:2048] [node 1] ICS01 socket: 499 reason# EndOfStream
2025-11-28T14:52:22.815556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789225237897293:2048] [node 1] ICP30 unregister session Session# [2:7577789225237897390:2048] VirtualId# [2:7577789225237897381:0]
2025-11-28T14:52:22.815583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789225237897390:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.815654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.815704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.816526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:22.816762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789223510213652:2048] [node 1] ICS01 socket: 498 reason# EndOfStream
2025-11-28T14:52:22.816806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789223510213532:2048] [node 1] ICP30 unregister session Session# [3:7577789223510213652:2048] VirtualId# [3:7577789223510213645:0]
2025-11-28T14:52:22.816836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789223510213652:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.816932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.816988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.818749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:22.818918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789224198701377:2048] [node 1] ICS01 socket: 496 reason# EndOfStream
2025-11-28T14:52:22.818968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789224198701284:2048] [node 1] ICP30 unregister session Session# [4:7577789224198701377:2048] VirtualId# [4:7577789224198701372:0]
2025-11-28T14:52:22.819010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789224198701377:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.819068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.819121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.819551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:22.819677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789225601053852:2048] [node 1] ICS07 socket disconnect 519 reason# EndOfStream
2025-11-28T14:52:22.819694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789225601053852:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.819747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.819785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789225601053852:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:22.820435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:22.820447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789225601053738:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27856) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.820489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789225601053852:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.820547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789225601053738:2048] [node 1] ICP30 unregister session Session# [5:7577789225601053852:2048] VirtualId# [5:7577789225601053846:0]
2025-11-28T14:52:22.820607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.820649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789225601053738:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27856) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.820659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27856) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.820743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789223338726879:2048] [node 1] ICS01 socket: 490 reason# EndOfStream
2025-11-28T14:52:22.820789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789223338726781:2048] [node 1] ICP30 unregister session Session# [6:7577789223338726879:2048] VirtualId# [6:7577789223338726871:0]
2025-11-28T14:52:22.820828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789223338726879:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.820935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:22.820998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:22.830757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789225601053738:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:22.858822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:22.859042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789223510213657:2049] [node 2] ICS01 socket: 497 reason# EndOfStream
2025-11-28T14:52:22.859086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789223510213533:2049] [node 2] ICP30 unregister session Session# [3:7577789223510213657:2049] VirtualId# [3:7577789223510213650:0]
2025-11-28T14:52:22.859116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789223510213657:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.859207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:22.859267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:22.860773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:22.861018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789224198701381:2049] [node 2] ICS07 socket disconnect 505 reason# EndOfStream
2025-11-28T14:52:22.861068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789224198701381:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.861177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:22.861234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789224198701381:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:22.861583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:22.861725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789225601053856:2049] [node 2] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-28T14:52:22.861742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789225601053856:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.861803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:22.861827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789225601053856:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:22.861898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789224198701285:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.861996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789224198701381:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.862035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789224198701285:2049] [node 2] ICP30 unregister session Session# [4:7577789224198701381:2049] VirtualId# [4:7577789224198701374:0]
2025-11-28T14:52:22.862137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:22.862162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789224198701285:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.862171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.862522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789225601053739:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.862557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789225601053856:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.862501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:22.862615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789225601053739:2049] [node 2] ICP30 unregister session Session# [5:7577789225601053856:2049] VirtualId# [5:7577789225601053854:0]
2025-11-28T14:52:22.862644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:22.862659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789225601053739:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.862684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789223338726883:2049] [node 2] ICS01 socket: 531 reason# EndOfStream
2025-11-28T14:52:22.862666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.862719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789223338726782:2049] [node 2] ICP30 unregister session Session# [6:7577789223338726883:2049] VirtualId# [6:7577789223338726881:0]
2025-11-28T14:52:22.862739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789223338726883:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.862810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:22.862850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:22.871732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789224198701285:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:22.872679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789225601053739:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:22.917083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:22.917253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789224198701373:2050] [node 3] ICS01 socket: 487 reason# EndOfStream
2025-11-28T14:52:22.917286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789224198701286:2050] [node 3] ICP30 unregister session Session# [4:7577789224198701373:2050] VirtualId# [4:7577789224198701365:0]
2025-11-28T14:52:22.917323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789224198701373:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.917391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:22.917434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:22.917973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:22.918097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789223338726877:2050] [node 3] ICS01 socket: 527 reason# EndOfStream
2025-11-28T14:52:22.918123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789223338726783:2050] [node 3] ICP30 unregister session Session# [6:7577789223338726877:2050] VirtualId# [6:7577789223338726870:0]
2025-11-28T14:52:22.918143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789223338726877:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.918190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:22.918226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:22.918827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:22.918971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789225601053850:2050] [node 3] ICS07 socket disconnect 513 reason# EndOfStream
2025-11-28T14:52:22.918993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789225601053850:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.919045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:22.919072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789225601053850:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:22.919636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789225601053740:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8235) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.919707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789225601053850:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.919744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789225601053740:2050] [node 3] ICP30 unregister session Session# [5:7577789225601053850:2050] VirtualId# [5:7577789225601053845:0]
2025-11-28T14:52:22.919778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:22.919803Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789225601053740:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8235) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.919812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8235) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.930063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789225601053740:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:22.949839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:22.950023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789225601053837:2108] [node 4] ICS07 socket disconnect 465 reason# EndOfStream
2025-11-28T14:52:22.950063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789225601053837:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.950168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:22.950223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789225601053837:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:22.950580Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:22.950698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789223338726875:2108] [node 4] ICS07 socket disconnect 476 reason# EndOfStream
2025-11-28T14:52:22.950716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789223338726875:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.950762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:22.950795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789223338726875:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:22.950879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789225601053817:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21783) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.950940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789225601053837:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.951035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789225601053817:2108] [node 4] ICP30 unregister session Session# [5:7577789225601053837:2108] VirtualId# [5:7577789225601053834:0]
2025-11-28T14:52:22.951087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:22.951121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789225601053817:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.951151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.951431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789223338726844:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21783) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:22.951556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789223338726875:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:22.951607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789223338726844:2108] [node 4] ICP30 unregister session Session# [6:7577789223338726875:2108] VirtualId# [6:7577789223338726868:0]
2025-11-28T14:52:22.951642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:22.951690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789223338726844:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.951707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:22.960798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789225601053817:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:22.961917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789223338726844:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:22.988911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:22.989134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789223338726864:2106] [node 5] ICS01 socket: 508 reason# EndOfStream
2025-11-28T14:52:22.989179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789223338726842:2106] [node 5] ICP30 unregister session Session# [6:7577789223338726864:2106] VirtualId# [6:7577789223338726860:0]
2025-11-28T14:52:22.989214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789223338726864:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:22.989281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:22.989324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:23.086364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789227994833796:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:23.086397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789227994833797:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:23.102320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789228048703690:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.102460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789228048703691:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:23.118917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789227947043003:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.118973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789227947043004:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:23.139741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789227174868600:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.139764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789227174868601:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:23.139786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789227174868602:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:23.159719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789229510755578:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.159758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789229510755580:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:23.159938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789229510755579:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:23.176456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229905680648:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:23.176481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229905680646:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.176491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229905680647:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:23.252958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229905680708:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:23.259076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789228048703754:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:23.265340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789227994833860:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:23.265367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789229510755643:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:23.265509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789227947043068:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:23.265623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789227174868664:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:23.265939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789228048703755:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:23.265624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229905680709:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:23.265500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789229510755642:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:23.266252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789227947043069:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:23.266199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789227994833861:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:23.266364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789227174868665:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:23.266658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789228048703756:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:23.267189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789227947043070:2111] [node 6] ICP01 ready to work
2025-11-28T14:52:23.267181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789227994833862:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:23.269761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789228048703691:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47436040ns) {Configure:104(47432889ns) {ProcessConfigured:135(242873ns) {StartInitialHandshake:157(239453ns) {DropHandshakes:470(2207ns) {DropIncomingHandshake:442(639ns) DropOutgoingHandshake:457(492ns)} GenerateSessionVirtualId:988(720ns) PrepareNewSessionHandshake:477(3387ns) {SwitchToState:179(1510ns)}}}}}
2025-11-28T14:52:23.271501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789227994833797:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18120067ns) {Configure:104(18102229ns) {ProcessConfigured:135(282193ns) {StartInitialHandshake:157(279869ns) {DropHandshakes:470(2181ns) {DropIncomingHandshake:442(632ns) DropOutgoingHandshake:457(360ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:477(1696ns) {SwitchToState:179(659ns)}}}}}
2025-11-28T14:52:23.271928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789229905680646:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(41644348ns) {Configure:104(41636081ns) {ProcessConfigured:135(338420ns) {StartInitialHandshake:157(336410ns) {DropHandshakes:470(3494ns) {DropIncomingHandshake:442(294ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(779ns) PrepareNewSessionHandshake:477(927ns) {SwitchToState:179(421ns)}}}}}
2025-11-28T14:52:23.271511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789227994833796:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46117312ns) {Configure:104(37982198ns) {ProcessConfigured:135(325018ns) {StartInitialHandshake:157(323208ns) {DropHandshakes:470(4637ns) {DropIncomingHandshake:442(686ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(712ns) PrepareNewSessionHandshake:477(888ns) {SwitchToState:179(427ns)}}}}}
2025-11-28T14:52:23.271933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789227947043003:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46270895ns) {Configure:104(46259111ns) {ProcessConfigured:135(374039ns) {StartInitialHandshake:157(371372ns) {DropHandshakes:470(1180ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(455ns) PrepareNewSessionHandshake:477(1745ns) {SwitchToState:179(653ns)}}}}}
2025-11-28T14:52:23.272146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789227947043004:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47567192ns) {Configure:104(47560330ns) {ProcessConfigured:135(45604ns) {StartInitialHandshake:157(44109ns) {DropHandshakes:470(1042ns) {DropIncomingHandshake:442(333ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(369ns) {SwitchToState:179(136ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.279048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789227994833871:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40727
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.281654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789227947043078:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64326
2025-11-28T14:52:23.282883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789227174868672:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.285870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789227994833877:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40727
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.286203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789227947043085:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64326
2025-11-28T14:52:23.286299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789227174868681:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.286904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789229510755653:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15418
2025-11-28T14:52:23.286856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833877:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.286904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789227994833878:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40727
2025-11-28T14:52:23.287121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789228048703771:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27486
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.286507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789227947043078:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.287420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833871:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:23.287884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789229510755653:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:23.287906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789228048703772:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27486
2025-11-28T14:52:23.289138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789227994833879:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40727
2025-11-28T14:52:23.289156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789227174868672:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.288251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789228048703770:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27486
2025-11-28T14:52:23.289577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789228048703773:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27486
2025-11-28T14:52:23.290356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789227174868680:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
2025-11-28T14:52:23.290114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789227947043087:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64326
2025-11-28T14:52:23.290357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789227947043086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64326
2025-11-28T14:52:23.289600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789228048703774:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27486
2025-11-28T14:52:23.290386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789227174868601:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.291052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789227174868681:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.289629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789228048703771:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.291042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833866:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.291073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789227174868680:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.290700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789227947043085:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.290886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789227947043086:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.291096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789227947043004:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.291077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789227994833866:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.290154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789228048703690:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.291163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833878:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.291148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789227174868602:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.291128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789227947043003:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.290371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789228048703770:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.291158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789227174868600:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.291304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833879:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.291519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789227947043087:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.290490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789228048703774:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.291359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789227994833880:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40727
2025-11-28T14:52:23.293073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789227947043074:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.290856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789228048703772:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.292997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833868:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.293104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789227947043074:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.290979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789228048703773:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.293001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833865:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.291267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789228048703760:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.293014Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789227994833868:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.291288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789228048703760:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.293026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789227994833865:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.291516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789228048703758:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.296577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833880:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.291529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789228048703758:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.296889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789227994833868:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:23.295908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789228048703760:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:23.296937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789227947043074:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:23.296949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789227994833866:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:23.297103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789228048703758:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.296968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789227994833865:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.297303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789228048703754:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.297255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789227174868685:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.297881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.297922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789227947043093:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64326
2025-11-28T14:52:23.298425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789227994833796:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.297949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789228048703754:2107] [node 4] ICP22 created new session: [2:7577789228048703783:2107]
2025-11-28T14:52:23.298659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789227947043068:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.298980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789227174868672:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:23.298492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-28T14:52:23.299192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789227174868686:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
2025-11-28T14:52:23.298777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789228048703783:2107] [node 4] ICS09 handshake done sender: [2:7577789228048703760:2112] self: [2:7577789228048703759:0] peer: [4:7577789227174868682:0] socket: 469 qp: -1
2025-11-28T14:52:23.299425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:23.298541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789227994833796:2048] [node 2] ICP22 created new session: [1:7577789227994833889:2048]
2025-11-28T14:52:23.299593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789227947043068:2107] [node 4] ICP22 created new session: [3:7577789227947043095:2107]
2025-11-28T14:52:23.299474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789227174868687:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
2025-11-28T14:52:23.299660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789227947043095:2107] [node 4] ICS09 handshake done sender: [3:7577789227947043074:2113] self: [3:7577789227947043072:0] peer: [4:7577789227174868683:0] socket: 491 qp: -1
2025-11-28T14:52:23.298967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789227994833797:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.298816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789228048703783:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.299423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789227994833889:2048] [node 2] ICS09 handshake done sender: [1:7577789227994833866:2112] self: [1:7577789227994833864:0] peer: [2:7577789228048703775:0] socket: 480 qp: -1
2025-11-28T14:52:23.300881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789227174868601:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.299820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789227947043095:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:23.298825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789228048703691:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.299429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789227994833860:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.299849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:23.301640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:23.299205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.301588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789227947043078:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:23.299441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.299213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:23.301682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789227174868601:2049] [node 2] ICP22 created new session: [4:7577789227174868688:2049]
2025-11-28T14:52:23.299474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:23.301711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789227947043094:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64326
2025-11-28T14:52:23.299251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789228048703691:2049] [node 3] ICP22 created new session: [2:7577789228048703784:2049]
2025-11-28T14:52:23.299491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789227994833889:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:23.301723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789227174868688:2049] [node 2] ICS09 handshake done sender: [4:7577789227174868672:2114] self: [4:7577789227174868682:0] peer: [2:7577789228048703759:0] socket: 474 qp: -1
2025-11-28T14:52:23.302627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789227947043004:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.299504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789228048703784:2049] [node 3] ICS09 handshake done sender: [2:7577789228048703758:2111] self: [2:7577789228048703757:0] peer: [3:7577789227947043088:0] socket: 468 qp: -1
2025-11-28T14:52:23.299517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789227994833860:2107] [node 4] ICP22 created new session: [1:7577789227994833891:2107]
2025-11-28T14:52:23.301901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789227174868688:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:23.302662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:23.301931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:23.299607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:23.302174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789227174868681:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.299798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789228048703784:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:23.303396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789227174868680:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.302891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789227947043004:2049] [node 2] ICP22 created new session: [3:7577789227947043098:2049]
2025-11-28T14:52:23.299778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789227994833797:2049] [node 3] ICP22 created new session: [1:7577789227994833890:2049]
2025-11-28T14:52:23.299828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:23.300114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789227994833890:2049] [node 3] ICS09 handshake done sender: [1:7577789227994833865:2111] self: [1:7577789227994833863:0] peer: [3:7577789227947043089:0] socket: 477 qp: -1
2025-11-28T14:52:23.303851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789227174868602:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789227174868681:2118] held: no
2025-11-28T14:52:23.303002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789227947043098:2049] [node 2] ICS09 handshake done sender: [3:7577789227947043078:2115] self: [3:7577789227947043088:0] peer: [2:7577789228048703757:0] socket: 473 qp: -1
2025-11-28T14:52:23.299965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789228048703782:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27486
2025-11-28T14:52:23.303367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789227947043098:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:23.300277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789227994833891:2107] [node 4] ICS09 handshake done sender: [1:7577789227994833868:2113] self: [1:7577789227994833867:0] peer: [4:7577789227174868684:0] socket: 481 qp: -1
2025-11-28T14:52:23.304054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789228048703771:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.303449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:23.305109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789228048703690:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789228048703771:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.300282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789227994833890:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:23.303872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789227174868601:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:23.304296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789227947043004:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:23.300430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789227994833891:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:23.304460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789227174868602:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.305197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789228048703690:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.304492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.305875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.305160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789227947043085:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.305835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680724:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
No RDMA context found for GID: 2025-11-28T14:52:23.300450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:23.304656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789227174868600:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.306299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789228048703690:2048] [node 1] ICP22 created new session: [2:7577789228048703789:2048]
2025-11-28T14:52:23.300839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:23.306768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789228048703789:2048] [node 1] ICS09 handshake done sender: [2:7577789228048703771:2117] self: [2:7577789228048703775:0] peer: [1:7577789227994833864:0] socket: 496 qp: -1
2025-11-28T14:52:23.307619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789229510755659:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15418
2025-11-28T14:52:23.304675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789227174868602:2050] [node 3] ICP22 created new session: [4:7577789227174868690:2050]
2025-11-28T14:52:23.305175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789227174868690:2050] [node 3] ICS09 handshake done sender: [4:7577789227174868681:2118] self: [4:7577789227174868683:0] peer: [3:7577789227947043072:0] socket: 501 qp: -1
2025-11-28T14:52:23.306811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789228048703789:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.308050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680725:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
2025-11-28T14:52:23.305187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.307768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:23.305711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789227174868690:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.308488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789227947043003:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.305736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:23.305772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789227174868600:2048] [node 1] ICP22 created new session: [4:7577789227174868691:2048]
2025-11-28T14:52:23.308818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.309636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789227947043003:2048] [node 1] ICP22 created new session: [3:7577789227947043101:2048]
2025-11-28T14:52:23.309621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680726:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
2025-11-28T14:52:23.305822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789227174868691:2048] [node 1] ICS09 handshake done sender: [4:7577789227174868680:2117] self: [4:7577789227174868684:0] peer: [1:7577789227994833867:0] socket: 488 qp: -1
2025-11-28T14:52:23.309715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789227947043101:2048] [node 1] ICS09 handshake done sender: [3:7577789227947043085:2118] self: [3:7577789227947043089:0] peer: [1:7577789227994833863:0] socket: 483 qp: -1
2025-11-28T14:52:23.309615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789229510755660:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15418
2025-11-28T14:52:23.305857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789227174868691:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.309753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789227947043101:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.306277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:23.309797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:23.310518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229905680724:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.310659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789229510755660:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.310624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229905680647:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.310643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229905680726:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.310651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229905680725:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.311002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789227947043003:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:23.311036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789229510755578:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.311015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229905680646:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.311214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789229510755661:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15418
2025-11-28T14:52:23.311026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229905680648:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.311296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833870:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.311440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789229510755659:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.311278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789228048703767:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.312550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789229510755662:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15418
2025-11-28T14:52:23.311327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789227994833870:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.311523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789227947043081:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.312982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789227994833873:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.312995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680728:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
2025-11-28T14:52:23.311309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789228048703767:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.311557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789227947043081:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.313012Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789227994833873:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.313067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789229510755579:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.313268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789227947043081:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:23.314036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789229510755662:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.313575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789228048703767:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:23.314060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229905680728:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.314042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789229510755661:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.314112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789228048703769:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.314164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789227947043070:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.314142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789229510755580:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.314201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789228048703769:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.314360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789228048703755:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.314371Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789227994833870:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:23.314549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789229510755643:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.314584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789227174868676:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.314568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229905680709:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.315187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789227947043079:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.315221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:23.316222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789227994833873:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:23.314616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789227174868676:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.315337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789228048703755:2110] [node 6] ICP22 created new session: [2:7577789228048703793:2110]
2025-11-28T14:52:23.316832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789227994833861:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.315197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.315227Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789227947043079:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.315243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789227947043070:2111] [node 6] ICP22 created new session: [3:7577789227947043104:2111]
2025-11-28T14:52:23.315226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789227174868675:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.316885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.316824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789228048703793:2110] [node 6] ICS09 handshake done sender: [2:7577789228048703767:2114] self: [2:7577789228048703766:0] peer: [6:7577789229905680727:0] socket: 494 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.317890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229905680647:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789229905680724:2115] held: no
2025-11-28T14:52:23.315249Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789227174868675:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.316931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789228048703769:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:23.315357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789227947043104:2111] [node 6] ICS09 handshake done sender: [3:7577789227947043081:2117] self: [3:7577789227947043080:0] peer: [6:7577789229905680730:0] socket: 486 qp: -1
2025-11-28T14:52:23.316931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789227994833861:2108] [node 5] ICP22 created new session: [1:7577789227994833899:2108]
2025-11-28T14:52:23.317162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789227174868676:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:23.318509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680733:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
2025-11-28T14:52:23.316206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789227947043104:2111] [node 6] ICS10 traffic start
2025-11-28T14:52:23.317066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789228048703793:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:23.317307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789227994833862:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.317166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:23.317455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789227174868675:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:23.317346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:23.318759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789229510755665:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15418
2025-11-28T14:52:23.316689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.317946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789228048703756:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.319794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680731:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
No RDMA context found for GID: 2025-11-28T14:52:23.317876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789227994833899:2108] [node 5] ICS09 handshake done sender: [1:7577789227994833870:2114] self: [1:7577789227994833869:0] peer: [5:7577789229510755663:0] socket: 470 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:23.318102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789227174868665:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.318084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.317329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789227947043079:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:23.318310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:23.318306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789228048703756:2109] [node 5] ICP22 created new session: [2:7577789228048703795:2109]
2025-11-28T14:52:23.317970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789227994833862:2110] [node 6] ICP22 created new session: [1:7577789227994833900:2110]
2025-11-28T14:52:23.318117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789227947043069:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.318771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789228048703795:2109] [node 5] ICS09 handshake done sender: [2:7577789228048703769:2115] self: [2:7577789228048703768:0] peer: [5:7577789229510755664:0] socket: 497 qp: -1
2025-11-28T14:52:23.320698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789229510755668:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15418
2025-11-28T14:52:23.317981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789227994833899:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:23.318918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789228048703795:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:23.318298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789227994833900:2110] [node 6] ICS09 handshake done sender: [1:7577789227994833873:2116] self: [1:7577789227994833872:0] peer: [6:7577789229905680729:0] socket: 490 qp: -1
2025-11-28T14:52:23.320795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680734:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.318731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789227174868664:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.318914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.319813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:23.318321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:23.321842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229905680725:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.319391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789227947043069:2109] [node 5] ICP22 created new session: [3:7577789227947043107:2109]
2025-11-28T14:52:23.322305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789229510755670:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15418
2025-11-28T14:52:23.318553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789227994833900:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:23.318759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.322438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229905680724:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:23.320394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789227947043107:2109] [node 5] ICS09 handshake done sender: [3:7577789227947043079:2116] self: [3:7577789227947043077:0] peer: [5:7577789229510755666:0] socket: 492 qp: -1
2025-11-28T14:52:23.318795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:23.322871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789229510755578:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789229510755660:2118] held: no
2025-11-28T14:52:23.322648Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229905680646:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789229905680726:2117] held: no
2025-11-28T14:52:23.320589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789227947043107:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:23.318770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789227174868665:2109] [node 6] ICP22 created new session: [4:7577789227174868698:2109]
No RDMA context found for GID: 2025-11-28T14:52:23.320736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.0
2025-11-28T14:52:23.319343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789227174868664:2108] [node 5] ICP22 created new session: [4:7577789227174868699:2108]
2025-11-28T14:52:23.322940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229905680647:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.323402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229905680648:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.323576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:23.319420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789227174868698:2109] [node 6] ICS09 handshake done sender: [4:7577789227174868675:2115] self: [4:7577789227174868674:0] peer: [6:7577789229905680732:0] socket: 523 qp: -1
2025-11-28T14:52:23.323590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680735:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
2025-11-28T14:52:23.323712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789229510755579:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789229510755659:2117] held: no
2025-11-28T14:52:23.323606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.323622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229905680647:2049] [node 2] ICP22 created new session: [6:7577789229905680737:2049]
2025-11-28T14:52:23.319823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789227174868698:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:23.323731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229905680737:2049] [node 2] ICS09 handshake done sender: [6:7577789229905680724:2115] self: [6:7577789229905680727:0] peer: [2:7577789228048703766:0] socket: 471 qp: -1
2025-11-28T14:52:23.323874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789229510755580:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789229510755661:2119] held: no
2025-11-28T14:52:23.319855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:23.323732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229905680648:2050] [node 3] ICP22 created new session: [6:7577789229905680738:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.319947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789227174868699:2108] [node 5] ICS09 handshake done sender: [4:7577789227174868676:2116] self: [4:7577789227174868673:0] peer: [5:7577789229510755667:0] socket: 525 qp: -1
2025-11-28T14:52:23.323758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229905680737:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:23.324060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789229510755660:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.323770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229905680738:2050] [node 3] ICS09 handshake done sender: [6:7577789229905680725:2116] self: [6:7577789229905680730:0] peer: [3:7577789227947043080:0] socket: 499 qp: -1
2025-11-28T14:52:23.323846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:23.324063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789229510755669:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15418
2025-11-28T14:52:23.320056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789227174868699:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:23.323861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229905680738:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:23.325101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789229510755659:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:23.323891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:23.320093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:23.325132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229905680726:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.325835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789229510755661:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.325859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789229510755578:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.325868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789229510755579:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.325974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:23.325978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.326004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789229510755579:2049] [node 2] ICP22 created new session: [5:7577789229510755672:2049]
2025-11-28T14:52:23.326021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789229510755578:2048] [node 1] ICP22 created new session: [5:7577789229510755671:2048]
2025-11-28T14:52:23.326065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789229510755672:2049] [node 2] ICS09 handshake done sender: [5:7577789229510755659:2117] self: [5:7577789229510755664:0] peer: [2:7577789228048703768:0] socket: 502 qp: -1
2025-11-28T14:52:23.326116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229905680646:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.326212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229905680728:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:23.326066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789229510755671:2048] [node 1] ICS09 handshake done sender: [5:7577789229510755660:2118] self: [5:7577789229510755663:0] peer: [1:7577789227994833869:0] socket: 493 qp: -1
2025-11-28T14:52:23.326224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.326089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789229510755672:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:23.326258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229905680646:2048] [node 1] ICP22 created new session: [6:7577789229905680742:2048]
2025-11-28T14:52:23.326101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789229510755671:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.326374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229905680742:2048] [node 1] ICS09 handshake done sender: [6:7577789229905680726:2117] self: [6:7577789229905680729:0] peer: [1:7577789227994833872:0] socket: 503 qp: -1
2025-11-28T14:52:23.326112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:23.326502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229905680742:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.326145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.326577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:23.326218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789229510755662:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:23.326634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789229510755580:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.326710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.326757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229905680709:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.326753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789229510755580:2050] [node 3] ICP22 created new session: [5:7577789229510755677:2050]
2025-11-28T14:52:23.326824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:23.326809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789229510755677:2050] [node 3] ICS09 handshake done sender: [5:7577789229510755661:2119] self: [5:7577789229510755666:0] peer: [3:7577789227947043077:0] socket: 522 qp: -1
2025-11-28T14:52:23.326773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789227174868702:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
2025-11-28T14:52:23.326856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229905680709:2108] [node 4] ICP22 created new session: [6:7577789229905680744:2108]
2025-11-28T14:52:23.326833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789229510755677:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:23.326860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:23.326887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229905680744:2108] [node 4] ICS09 handshake done sender: [6:7577789229905680728:2118] self: [6:7577789229905680732:0] peer: [4:7577789227174868674:0] socket: 524 qp: -1
2025-11-28T14:52:23.326932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229905680744:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:23.326936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789229510755643:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.326958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:23.327018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:23.327068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789229510755643:2108] [node 4] ICP22 created new session: [5:7577789229510755679:2108]
2025-11-28T14:52:23.327111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789229510755679:2108] [node 4] ICS09 handshake done sender: [5:7577789229510755662:2120] self: [5:7577789229510755667:0] peer: [4:7577789227174868673:0] socket: 526 qp: -1
2025-11-28T14:52:23.327133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789229510755679:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:23.327151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.327978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789227174868703:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.328156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680746:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
2025-11-28T14:52:23.328794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789227174868665:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:23.329652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229905680746:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.329777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229905680708:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.329949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789229510755655:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.329974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789229510755655:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.330558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789229510755655:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.330719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789229510755642:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.330675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229905680748:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40019
2025-11-28T14:52:23.330805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:23.330854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789229510755642:2109] [node 6] ICP22 created new session: [5:7577789229510755682:2109]
2025-11-28T14:52:23.330891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789229510755682:2109] [node 6] ICS09 handshake done sender: [5:7577789229510755655:2115] self: [5:7577789229510755654:0] peer: [6:7577789229905680747:0] socket: 482 qp: -1
2025-11-28T14:52:23.330917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789229510755682:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:23.330932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229905680746:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:23.330944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:23.331116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229905680708:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.331297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.331340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229905680708:2107] [node 5] ICP22 created new session: [6:7577789229905680749:2107]
2025-11-28T14:52:23.331385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229905680749:2107] [node 5] ICS09 handshake done sender: [6:7577789229905680746:2123] self: [6:7577789229905680747:0] peer: [5:7577789229510755654:0] socket: 500 qp: -1
2025-11-28T14:52:23.331413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229905680749:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:23.331438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:23.331784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789229905680708:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:23.345951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:23.346181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789228048703789:2048] [node 1] ICS01 socket: 496 reason# EndOfStream
2025-11-28T14:52:23.346223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789228048703690:2048] [node 1] ICP30 unregister session Session# [2:7577789228048703789:2048] VirtualId# [2:7577789228048703775:0]
2025-11-28T14:52:23.346277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789228048703789:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.346375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.346429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.346770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:23.346958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789227947043101:2048] [node 1] ICS07 socket disconnect 483 reason# EndOfStream
2025-11-28T14:52:23.346990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789227947043101:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.347081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.347146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789227947043101:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:23.347901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789227947043003:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6047) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.348047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789227947043101:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.348100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789227947043003:2048] [node 1] ICP30 unregister session Session# [3:7577789227947043101:2048] VirtualId# [3:7577789227947043089:0]
2025-11-28T14:52:23.348175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.348217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789227947043003:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.348237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.349185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:23.349333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789227174868691:2048] [node 1] ICS07 socket disconnect 488 reason# EndOfStream
2025-11-28T14:52:23.349352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789227174868691:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.349412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.349459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789227174868691:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:23.349830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:23.349959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789229510755671:2048] [node 1] ICS01 socket: 493 reason# EndOfStream
2025-11-28T14:52:23.349998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789229510755578:2048] [node 1] ICP30 unregister session Session# [5:7577789229510755671:2048] VirtualId# [5:7577789229510755663:0]
2025-11-28T14:52:23.350034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789229510755671:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.350097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.350136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.350062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789227174868600:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6047) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.350129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789227174868691:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.350173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789227174868600:2048] [node 1] ICP30 unregister session Session# [4:7577789227174868691:2048] VirtualId# [4:7577789227174868684:0]
2025-11-28T14:52:23.350278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.350309Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789227174868600:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.350322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.350599Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:23.350748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229905680742:2048] [node 1] ICS01 socket: 503 reason# EndOfStream
2025-11-28T14:52:23.350773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229905680646:2048] [node 1] ICP30 unregister session Session# [6:7577789229905680742:2048] VirtualId# [6:7577789229905680729:0]
2025-11-28T14:52:23.350789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229905680742:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.350850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.350910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.358089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789227947043003:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:23.360144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789227174868600:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:23.389668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:23.389862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789227947043098:2049] [node 2] ICS07 socket disconnect 473 reason# EndOfStream
2025-11-28T14:52:23.389893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789227947043098:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.389985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:23.390053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789227947043098:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:23.390837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789227947043004:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24171) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.390914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789227947043098:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.390980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789227947043004:2049] [node 2] ICP30 unregister session Session# [3:7577789227947043098:2049] VirtualId# [3:7577789227947043088:0]
2025-11-28T14:52:23.391050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:23.391086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789227947043004:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.391104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.392432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:23.392643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789227174868688:2049] [node 2] ICS07 socket disconnect 474 reason# EndOfStream
2025-11-28T14:52:23.392672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789227174868688:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.392787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:23.392834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789227174868688:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:23.393157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:23.393289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789229510755672:2049] [node 2] ICS01 socket: 502 reason# EndOfStream
2025-11-28T14:52:23.393317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789229510755579:2049] [node 2] ICP30 unregister session Session# [5:7577789229510755672:2049] VirtualId# [5:7577789229510755664:0]
2025-11-28T14:52:23.393337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789229510755672:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.393400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:23.393439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:23.393469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789227174868601:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24171) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.393512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789227174868688:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.393532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789227174868601:2049] [node 2] ICP30 unregister session Session# [4:7577789227174868688:2049] VirtualId# [4:7577789227174868682:0]
2025-11-28T14:52:23.393570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:23.393591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789227174868601:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.393601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.393749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:23.393892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229905680737:2049] [node 2] ICS01 socket: 471 reason# EndOfStream
2025-11-28T14:52:23.393916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229905680647:2049] [node 2] ICP30 unregister session Session# [6:7577789229905680737:2049] VirtualId# [6:7577789229905680727:0]
2025-11-28T14:52:23.393943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229905680737:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.393991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:23.394015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:23.401132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789227947043004:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:23.403164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789227174868601:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:23.441322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:23.441472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789227174868690:2050] [node 3] ICS01 socket: 501 reason# EndOfStream
2025-11-28T14:52:23.441513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789227174868602:2050] [node 3] ICP30 unregister session Session# [4:7577789227174868690:2050] VirtualId# [4:7577789227174868683:0]
2025-11-28T14:52:23.441536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789227174868690:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.441599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:23.441644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:23.442135Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:23.442375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789229510755677:2050] [node 3] ICS01 socket: 522 reason# EndOfStream
2025-11-28T14:52:23.442419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789229510755580:2050] [node 3] ICP30 unregister session Session# [5:7577789229510755677:2050] VirtualId# [5:7577789229510755666:0]
2025-11-28T14:52:23.442455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789229510755677:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.442529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:23.442598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:23.442998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:23.443231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789229905680738:2050] [node 3] ICS07 socket disconnect 499 reason# EndOfStream
2025-11-28T14:52:23.443277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229905680738:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.443355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:23.443408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789229905680738:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:23.444012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229905680648:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18171) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.444058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229905680738:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.444083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229905680648:2050] [node 3] ICP30 unregister session Session# [6:7577789229905680738:2050] VirtualId# [6:7577789229905680730:0]
2025-11-28T14:52:23.444120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:23.444146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789229905680648:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.444160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18171) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.454396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789229905680648:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:23.474163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:23.474389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789229510755679:2108] [node 4] ICS07 socket disconnect 526 reason# EndOfStream
2025-11-28T14:52:23.474421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789229510755679:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.474494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:23.474537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789229510755679:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:23.475052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:23.475153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789229510755643:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.475209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789229510755679:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.475274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789229510755643:2108] [node 4] ICP30 unregister session Session# [5:7577789229510755679:2108] VirtualId# [5:7577789229510755667:0]
2025-11-28T14:52:23.475244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789229905680744:2108] [node 4] ICS07 socket disconnect 524 reason# EndOfStream
2025-11-28T14:52:23.475316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:23.475273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229905680744:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.475351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789229510755643:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.475371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.475360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:23.475398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789229905680744:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:23.476015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229905680709:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.476056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229905680744:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.476132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229905680709:2108] [node 4] ICP30 unregister session Session# [6:7577789229905680744:2108] VirtualId# [6:7577789229905680732:0]
2025-11-28T14:52:23.476167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:23.476193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789229905680709:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.476209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.485119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789229510755643:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:23.486102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789229905680709:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:23.498664Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:23.498815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789229905680749:2107] [node 5] ICS07 socket disconnect 500 reason# EndOfStream
2025-11-28T14:52:23.498837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229905680749:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.498899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:23.498929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789229905680749:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:23.499558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229905680708:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.499609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229905680749:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.499664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229905680708:2107] [node 5] ICP30 unregister session Session# [6:7577789229905680749:2107] VirtualId# [6:7577789229905680747:0]
2025-11-28T14:52:23.499707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:23.499733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789229905680708:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.499741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:23.570246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789228870113657:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:23.570245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789228870113658:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:23.591217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789227938850845:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.591395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789227938850846:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:23.610408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789226472337572:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.610523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789226472337573:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:23.634283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789226727773065:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.634314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789226727773066:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:23.634330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789226727773067:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:23.659613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789226787562281:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.659640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789226787562282:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:23.659670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789226787562283:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:23.679573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229381543708:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:23.679595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229381543709:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:23.679874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229381543710:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:23.778201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789228870113720:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:23.773256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789227938850909:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:23.778796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789226472337637:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:23.778294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229381543767:2104] [node 5] ICP01 ready to work
2025-11-28T14:52:23.779125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789226787562344:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:23.778367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789227938850910:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:23.779465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789226727773147:2116] [node 6] ICP01 ready to work
2025-11-28T14:52:23.779268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789226472337638:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:23.779882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789228870113722:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:23.779684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789226787562345:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:23.780108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789229381543770:2103] [node 4] ICP01 ready to work
2025-11-28T14:52:23.779551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789226727773146:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:23.781277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789226472337639:2111] [node 6] ICP01 ready to work
2025-11-28T14:52:23.779896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789228870113721:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:23.781886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789227938850911:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:23.785143Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789226472337572:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47821571ns) {Configure:104(47804287ns) {ProcessConfigured:135(313970ns) {StartInitialHandshake:157(311582ns) {DropHandshakes:470(2138ns) {DropIncomingHandshake:442(543ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(810ns) PrepareNewSessionHandshake:477(2035ns) {SwitchToState:179(1150ns)}}}}}
2025-11-28T14:52:23.786080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789228870113658:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49063014ns) {Configure:104(49057784ns) {ProcessConfigured:135(206274ns) {StartInitialHandshake:157(204500ns) {DropHandshakes:470(1484ns) {DropIncomingHandshake:442(434ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(440ns) PrepareNewSessionHandshake:477(1128ns) {SwitchToState:179(683ns)}}}}}
2025-11-28T14:52:23.786695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789227938850846:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51735724ns) {Configure:104(51731548ns) {ProcessConfigured:135(221883ns) {StartInitialHandshake:157(220203ns) {DropHandshakes:470(1139ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(768ns) {SwitchToState:179(297ns)}}}}}
2025-11-28T14:52:23.786293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789228870113657:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(53744415ns) {Configure:104(53738881ns) {ProcessConfigured:135(413633ns) {StartInitialHandshake:157(412122ns) {DropHandshakes:470(51173ns) {DropIncomingHandshake:442(727ns) DropOutgoingHandshake:457(317ns)} GenerateSessionVirtualId:988(1021ns) PrepareNewSessionHandshake:477(807ns) {SwitchToState:179(299ns)}}}}}
2025-11-28T14:52:23.786771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789227938850845:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49426701ns) {Configure:104(49421177ns) {ProcessConfigured:135(477088ns) {StartInitialHandshake:157(475594ns) {DropHandshakes:470(898ns) {DropIncomingHandshake:442(225ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(277ns) PrepareNewSessionHandshake:477(1957ns) {SwitchToState:179(677ns)}}}}}
2025-11-28T14:52:23.788292Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789229381543708:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27790003ns) {Configure:104(27782412ns) {ProcessConfigured:135(321117ns) {StartInitialHandshake:157(319330ns) {DropHandshakes:470(891ns) {DropIncomingHandshake:442(218ns) DropOutgoingHandshake:457(161ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(1341ns) {SwitchToState:179(579ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.794071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789228870113732:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43859
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.794774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789227938850920:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33592
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:23.797513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789226727773158:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29507
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.798588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789227938850924:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33592
2025-11-28T14:52:23.798255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789226787562357:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.798274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789226727773160:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29507
2025-11-28T14:52:23.798279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789228870113736:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43859
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.799727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789227938850925:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33592
2025-11-28T14:52:23.798335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789228870113739:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43859
2025-11-28T14:52:23.802758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789226472337653:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45863
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.800747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789228870113738:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43859
No RDMA context found for GID: 2025-11-28T14:52:23.806524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789227938850925:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.805475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789226787562357:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:23.805804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789228870113740:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43859
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.807059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789226727773161:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29507
2025-11-28T14:52:23.807313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789226472337654:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45863
2025-11-28T14:52:23.807124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789227938850920:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.807552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789227938850924:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.806541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113738:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.807422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789226472337657:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45863
2025-11-28T14:52:23.807116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789226727773158:2132] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789227938850845:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.807785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789226472337655:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45863
2025-11-28T14:52:23.807350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113736:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.807737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789226727773067:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.808122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789226472337656:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45863
2025-11-28T14:52:23.808095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789226727773161:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789226472337653:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789226727773164:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29507
2025-11-28T14:52:23.807599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113732:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789226472337645:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789226727773160:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789226472337654:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.807879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113739:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789226727773065:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.808745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113740:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789226472337645:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.808817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113726:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789226472337655:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.810718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789226727773164:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.808840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789228870113726:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.808880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789226472337572:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.810189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113728:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.812178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789226727773066:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.812257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789227938850914:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.810210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789228870113728:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.810202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789226472337656:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.812293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789227938850914:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.810259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113725:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.810549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789226472337657:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.812426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789227938850917:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.810282Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789228870113725:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.811000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789226472337573:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.812449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789227938850917:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.811145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789228870113726:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:23.811424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789226472337645:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:23.813198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789228870113725:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.813411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789226472337637:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.814403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789227938850931:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33592
2025-11-28T14:52:23.813256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789228870113728:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.814550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789227938850914:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.813697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:23.813825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789228870113657:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.813835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789226472337637:2107] [node 4] ICP22 created new session: [3:7577789226472337666:2107]
2025-11-28T14:52:23.813874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:23.814150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789226472337666:2107] [node 4] ICS09 handshake done sender: [3:7577789226472337645:2114] self: [3:7577789226472337644:0] peer: [4:7577789226727773165:0] socket: 478 qp: -1
2025-11-28T14:52:23.815108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789226727773067:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10163) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789226727773158:2132] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.814389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789226472337666:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:23.815127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789226727773169:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29507
2025-11-28T14:52:23.814359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789228870113657:2048] [node 2] ICP22 created new session: [1:7577789228870113749:2048]
2025-11-28T14:52:23.815837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789227938850917:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:23.814435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:23.815081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789226472337665:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45863
2025-11-28T14:52:23.815127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789226727773170:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29507
2025-11-28T14:52:23.814411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789228870113749:2048] [node 2] ICS09 handshake done sender: [1:7577789228870113726:2111] self: [1:7577789228870113724:0] peer: [2:7577789227938850930:0] socket: 480 qp: -1
2025-11-28T14:52:23.817983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789226787562361:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-28T14:52:23.817821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789227938850924:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.815702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789226472337654:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.814449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789228870113749:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.819341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789226727773158:2132] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.819262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789226787562361:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.818111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789226472337572:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.814855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789228870113720:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.817959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789227938850846:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.819032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.818031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789227938850845:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789227938850924:2116] held: no
2025-11-28T14:52:23.819540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789226472337572:2048] [node 1] ICP22 created new session: [3:7577789226472337669:2048]
2025-11-28T14:52:23.820338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789226727773160:2133] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.814938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:23.818574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789227938850845:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.821267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789226787562283:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.815588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789228870113658:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.820348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789226727773065:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789226727773160:2133] held: no
No RDMA context found for GID: 2025-11-28T14:52:23.819746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789226472337668:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45863
::ffff:127.1.0.0
2025-11-28T14:52:23.818594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789227938850909:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.821301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789226727773065:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.815622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:23.821352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.820317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789226472337669:2048] [node 1] ICS09 handshake done sender: [3:7577789226472337654:2118] self: [3:7577789226472337660:0] peer: [1:7577789228870113723:0] socket: 495 qp: -1
2025-11-28T14:52:23.818611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.821009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789226472337669:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.821394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789226727773065:2048] [node 1] ICP22 created new session: [4:7577789226727773172:2048]
2025-11-28T14:52:23.818761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:23.822036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:23.815825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789228870113720:2107] [node 4] ICP22 created new session: [1:7577789228870113751:2107]
2025-11-28T14:52:23.819084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789227938850909:2107] [node 4] ICP22 created new session: [2:7577789227938850935:2107]
2025-11-28T14:52:23.821515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789226727773067:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.819304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789227938850845:2048] [node 1] ICP22 created new session: [2:7577789227938850934:2048]
2025-11-28T14:52:23.817001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789228870113751:2107] [node 4] ICS09 handshake done sender: [1:7577789228870113728:2112] self: [1:7577789228870113727:0] peer: [4:7577789226727773167:0] socket: 489 qp: -1
2025-11-28T14:52:23.822703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789226472337573:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14467) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789226472337657:2121] held: no
2025-11-28T14:52:23.821558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.819308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.817368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789228870113751:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:23.822712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789226472337648:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.821768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789226727773067:2050] [node 3] ICP22 created new session: [4:7577789226727773174:2050]
2025-11-28T14:52:23.819555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789227938850846:2049] [node 3] ICP22 created new session: [2:7577789227938850936:2049]
2025-11-28T14:52:23.817458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.822737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789226472337648:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.819586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789227938850936:2049] [node 3] ICS09 handshake done sender: [2:7577789227938850914:2111] self: [2:7577789227938850913:0] peer: [3:7577789226472337662:0] socket: 504 qp: -1
2025-11-28T14:52:23.823548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789226472337657:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:23.817746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:23.822124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789226727773172:2048] [node 1] ICS09 handshake done sender: [4:7577789226727773160:2133] self: [4:7577789226727773167:0] peer: [1:7577789228870113727:0] socket: 493 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.819725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789227938850934:2048] [node 1] ICS09 handshake done sender: [2:7577789227938850924:2116] self: [2:7577789227938850930:0] peer: [1:7577789228870113724:0] socket: 497 qp: -1
2025-11-28T14:52:23.826244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789226472337573:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.817780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789228870113658:2049] [node 3] ICP22 created new session: [1:7577789228870113753:2049]
2025-11-28T14:52:23.819744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789227938850934:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.822372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789226727773172:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.819760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789227938850935:2107] [node 4] ICS09 handshake done sender: [2:7577789227938850917:2113] self: [2:7577789227938850916:0] peer: [4:7577789226727773168:0] socket: 513 qp: -1
2025-11-28T14:52:23.826384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:23.822371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789226727773174:2050] [node 3] ICS09 handshake done sender: [4:7577789226727773158:2132] self: [4:7577789226727773165:0] peer: [3:7577789226472337644:0] socket: 486 qp: -1
2025-11-28T14:52:23.818255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789228870113753:2049] [node 3] ICS09 handshake done sender: [1:7577789228870113725:2110] self: [1:7577789228870113723:0] peer: [3:7577789226472337660:0] socket: 485 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.826684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789226472337573:2049] [node 2] ICP22 created new session: [3:7577789226472337673:2049]
2025-11-28T14:52:23.822372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789226727773171:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29507
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.826679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789226787562363:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-28T14:52:23.820324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789227938850936:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:23.818553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789228870113753:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:23.820989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:23.820996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789227938850935:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:23.826948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789226472337673:2049] [node 2] ICS09 handshake done sender: [3:7577789226472337657:2121] self: [3:7577789226472337662:0] peer: [2:7577789227938850913:0] socket: 511 qp: -1
2025-11-28T14:52:23.821038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:23.827085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789226472337673:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:23.822395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789226727773174:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:23.821262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:23.827109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:23.818588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:23.827321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789226472337648:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.827535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789226787562364:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.827565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543787:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.822399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.822422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.827970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789226787562365:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
No RDMA context found for GID: 2025-11-28T14:52:23.824088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789226727773066:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14467) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789226727773164:2136] held: no
::ffff:127.1.0.0
2025-11-28T14:52:23.828030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789226787562366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-28T14:52:23.827976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543788:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.825866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789226727773164:2136] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:23.827507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789226727773066:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.828182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543789:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.828961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789226787562364:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.827677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:23.828386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789226472337638:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.829297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789226787562281:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.827968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789226727773066:2049] [node 2] ICP22 created new session: [4:7577789226727773178:2049]
2025-11-28T14:52:23.828320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229381543787:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.829271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.828026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789226727773178:2049] [node 2] ICS09 handshake done sender: [4:7577789226727773164:2136] self: [4:7577789226727773168:0] peer: [2:7577789227938850916:0] socket: 514 qp: -1
2025-11-28T14:52:23.828962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543790:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.828148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789226727773178:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:23.829491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789226787562361:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.828183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:23.829291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229381543708:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.829345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789226472337638:2109] [node 5] ICP22 created new session: [3:7577789226472337675:2109]
2025-11-28T14:52:23.829488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229381543789:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.829372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789226472337675:2109] [node 5] ICS09 handshake done sender: [3:7577789226472337648:2115] self: [3:7577789226472337647:0] peer: [5:7577789226787562362:0] socket: 524 qp: -1
2025-11-28T14:52:23.830124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113730:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.829511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789226787562366:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.830159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789228870113730:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.829510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789226472337675:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:23.829776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229381543709:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.830687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789226787562282:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.830936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789227938850926:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.830899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789228870113737:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.830676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229381543790:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.829828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:23.830731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789226787562283:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.830925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789228870113737:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.830991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229381543770:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.830961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789227938850919:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.830971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789227938850926:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.833258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789226727773155:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.830797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789226787562363:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.831279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229381543788:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.833342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789228870113730:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:23.830975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789227938850919:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.830913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.833281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789226727773163:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.834787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789228870113721:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.833286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789226727773155:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.834749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789227938850919:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:23.833301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229381543710:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.830932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789226787562344:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.835204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789227938850926:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:23.830972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789226787562283:2050] [node 3] ICP22 created new session: [5:7577789226787562369:2050]
2025-11-28T14:52:23.834916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789226472337652:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.835726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789227938850910:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.832879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789226787562369:2050] [node 3] ICS09 handshake done sender: [5:7577789226787562361:2116] self: [5:7577789226787562362:0] peer: [3:7577789226472337647:0] socket: 525 qp: -1
2025-11-28T14:52:23.834839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.835760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:23.835169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789226472337652:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.833281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789226787562369:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:23.834884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789228870113721:2108] [node 5] ICP22 created new session: [1:7577789228870113759:2108]
2025-11-28T14:52:23.833309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789226727773163:2135] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.836953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789227938850910:2110] [node 6] ICP22 created new session: [2:7577789227938850945:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.833320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:23.835196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789226727773163:2135] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:23.837336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789227938850945:2110] [node 6] ICS09 handshake done sender: [2:7577789227938850919:2114] self: [2:7577789227938850918:0] peer: [6:7577789229381543792:0] socket: 508 qp: -1
2025-11-28T14:52:23.837675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789226472337652:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:23.835191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789228870113737:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:23.837075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789226727773155:2131] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:23.837529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789227938850911:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.836557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789228870113759:2108] [node 5] ICS09 handshake done sender: [1:7577789228870113730:2113] self: [1:7577789228870113729:0] peer: [5:7577789226787562367:0] socket: 476 qp: -1
2025-11-28T14:52:23.838530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543796:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.837084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789226727773147:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.837843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789227938850945:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:23.837070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789228870113759:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:23.838833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789226472337639:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.837387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:23.838944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229381543708:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789229381543787:2116] held: no
2025-11-28T14:52:23.837553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:23.837585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.837547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789228870113722:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.839195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789226787562281:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789226787562364:2118] held: no
No RDMA context found for GID: 2025-11-28T14:52:23.838616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:23.837922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789226727773146:2108] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:23.839733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-11-28T14:52:23.837891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789227938850911:2108] [node 5] ICP22 created new session: [2:7577789227938850946:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.841409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543798:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.838845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789228870113722:2109] [node 6] ICP22 created new session: [1:7577789228870113761:2109]
2025-11-28T14:52:23.838193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789226727773147:2116] [node 6] ICP22 created new session: [4:7577789226727773182:2116]
2025-11-28T14:52:23.839890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789228870113761:2109] [node 6] ICS09 handshake done sender: [1:7577789228870113737:2116] self: [1:7577789228870113735:0] peer: [6:7577789229381543791:0] socket: 477 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:23.838567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789226727773182:2116] [node 6] ICS09 handshake done sender: [4:7577789226727773163:2135] self: [4:7577789226727773162:0] peer: [6:7577789229381543793:0] socket: 532 qp: -1
2025-11-28T14:52:23.839937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789228870113761:2109] [node 6] ICS10 traffic start
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.841620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543797:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.838195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:23.841630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543795:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.840416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789226472337639:2111] [node 6] ICP22 created new session: [3:7577789226472337678:2111]
2025-11-28T14:52:23.840412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:23.838598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789226727773182:2116] [node 6] ICS10 traffic start
2025-11-28T14:52:23.838566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789227938850946:2108] [node 5] ICS09 handshake done sender: [2:7577789227938850926:2118] self: [2:7577789227938850923:0] peer: [5:7577789226787562368:0] socket: 529 qp: -1
2025-11-28T14:52:23.842010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789226472337678:2111] [node 6] ICS09 handshake done sender: [3:7577789226472337652:2116] self: [3:7577789226472337650:0] peer: [6:7577789229381543794:0] socket: 475 qp: -1
2025-11-28T14:52:23.842296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229381543787:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.842011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789226787562374:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-28T14:52:23.838638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:23.838574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789227938850944:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33592
2025-11-28T14:52:23.842284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789226787562373:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-28T14:52:23.842785Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229381543710:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10163) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789229381543788:2117] held: no
2025-11-28T14:52:23.842198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789226472337678:2111] [node 6] ICS10 traffic start
2025-11-28T14:52:23.842719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789226787562344:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789226787562363:2117] held: no
2025-11-28T14:52:23.839195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.843540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229381543708:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.842247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:23.843783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229381543790:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:23.839732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789226727773146:2108] [node 5] ICP22 created new session: [4:7577789226727773184:2108]
2025-11-28T14:52:23.843831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229381543789:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:23.841608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789226727773184:2108] [node 5] ICS09 handshake done sender: [4:7577789226727773155:2131] self: [4:7577789226727773150:0] peer: [5:7577789226787562370:0] socket: 474 qp: -1
2025-11-28T14:52:23.838651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789227938850946:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:23.841655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789226727773184:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:23.843924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.838673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:23.844073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789226787562366:2120] [node 2] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:23.841711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:23.844240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789226787562364:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:23.841675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789227938850944:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.844218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229381543788:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:23.844934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789226787562375:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-28T14:52:23.844785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789227938850911:2108] [node 5] ICP18 handshake (actor [2:7577789227938850944:2120]) is too old
2025-11-28T14:52:23.845160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789226787562282:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.844220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229381543708:2048] [node 1] ICP22 created new session: [6:7577789229381543801:2048]
2025-11-28T14:52:23.844265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229381543801:2048] [node 1] ICS09 handshake done sender: [6:7577789229381543787:2116] self: [6:7577789229381543791:0] peer: [1:7577789228870113735:0] socket: 499 qp: -1
2025-11-28T14:52:23.845318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789226787562281:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.844516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229381543801:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.845356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:23.844788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:23.845505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789226787562282:2049] [node 2] ICP22 created new session: [5:7577789226787562377:2049]
2025-11-28T14:52:23.845028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229381543770:2103] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.845610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789226787562377:2049] [node 2] ICS09 handshake done sender: [5:7577789226787562366:2120] self: [5:7577789226787562368:0] peer: [2:7577789227938850923:0] socket: 530 qp: -1
2025-11-28T14:52:23.845052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229381543709:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.845674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789226787562377:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:23.845198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:23.845693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: 2025-11-28T14:52:23.845222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-28T14:52:23.845715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:23.846789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789227938850949:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33592
2025-11-28T14:52:23.845795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789226787562281:2048] [node 1] ICP22 created new session: [5:7577789226787562379:2048]
2025-11-28T14:52:23.845380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229381543770:2103] [node 4] ICP22 created new session: [6:7577789229381543803:2103]
2025-11-28T14:52:23.846107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789226787562379:2048] [node 1] ICS09 handshake done sender: [5:7577789226787562364:2118] self: [5:7577789226787562367:0] peer: [1:7577789228870113729:0] socket: 502 qp: -1
2025-11-28T14:52:23.845394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229381543709:2049] [node 2] ICP22 created new session: [6:7577789229381543804:2049]
2025-11-28T14:52:23.846130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789226787562379:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:23.845410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229381543710:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.847080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789226727773186:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29507
2025-11-28T14:52:23.846584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:23.845643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:23.846585Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789226787562363:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:23.847153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789226787562344:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.845702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229381543803:2103] [node 4] ICS09 handshake done sender: [6:7577789229381543790:2119] self: [6:7577789229381543793:0] peer: [4:7577789226727773162:0] socket: 533 qp: -1
2025-11-28T14:52:23.845720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229381543710:2050] [node 3] ICP22 created new session: [6:7577789229381543806:2050]
2025-11-28T14:52:23.847236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:23.845735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229381543803:2103] [node 4] ICS10 traffic start
2025-11-28T14:52:23.845762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:23.847281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789226787562344:2108] [node 4] ICP22 created new session: [5:7577789226787562381:2108]
2025-11-28T14:52:23.845765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229381543806:2050] [node 3] ICS09 handshake done sender: [6:7577789229381543788:2117] self: [6:7577789229381543794:0] peer: [3:7577789226472337650:0] socket: 487 qp: -1
2025-11-28T14:52:23.845787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229381543804:2049] [node 2] ICS09 handshake done sender: [6:7577789229381543789:2118] self: [6:7577789229381543792:0] peer: [2:7577789227938850918:0] socket: 531 qp: -1
2025-11-28T14:52:23.847330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789226787562381:2108] [node 4] ICS09 handshake done sender: [5:7577789226787562363:2117] self: [5:7577789226787562370:0] peer: [4:7577789226727773150:0] socket: 490 qp: -1
2025-11-28T14:52:23.845917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229381543804:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:23.847363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789226787562381:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:23.847472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:23.845945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:23.846053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229381543806:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:23.846086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.848026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543805:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.849078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789229381543805:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.849256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789229381543767:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:23.849399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789226787562359:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:23.849425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789226787562359:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:23.850193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789226787562359:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:23.850383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789226787562345:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:23.850400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789229381543811:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-11-28T14:52:23.850454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:23.850493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789226787562345:2109] [node 6] ICP22 created new session: [5:7577789226787562384:2109]
2025-11-28T14:52:23.850532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789226787562384:2109] [node 6] ICS09 handshake done sender: [5:7577789226787562359:2115] self: [5:7577789226787562358:0] peer: [6:7577789229381543810:0] socket: 481 qp: -1
2025-11-28T14:52:23.850563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789226787562384:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:23.850601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:23.850647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789229381543805:2124] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:23.850914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229381543767:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20966) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789229381543805:2124] held: no
2025-11-28T14:52:23.851515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789229381543767:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:23.851554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:23.851651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789229381543767:2104] [node 5] ICP22 created new session: [6:7577789229381543812:2104]
2025-11-28T14:52:23.851706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789229381543812:2104] [node 5] ICS09 handshake done sender: [6:7577789229381543805:2124] self: [6:7577789229381543810:0] peer: [5:7577789226787562358:0] socket: 492 qp: -1
2025-11-28T14:52:23.851749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789229381543812:2104] [node 5] ICS10 traffic start
2025-11-28T14:52:23.851788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:23.883840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:23.884018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789227938850934:2048] [node 1] ICS01 socket: 497 reason# EndOfStream
2025-11-28T14:52:23.884057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789227938850845:2048] [node 1] ICP30 unregister session Session# [2:7577789227938850934:2048] VirtualId# [2:7577789227938850930:0]
2025-11-28T14:52:23.884085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789227938850934:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.884144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.884179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.885017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:23.885142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789226472337669:2048] [node 1] ICS07 socket disconnect 495 reason# EndOfStream
2025-11-28T14:52:23.885164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789226472337669:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.885209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.885253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789226472337669:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:23.887215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789226472337572:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21419) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:23.887245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789226472337669:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:23.888033Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:23.888337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789226727773172:2048] [node 1] ICS01 socket: 493 reason# EndOfStream
2025-11-28T14:52:23.888408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789226727773065:2048] [node 1] ICP30 unregister session Session# [4:7577789226727773172:2048] VirtualId# [4:7577789226727773167:0]
2025-11-28T14:52:23.888445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789226727773172:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.888529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.888586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.888564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789226472337572:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21419) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.888786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789226472337669:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.888848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789226472337572:2048] [node 1] ICP30 unregister session Session# [3:7577789226472337669:2048] VirtualId# [3:7577789226472337660:0]
2025-11-28T14:52:23.888917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.888962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789226472337572:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.888984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.889277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:23.889419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789226787562379:2048] [node 1] ICS01 socket: 502 reason# EndOfStream
2025-11-28T14:52:23.889443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789226787562281:2048] [node 1] ICP30 unregister session Session# [5:7577789226787562379:2048] VirtualId# [5:7577789226787562367:0]
2025-11-28T14:52:23.889466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789226787562379:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.889520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.889554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.890191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:23.890371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229381543801:2048] [node 1] ICS01 socket: 499 reason# EndOfStream
2025-11-28T14:52:23.890419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229381543708:2048] [node 1] ICP30 unregister session Session# [6:7577789229381543801:2048] VirtualId# [6:7577789229381543791:0]
2025-11-28T14:52:23.890444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229381543801:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.890518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:23.890567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:23.898785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789226472337572:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:23.956326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:23.956511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789226472337673:2049] [node 2] ICS01 socket: 511 reason# EndOfStream
2025-11-28T14:52:23.956556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789226472337573:2049] [node 2] ICP30 unregister session Session# [3:7577789226472337673:2049] VirtualId# [3:7577789226472337662:0]
2025-11-28T14:52:23.956577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789226472337673:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.956651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:23.956699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:23.959155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:23.959372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789226727773178:2049] [node 2] ICS01 socket: 514 reason# EndOfStream
2025-11-28T14:52:23.959428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789226727773066:2049] [node 2] ICP30 unregister session Session# [4:7577789226727773178:2049] VirtualId# [4:7577789226727773168:0]
2025-11-28T14:52:23.959456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789226727773178:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.959530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:23.959586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:23.960158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:23.960289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789226787562377:2049] [node 2] ICS07 socket disconnect 530 reason# EndOfStream
2025-11-28T14:52:23.960318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789226787562377:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.960395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:23.960431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789226787562377:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:23.960949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789226787562282:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14467) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.961041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789226787562377:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.961073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789226787562282:2049] [node 2] ICP30 unregister session Session# [5:7577789226787562377:2049] VirtualId# [5:7577789226787562368:0]
2025-11-28T14:52:23.961122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:23.961154Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789226787562282:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14467) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.961167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14467) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.961207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:23.961411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789229381543804:2049] [node 2] ICS07 socket disconnect 531 reason# EndOfStream
2025-11-28T14:52:23.961445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229381543804:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:23.961515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:23.961559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789229381543804:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:23.962253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229381543709:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14467) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:23.962398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229381543804:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:23.962436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229381543709:2049] [node 2] ICP30 unregister session Session# [6:7577789229381543804:2049] VirtualId# [6:7577789229381543792:0]
2025-11-28T14:52:23.962482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:23.962519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789229381543709:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14467) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.962532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14467) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:23.971482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789226787562282:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:23.972476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789229381543709:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:24.011127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:24.011346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789226727773174:2050] [node 3] ICS01 socket: 486 reason# EndOfStream
2025-11-28T14:52:24.011393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789226727773067:2050] [node 3] ICP30 unregister session Session# [4:7577789226727773174:2050] VirtualId# [4:7577789226727773165:0]
2025-11-28T14:52:24.011421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789226727773174:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.011504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:24.011558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:24.012148Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:24.012268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789226787562369:2050] [node 3] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-28T14:52:24.012302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789226787562369:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.012353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:24.012414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789226787562369:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:24.013056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:24.013160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789226787562283:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10163) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:24.013327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229381543806:2050] [node 3] ICS01 socket: 487 reason# EndOfStream
2025-11-28T14:52:24.013265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789226787562369:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:24.013296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789226787562283:2050] [node 3] ICP30 unregister session Session# [5:7577789226787562369:2050] VirtualId# [5:7577789226787562362:0]
2025-11-28T14:52:24.013375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229381543710:2050] [node 3] ICP30 unregister session Session# [6:7577789229381543806:2050] VirtualId# [6:7577789229381543794:0]
2025-11-28T14:52:24.013369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:24.013409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229381543806:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.013401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789226787562283:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10163) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.013416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10163) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.013480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:24.013542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:24.023706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789226787562283:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:24.061153Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:24.061391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789226787562381:2108] [node 4] ICS01 socket: 490 reason# EndOfStream
2025-11-28T14:52:24.061437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789226787562344:2108] [node 4] ICP30 unregister session Session# [5:7577789226787562381:2108] VirtualId# [5:7577789226787562370:0]
2025-11-28T14:52:24.061464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789226787562381:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.061544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:24.061591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:24.062185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:24.062394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789229381543803:2103] [node 4] ICS07 socket disconnect 533 reason# EndOfStream
2025-11-28T14:52:24.062432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229381543803:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.062508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:24.062557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789229381543803:2103] [node 4] ICS15 start handshake
2025-11-28T14:52:24.063305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789229381543770:2103] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17267) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:24.063372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229381543803:2103] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:24.063418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229381543770:2103] [node 4] ICP30 unregister session Session# [6:7577789229381543803:2103] VirtualId# [6:7577789229381543793:0]
2025-11-28T14:52:24.063459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:24.063496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789229381543770:2103] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17267) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.063509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17267) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.072901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789229381543770:2103] [node 4] ICP33 wake up from error state
2025-11-28T14:52:24.080818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:24.081041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789229381543812:2104] [node 5] ICS01 socket: 492 reason# EndOfStream
2025-11-28T14:52:24.081088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789229381543767:2104] [node 5] ICP30 unregister session Session# [6:7577789229381543812:2104] VirtualId# [6:7577789229381543810:0]
2025-11-28T14:52:24.081122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789229381543812:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.081208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:24.081260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:24.189422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789231079213823:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:24.189439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789231079213824:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:24.209585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230631248125:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.209656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230631248126:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:24.226297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789232566317802:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.226327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789232566317803:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:24.246685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789233599599750:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.246713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789233599599752:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:24.246727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789233599599751:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:24.267381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789230903623634:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:24.267384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789230903623633:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.267501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789230903623635:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:24.286439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789231010527904:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.286480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789231010527906:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:24.286932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789231010527905:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:24.357291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789231079213888:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:24.377188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789231079213889:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:24.378090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789232566317865:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:24.378905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789230903623698:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:24.378410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789232566317866:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:24.379223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789233599599813:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:24.378859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230631248188:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:24.379682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789230903623697:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:24.379679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789231010527967:2104] [node 4] ICP01 ready to work
2025-11-28T14:52:24.379447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230631248189:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:24.380320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230631248190:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:24.379431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789231079213890:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:24.379725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789232566317867:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:24.380345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789233599599814:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:24.380613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789231010527968:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:24.383235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789231079213823:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(53113555ns) {Configure:104(53108129ns) {ProcessConfigured:135(324932ns) {StartInitialHandshake:157(322521ns) {DropHandshakes:470(1528ns) {DropIncomingHandshake:442(417ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(561ns) PrepareNewSessionHandshake:477(1136ns) {SwitchToState:179(448ns)}}}}}
2025-11-28T14:52:24.383764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789231079213824:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52706731ns) {Configure:104(52699026ns) {ProcessConfigured:135(405021ns) {StartInitialHandshake:157(402340ns) {DropHandshakes:470(2230ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(523ns) PrepareNewSessionHandshake:477(1069ns) {SwitchToState:179(484ns)}}}}}
2025-11-28T14:52:24.386210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789233599599750:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28252940ns) {Configure:104(28244594ns) {ProcessConfigured:135(289164ns) {StartInitialHandshake:157(287882ns) {DropHandshakes:470(3995ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:477(727ns) {SwitchToState:179(286ns)}}}}}
2025-11-28T14:52:24.386026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789230631248125:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53308951ns) {Configure:104(53300356ns) {ProcessConfigured:135(395059ns) {StartInitialHandshake:157(393841ns) {DropHandshakes:470(967ns) {DropIncomingHandshake:442(189ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:477(1074ns) {SwitchToState:179(213ns)}}}}}
2025-11-28T14:52:24.387657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789232566317802:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48657440ns) {Configure:104(48651653ns) {ProcessConfigured:135(870631ns) {StartInitialHandshake:157(868825ns) {DropHandshakes:470(1117ns) {DropIncomingHandshake:442(282ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(456ns) PrepareNewSessionHandshake:477(165644ns) {SwitchToState:179(522ns)}}}}}
2025-11-28T14:52:24.388859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789230903623633:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40778237ns) {Configure:104(40769603ns) {ProcessConfigured:135(846991ns) {StartInitialHandshake:157(845386ns) {DropHandshakes:470(1117ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:477(815ns) {SwitchToState:179(436ns)}}}}}
2025-11-28T14:52:24.389720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789231010527905:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22661832ns) {Configure:104(22657067ns) {ProcessConfigured:135(712866ns) {StartInitialHandshake:157(710683ns) {DropHandshakes:470(1346ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(555ns)} GenerateSessionVirtualId:988(551ns) PrepareNewSessionHandshake:477(1355ns) {SwitchToState:179(626ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.395814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789231079213904:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3425
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.396382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230631248202:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13132
No RDMA context found for GID: 2025-11-28T14:52:24.396358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789232566317878:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59192
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.398242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789232566317878:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.398148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230631248202:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:24.398742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789231079213906:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3425
::ffff:127.1.0.0
2025-11-28T14:52:24.398661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010527980:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.399303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789230631248125:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.399394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789232566317882:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59192
2025-11-28T14:52:24.399457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230631248205:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13132
2025-11-28T14:52:24.400002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010527982:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
2025-11-28T14:52:24.399246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789231079213905:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3425
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.400772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789232566317802:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.401625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230631248206:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13132
2025-11-28T14:52:24.400758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010527981:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
2025-11-28T14:52:24.399663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789231079213907:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3425
2025-11-28T14:52:24.402093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230631248205:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.402259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789232566317884:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59192
2025-11-28T14:52:24.400766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213893:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.400826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789231010527980:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.402257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789232566317881:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59192
2025-11-28T14:52:24.400795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789231079213893:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.403051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230631248208:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13132
2025-11-28T14:52:24.402054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789231010527904:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.400839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213904:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.403063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230631248206:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.402169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789231010527981:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.402521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789232566317882:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.403385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230631248195:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.404575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789232566317881:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.402045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213905:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.402270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789231010527905:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.403407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789230631248195:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.406825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010527986:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
2025-11-28T14:52:24.402725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213894:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.404691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789232566317803:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.406845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230631248195:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.402749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789231079213894:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.407288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789232566317885:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59192
2025-11-28T14:52:24.408531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789231010527986:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.402793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213896:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.407592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789232566317884:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.407564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230631248197:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.409226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789232566317885:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.409268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789231010527906:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.407586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789230631248197:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.402814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789231079213896:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.409625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789232566317889:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59192
2025-11-28T14:52:24.402836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789231079213908:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3425
2025-11-28T14:52:24.409948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230631248210:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13132
2025-11-28T14:52:24.409978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789231010527982:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.410514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789232566317874:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.410708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789231010527968:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.412176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789230903623705:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.402942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789231079213893:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.410088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789230631248189:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.412214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789230903623705:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.410542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789232566317874:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.411175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010527989:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
2025-11-28T14:52:24.403000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213906:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.410126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:24.412136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010527990:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
2025-11-28T14:52:24.412397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789232566317878:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:24.414101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789230903623705:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:24.410231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230631248197:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:24.414855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789231010527981:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.412462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789232566317874:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:24.410237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230631248208:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.415236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789231010527980:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:24.413848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789232566317802:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.407791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789231079213896:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:24.410554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230631248189:2109] [node 6] ICP22 created new session: [2:7577789230631248214:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.414100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.408431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789231079213894:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:24.411085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789230631248125:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789230631248202:2115] held: no
2025-11-28T14:52:24.417150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789231010527905:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.418053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789230903623698:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.408823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789231079213823:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.418881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789231010527904:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.414398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789232566317892:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59192
2025-11-28T14:52:24.411096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230631248214:2109] [node 6] ICS09 handshake done sender: [2:7577789230631248195:2111] self: [2:7577789230631248193:0] peer: [6:7577789231010527988:0] socket: 487 qp: -1
2025-11-28T14:52:24.419643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789233599599828:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49161
2025-11-28T14:52:24.419935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.408876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.418927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:24.411680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230631248214:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.414459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789232566317802:2048] [node 1] ICP22 created new session: [3:7577789232566317894:2048]
2025-11-28T14:52:24.421540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789233599599829:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49161
2025-11-28T14:52:24.419976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789230903623698:2110] [node 6] ICP22 created new session: [5:7577789230903623716:2110]
2025-11-28T14:52:24.408906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789231079213823:2048] [node 2] ICP22 created new session: [1:7577789231079213915:2048]
2025-11-28T14:52:24.418979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789231010527904:2048] [node 1] ICP22 created new session: [6:7577789231010527995:2048]
2025-11-28T14:52:24.421863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789233599599828:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.414727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789232566317866:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.412176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:24.409627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789231079213915:2048] [node 2] ICS09 handshake done sender: [1:7577789231079213893:2112] self: [1:7577789231079213892:0] peer: [2:7577789230631248207:0] socket: 480 qp: -1
2025-11-28T14:52:24.421562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789230903623713:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-28T14:52:24.419919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.414775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:24.412210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789230631248126:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.421851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789230903623716:2110] [node 6] ICS09 handshake done sender: [5:7577789230903623705:2112] self: [5:7577789230903623701:0] peer: [6:7577789231010527992:0] socket: 501 qp: -1
2025-11-28T14:52:24.410100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789231079213915:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:24.423134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789233599599751:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.420212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789231010527995:2048] [node 1] ICS09 handshake done sender: [6:7577789231010527980:2116] self: [6:7577789231010527987:0] peer: [1:7577789231079213895:0] socket: 489 qp: -1
2025-11-28T14:52:24.415225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789232566317894:2048] [node 1] ICS09 handshake done sender: [3:7577789232566317878:2115] self: [3:7577789232566317886:0] peer: [1:7577789231079213891:0] socket: 485 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.412411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.410337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213908:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.423183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789233599599829:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.422392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789230903623716:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:24.420223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789231010527905:2049] [node 2] ICP22 created new session: [6:7577789231010527996:2049]
2025-11-28T14:52:24.415228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789232566317866:2109] [node 6] ICP22 created new session: [3:7577789232566317895:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.422447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:24.412446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230631248126:2049] [node 3] ICP22 created new session: [2:7577789230631248217:2049]
2025-11-28T14:52:24.423937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789233599599830:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49161
2025-11-28T14:52:24.410569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213907:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.421548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789231010527995:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.422937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789230903623713:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.415765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789232566317894:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.410681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.423945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789233599599750:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.423356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789230903623714:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-28T14:52:24.410695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789231079213889:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.421826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.415796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.428638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789233599599830:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.413521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230631248217:2049] [node 3] ICS09 handshake done sender: [2:7577789230631248197:2112] self: [2:7577789230631248194:0] peer: [3:7577789232566317888:0] socket: 488 qp: -1
2025-11-28T14:52:24.423387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789230903623634:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.417024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789232566317895:2109] [node 6] ICS09 handshake done sender: [3:7577789232566317874:2112] self: [3:7577789232566317872:0] peer: [6:7577789231010527991:0] socket: 511 qp: -1
2025-11-28T14:52:24.421848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789231010527996:2049] [node 2] ICS09 handshake done sender: [6:7577789231010527981:2117] self: [6:7577789231010527988:0] peer: [2:7577789230631248193:0] socket: 497 qp: -1
2025-11-28T14:52:24.425128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789230903623714:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.413723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230631248202:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:24.411093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789231079213824:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.433630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789233599599751:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19507) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789233599599828:2115] held: no
2025-11-28T14:52:24.422039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789231010527996:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:24.427660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789230903623633:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.414089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230631248217:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:24.411399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.434736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789230903623721:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-28T14:52:24.433654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789233599599833:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49161
2025-11-28T14:52:24.422069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.414437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:24.417058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789232566317895:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:24.411710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789231079213824:2049] [node 3] ICP22 created new session: [1:7577789231079213918:2049]
2025-11-28T14:52:24.434823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789233599599834:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49161
2025-11-28T14:52:24.428090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789231010527906:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14567) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789231010527986:2119] held: no
2025-11-28T14:52:24.417113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:24.414961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789230631248125:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.412189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:24.434896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789233599599836:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49161
2025-11-28T14:52:24.430459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010527994:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
2025-11-28T14:52:24.412203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789231079213918:2049] [node 3] ICS09 handshake done sender: [1:7577789231079213894:2113] self: [1:7577789231079213891:0] peer: [3:7577789232566317886:0] socket: 481 qp: -1
2025-11-28T14:52:24.414992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.434978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789230903623719:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-28T14:52:24.435964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789233599599833:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.430758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010527993:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
2025-11-28T14:52:24.421423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789232566317803:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19507) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789232566317881:2116] held: no
2025-11-28T14:52:24.412226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789231079213889:2110] [node 6] ICP22 created new session: [1:7577789231079213919:2110]
2025-11-28T14:52:24.436554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789230903623719:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.436033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789233599599828:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.417017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230631248125:2048] [node 1] ICP22 created new session: [2:7577789230631248220:2048]
2025-11-28T14:52:24.422907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789232566317802:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:24.433984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789231010527986:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:24.417064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789230631248189:2109] [node 6] ICP18 handshake (actor [2:7577789230631248208:2118]) is too old
2025-11-28T14:52:24.412376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789231079213918:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:24.436741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789230903623713:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.437662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789233599599750:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789233599599829:2116] held: no
2025-11-28T14:52:24.423357Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789232566317881:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.434002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789231010527982:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:24.412409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.417068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230631248220:2048] [node 1] ICS09 handshake done sender: [2:7577789230631248202:2115] self: [2:7577789230631248207:0] peer: [1:7577789231079213892:0] socket: 484 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.437663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789233599599837:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49161
2025-11-28T14:52:24.436934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789230903623635:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.426414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789232566317803:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.434894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789231010527906:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.418010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230631248220:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.413508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789231079213919:2110] [node 6] ICS09 handshake done sender: [1:7577789231079213896:2114] self: [1:7577789231079213895:0] peer: [6:7577789231010527987:0] socket: 482 qp: -1
2025-11-28T14:52:24.426457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.437998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789230903623634:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.438041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.414092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789231079213919:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:24.434934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.428225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789232566317803:2049] [node 2] ICP22 created new session: [3:7577789232566317900:2049]
2025-11-28T14:52:24.418047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.438436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789230903623634:2049] [node 2] ICP22 created new session: [5:7577789230903623725:2049]
2025-11-28T14:52:24.437948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789233599599751:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.414849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:24.429785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789232566317900:2049] [node 2] ICS09 handshake done sender: [3:7577789232566317881:2116] self: [3:7577789232566317888:0] peer: [2:7577789230631248194:0] socket: 494 qp: -1
2025-11-28T14:52:24.435891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789231010527906:2050] [node 3] ICP22 created new session: [6:7577789231010528001:2050]
2025-11-28T14:52:24.423223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230631248198:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.439877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789230903623725:2049] [node 2] ICS09 handshake done sender: [5:7577789230903623713:2116] self: [5:7577789230903623718:0] peer: [2:7577789230631248199:0] socket: 537 qp: -1
2025-11-28T14:52:24.430449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789232566317900:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:24.438254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789233599599829:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:24.435910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789231010527968:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.423240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789230631248198:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.440095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789230903623725:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:24.424733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789231079213889:2110] [node 6] ICP18 handshake (actor [1:7577789231079213908:2121]) is too old
2025-11-28T14:52:24.438683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789233599599752:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.435939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.426424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230631248198:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:24.430827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.436526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789231010527968:2105] [node 5] ICP22 created new session: [6:7577789231010528002:2105]
2025-11-28T14:52:24.426540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213898:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.438669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789232566317877:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.441102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789230903623723:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-28T14:52:24.439650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789233599599750:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.427653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230631248201:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.436559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789231010528001:2050] [node 3] ICS09 handshake done sender: [6:7577789231010527986:2119] self: [6:7577789231010527991:0] peer: [3:7577789232566317872:0] socket: 513 qp: -1
2025-11-28T14:52:24.426581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789231079213898:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.441150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789230903623722:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-28T14:52:24.439681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:24.427689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789230631248201:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.438694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789232566317877:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.429551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789231079213898:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.436587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789231010528001:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:24.441213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.439848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.440329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789232566317876:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.430434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789230631248190:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.436632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:24.440354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789232566317876:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.430435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789231079213900:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.436979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789231010528002:2105] [node 5] ICS09 handshake done sender: [6:7577789231010527982:2118] self: [6:7577789231010527992:0] peer: [5:7577789230903623701:0] socket: 504 qp: -1
2025-11-28T14:52:24.440093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789233599599750:2048] [node 1] ICP22 created new session: [4:7577789233599599842:2048]
2025-11-28T14:52:24.430491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.441082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789232566317877:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:24.438244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789231010528002:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:24.440316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789233599599751:2049] [node 2] ICP22 created new session: [4:7577789233599599841:2049]
2025-11-28T14:52:24.431088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230631248190:2107] [node 4] ICP22 created new session: [2:7577789230631248224:2107]
2025-11-28T14:52:24.441212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789230903623633:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789230903623714:2117] held: no
2025-11-28T14:52:24.430461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789231079213900:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.441118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789233599599841:2049] [node 2] ICS09 handshake done sender: [4:7577789233599599828:2115] self: [4:7577789233599599831:0] peer: [2:7577789230631248196:0] socket: 533 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:24.431688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230631248201:2114] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:24.438431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:24.431755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789231079213888:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.441565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789232566317867:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.433333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230631248224:2107] [node 4] ICS09 handshake done sender: [2:7577789230631248198:2113] self: [2:7577789230631248196:0] peer: [4:7577789233599599831:0] socket: 492 qp: -1
2025-11-28T14:52:24.441156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789233599599841:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:24.433484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789230631248188:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.441165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789233599599842:2048] [node 1] ICS09 handshake done sender: [4:7577789233599599829:2116] self: [4:7577789233599599832:0] peer: [1:7577789231079213897:0] socket: 534 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:24.432528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.433642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230631248224:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:24.445290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010528005:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
2025-11-28T14:52:24.433690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789231079213888:2107] [node 4] ICP22 created new session: [1:7577789231079213925:2107]
2025-11-28T14:52:24.441606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.441972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789232566317876:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:24.441199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.433684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:24.445310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789230903623635:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14567) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789230903623719:2118] held: no
2025-11-28T14:52:24.434123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789231079213925:2107] [node 4] ICS09 handshake done sender: [1:7577789231079213898:2115] self: [1:7577789231079213897:0] peer: [4:7577789233599599832:0] socket: 532 qp: -1
2025-11-28T14:52:24.441980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789232566317867:2108] [node 5] ICP22 created new session: [3:7577789232566317904:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.441333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789233599599842:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.433700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.443311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789232566317904:2108] [node 5] ICS09 handshake done sender: [3:7577789232566317877:2114] self: [3:7577789232566317873:0] peer: [5:7577789230903623724:0] socket: 517 qp: -1
2025-11-28T14:52:24.434150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789231079213925:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:24.446426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789230903623714:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:24.443411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789232566317904:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:24.434908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230631248188:2108] [node 5] ICP22 created new session: [2:7577789230631248225:2108]
2025-11-28T14:52:24.434724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789231079213900:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:24.446518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789230903623726:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-28T14:52:24.447484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789230903623727:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-28T14:52:24.441721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.447669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789230903623633:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.434956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230631248225:2108] [node 5] ICS09 handshake done sender: [2:7577789230631248201:2114] self: [2:7577789230631248199:0] peer: [5:7577789230903623718:0] socket: 535 qp: -1
2025-11-28T14:52:24.443447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:24.434949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:24.443519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789233599599837:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.448281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789230903623719:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:24.447960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789231010528005:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.435900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230631248225:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:24.449246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:24.444996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789233599599844:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49161
2025-11-28T14:52:24.448449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789231010527967:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.436729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789231079213890:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.435948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:24.444146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789232566317865:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.449339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789230903623726:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.446122Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789233599599833:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:24.436994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.449366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789230903623635:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.444383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.437204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789231079213890:2108] [node 5] ICP22 created new session: [1:7577789231079213928:2108]
2025-11-28T14:52:24.447482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789233599599752:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14567) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789233599599833:2118] held: no
2025-11-28T14:52:24.444625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789232566317865:2107] [node 4] ICP22 created new session: [3:7577789232566317906:2107]
2025-11-28T14:52:24.449372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789230903623633:2048] [node 1] ICP22 created new session: [5:7577789230903623729:2048]
2025-11-28T14:52:24.437244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789231079213928:2108] [node 5] ICS09 handshake done sender: [1:7577789231079213900:2116] self: [1:7577789231079213899:0] peer: [5:7577789230903623720:0] socket: 536 qp: -1
2025-11-28T14:52:24.447562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789233599599752:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.449750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.444953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789232566317906:2107] [node 4] ICS09 handshake done sender: [3:7577789232566317876:2113] self: [3:7577789232566317875:0] peer: [4:7577789233599599839:0] socket: 505 qp: -1
2025-11-28T14:52:24.449846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789230903623697:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.437709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789231079213928:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:24.444984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789232566317906:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:24.437739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:24.447723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.449850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789230903623729:2048] [node 1] ICS09 handshake done sender: [5:7577789230903623714:2117] self: [5:7577789230903623720:0] peer: [1:7577789231079213899:0] socket: 507 qp: -1
2025-11-28T14:52:24.445044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:24.447872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789233599599752:2050] [node 3] ICP22 created new session: [4:7577789233599599847:2050]
2025-11-28T14:52:24.449865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789230903623635:2050] [node 3] ICP22 created new session: [5:7577789230903623730:2050]
2025-11-28T14:52:24.448444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789233599599847:2050] [node 3] ICS09 handshake done sender: [4:7577789233599599833:2118] self: [4:7577789233599599839:0] peer: [3:7577789232566317875:0] socket: 519 qp: -1
2025-11-28T14:52:24.448484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789233599599847:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:24.450013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789230903623729:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.450039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.450116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789230903623730:2050] [node 3] ICS09 handshake done sender: [5:7577789230903623719:2118] self: [5:7577789230903623724:0] peer: [3:7577789232566317873:0] socket: 518 qp: -1
2025-11-28T14:52:24.449210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:24.450139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789230903623730:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:24.450206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:24.449768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789233599599822:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.449792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789233599599822:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.450190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789233599599825:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.450217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789233599599825:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.451307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789233599599822:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.451465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789233599599813:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.451580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:24.451652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789233599599825:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:24.451678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789233599599813:2109] [node 6] ICP22 created new session: [4:7577789233599599851:2109]
2025-11-28T14:52:24.451674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789231010528007:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11276
2025-11-28T14:52:24.451731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789233599599851:2109] [node 6] ICS09 handshake done sender: [4:7577789233599599822:2113] self: [4:7577789233599599821:0] peer: [6:7577789231010528006:0] socket: 509 qp: -1
2025-11-28T14:52:24.451771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789233599599851:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.451849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789233599599814:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.451867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:24.452248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789231010528005:2124] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:24.452091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.452254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789233599599814:2108] [node 5] ICP22 created new session: [4:7577789233599599853:2108]
2025-11-28T14:52:24.452269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789230903623734:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-28T14:52:24.452263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789231010527967:2104] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2496) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789231010528005:2124] held: no
2025-11-28T14:52:24.452282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789233599599853:2108] [node 5] ICS09 handshake done sender: [4:7577789233599599825:2114] self: [4:7577789233599599823:0] peer: [5:7577789230903623731:0] socket: 486 qp: -1
2025-11-28T14:52:24.452582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789231010527967:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.454088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789230903623726:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:24.453289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.452300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789233599599853:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:24.454386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789230903623697:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2496) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789230903623726:2122] held: no
2025-11-28T14:52:24.453292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:24.454312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789231010527967:2104] [node 4] ICP22 created new session: [6:7577789231010528008:2104]
2025-11-28T14:52:24.454539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789230903623697:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.454756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.454406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789231010528008:2104] [node 4] ICS09 handshake done sender: [6:7577789231010528005:2124] self: [6:7577789231010528006:0] peer: [4:7577789233599599821:0] socket: 551 qp: -1
2025-11-28T14:52:24.454434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789231010528008:2104] [node 4] ICS10 traffic start
2025-11-28T14:52:24.454477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:24.454887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789230903623697:2108] [node 4] ICP22 created new session: [5:7577789230903623735:2108]
2025-11-28T14:52:24.454982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789230903623735:2108] [node 4] ICS09 handshake done sender: [5:7577789230903623726:2122] self: [5:7577789230903623731:0] peer: [4:7577789233599599823:0] socket: 495 qp: -1
2025-11-28T14:52:24.455017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789230903623735:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:24.455139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:24.485319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:24.485483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789230631248220:2048] [node 1] ICS01 socket: 484 reason# EndOfStream
2025-11-28T14:52:24.485516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789230631248125:2048] [node 1] ICP30 unregister session Session# [2:7577789230631248220:2048] VirtualId# [2:7577789230631248207:0]
2025-11-28T14:52:24.485537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789230631248220:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.485596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:24.485634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:24.486389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:24.486548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789232566317894:2048] [node 1] ICS07 socket disconnect 485 reason# EndOfStream
2025-11-28T14:52:24.486585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789232566317894:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.486645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:24.486687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789232566317894:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:24.487338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789232566317802:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24845) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:24.487452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789232566317894:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:24.487481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789232566317802:2048] [node 1] ICP30 unregister session Session# [3:7577789232566317894:2048] VirtualId# [3:7577789232566317886:0]
2025-11-28T14:52:24.487517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:24.487620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789232566317802:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.487630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.488333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:24.488495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789233599599842:2048] [node 1] ICS01 socket: 534 reason# EndOfStream
2025-11-28T14:52:24.488526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789233599599750:2048] [node 1] ICP30 unregister session Session# [4:7577789233599599842:2048] VirtualId# [4:7577789233599599832:0]
2025-11-28T14:52:24.488547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789233599599842:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.488602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:24.488642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:24.489209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:24.489414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789230903623729:2048] [node 1] ICS01 socket: 507 reason# EndOfStream
2025-11-28T14:52:24.489463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789230903623633:2048] [node 1] ICP30 unregister session Session# [5:7577789230903623729:2048] VirtualId# [5:7577789230903623720:0]
2025-11-28T14:52:24.489491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789230903623729:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.489552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:24.489603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:24.489758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:24.489909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789231010527995:2048] [node 1] ICS07 socket disconnect 489 reason# EndOfStream
2025-11-28T14:52:24.489935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789231010527995:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.490015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:24.490053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789231010527995:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:24.490586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789231010527904:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24845) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:24.490660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789231010527995:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:24.490703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789231010527904:2048] [node 1] ICP30 unregister session Session# [6:7577789231010527995:2048] VirtualId# [6:7577789231010527987:0]
2025-11-28T14:52:24.490742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:24.490772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789231010527904:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.490788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.497848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789232566317802:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:24.500892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789231010527904:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:24.516707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:24.516921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789232566317900:2049] [node 2] ICS01 socket: 494 reason# EndOfStream
2025-11-28T14:52:24.516974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789232566317803:2049] [node 2] ICP30 unregister session Session# [3:7577789232566317900:2049] VirtualId# [3:7577789232566317888:0]
2025-11-28T14:52:24.517005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789232566317900:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.517084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:24.517170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:24.518638Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:24.518793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789233599599841:2049] [node 2] ICS01 socket: 533 reason# EndOfStream
2025-11-28T14:52:24.518815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789233599599751:2049] [node 2] ICP30 unregister session Session# [4:7577789233599599841:2049] VirtualId# [4:7577789233599599831:0]
2025-11-28T14:52:24.518834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789233599599841:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.518878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:24.518922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:24.519617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:24.519841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789230903623725:2049] [node 2] ICS07 socket disconnect 537 reason# EndOfStream
2025-11-28T14:52:24.519886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789230903623725:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.519966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:24.520018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789230903623725:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:24.520416Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:24.520625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789231010527996:2049] [node 2] ICS07 socket disconnect 497 reason# EndOfStream
2025-11-28T14:52:24.520661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789231010527996:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.520740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:24.520775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789230903623634:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19507) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:24.520797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789231010527996:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:24.520936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789230903623725:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:24.520993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789230903623634:2049] [node 2] ICP30 unregister session Session# [5:7577789230903623725:2049] VirtualId# [5:7577789230903623718:0]
2025-11-28T14:52:24.521050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:24.521085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789230903623634:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19507) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.521108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19507) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.521501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789231010527905:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19507) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:24.521542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789231010527996:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:24.521563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789231010527905:2049] [node 2] ICP30 unregister session Session# [6:7577789231010527996:2049] VirtualId# [6:7577789231010527988:0]
2025-11-28T14:52:24.521600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:24.521641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789231010527905:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19507) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.521664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19507) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.530642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789230903623634:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:24.531619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789231010527905:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:24.565526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:24.565677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789233599599847:2050] [node 3] ICS01 socket: 519 reason# EndOfStream
2025-11-28T14:52:24.565704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789233599599752:2050] [node 3] ICP30 unregister session Session# [4:7577789233599599847:2050] VirtualId# [4:7577789233599599839:0]
2025-11-28T14:52:24.565728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789233599599847:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.565783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:24.565821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:24.566599Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:24.566811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789230903623730:2050] [node 3] ICS01 socket: 518 reason# EndOfStream
2025-11-28T14:52:24.566862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789230903623635:2050] [node 3] ICP30 unregister session Session# [5:7577789230903623730:2050] VirtualId# [5:7577789230903623724:0]
2025-11-28T14:52:24.566894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789230903623730:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.566966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:24.567025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:24.567522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:24.567667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789231010528001:2050] [node 3] ICS01 socket: 513 reason# EndOfStream
2025-11-28T14:52:24.567696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789231010527906:2050] [node 3] ICP30 unregister session Session# [6:7577789231010528001:2050] VirtualId# [6:7577789231010527991:0]
2025-11-28T14:52:24.567721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789231010528001:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.567767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:24.567816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:24.609163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:24.609405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789230903623735:2108] [node 4] ICS01 socket: 495 reason# EndOfStream
2025-11-28T14:52:24.609456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789230903623697:2108] [node 4] ICP30 unregister session Session# [5:7577789230903623735:2108] VirtualId# [5:7577789230903623731:0]
2025-11-28T14:52:24.609497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789230903623735:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.609594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:24.609643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:24.610269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:24.610425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789231010528008:2104] [node 4] ICS01 socket: 551 reason# EndOfStream
2025-11-28T14:52:24.610462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789231010527967:2104] [node 4] ICP30 unregister session Session# [6:7577789231010528008:2104] VirtualId# [6:7577789231010528006:0]
2025-11-28T14:52:24.610489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789231010528008:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.610548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:24.610578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:24.661586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:24.661759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789231010528002:2105] [node 5] ICS07 socket disconnect 504 reason# EndOfStream
2025-11-28T14:52:24.661785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789231010528002:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:24.661868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:24.661909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789231010528002:2105] [node 5] ICS15 start handshake
2025-11-28T14:52:24.662482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789231010527968:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:24.662534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789231010528002:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:24.662569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789231010527968:2105] [node 5] ICP30 unregister session Session# [6:7577789231010528002:2105] VirtualId# [6:7577789231010527992:0]
2025-11-28T14:52:24.662613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:24.662638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789231010527968:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:24.662647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:24.761153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789234644005405:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:24.761191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789234644005406:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:24.779591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230747847829:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.779654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230747847830:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:24.798536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789231666852749:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.798588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789231666852750:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:24.817966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789230594002695:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.818010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789230594002696:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:24.818032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789230594002697:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:24.834579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789234308895589:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.834609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789234308895590:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:24.834635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789234308895591:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:24.851177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789232338474782:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:24.851220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789232338474783:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:24.851248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789232338474784:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:24.937210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789234644005468:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:24.937210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230747847893:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:24.937388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789230594002758:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:24.937759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230747847894:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:24.937879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789234644005469:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:24.937827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789234308895652:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:24.937898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789232338474846:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:24.937812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789231666852812:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:24.938143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789230747847895:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:24.938201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789231666852813:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:24.938334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789230594002759:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:24.938622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789234308895653:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:24.938755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789234644005470:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:24.938820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789232338474847:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:24.938821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789231666852814:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.948771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230747847906:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35932
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.949757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230747847910:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35932
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.949921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230747847911:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35932
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.949994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789231666852825:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53000
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.950303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789231666852826:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53000
2025-11-28T14:52:24.950719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789231666852827:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53000
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.951568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789231666852828:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53000
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.955970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789230594002773:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29270
2025-11-28T14:52:24.956061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789230594002774:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29270
2025-11-28T14:52:24.956001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789234644005481:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30003
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.956167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789234644005482:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30003
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.956263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789234644005483:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30003
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.956403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789234644005484:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30003
2025-11-28T14:52:24.957049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005481:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.957221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005482:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.957297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789234644005485:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30003
2025-11-28T14:52:24.957336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789230594002775:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29270
2025-11-28T14:52:24.957426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005483:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.958730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789231666852825:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.958840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789231666852826:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.958948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230747847906:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.959294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789231666852827:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.959325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789230594002774:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.959476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789230594002773:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.959675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789230594002697:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.959638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230747847910:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.959795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789231666852832:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53000
2025-11-28T14:52:24.959811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230747847911:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.959873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789231666852828:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.959895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789230594002696:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.959970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789231666852820:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.959996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230747847912:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35932
2025-11-28T14:52:24.959987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789231666852820:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.960220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230747847901:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.960248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789230747847901:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.960683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789231666852750:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.960941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789231666852820:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:24.960946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230747847913:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35932
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.961078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789230594002778:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29270
2025-11-28T14:52:24.961157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230747847912:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.961302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789231666852812:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.961340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.961371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789231666852812:2107] [node 4] ICP22 created new session: [3:7577789231666852838:2107]
2025-11-28T14:52:24.961362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230747847899:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.961399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789231666852838:2107] [node 4] ICS09 handshake done sender: [3:7577789231666852820:2112] self: [3:7577789231666852819:0] peer: [4:7577789230594002776:0] socket: 516 qp: -1
2025-11-28T14:52:24.961385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789230747847899:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.961420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789231666852838:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:24.961453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:24.962176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005485:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.962239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789230594002774:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:24.962270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005484:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.962549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789230594002697:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.962589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.962626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230747847901:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:24.962630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789230594002697:2050] [node 3] ICP22 created new session: [4:7577789230594002780:2050]
2025-11-28T14:52:24.962703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789230594002780:2050] [node 3] ICS09 handshake done sender: [4:7577789230594002774:2116] self: [4:7577789230594002776:0] peer: [3:7577789231666852819:0] socket: 520 qp: -1
2025-11-28T14:52:24.962747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789230594002780:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:24.962801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:24.963066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230747847913:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.963139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789230747847893:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.963180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.963219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230747847893:2107] [node 4] ICP22 created new session: [2:7577789230747847920:2107]
2025-11-28T14:52:24.963217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789231666852832:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.963271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230747847920:2107] [node 4] ICS09 handshake done sender: [2:7577789230747847901:2113] self: [2:7577789230747847900:0] peer: [4:7577789230594002777:0] socket: 506 qp: -1
2025-11-28T14:52:24.963316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230747847920:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:24.963348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.963718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789231666852749:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.963825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789231666852840:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53000
2025-11-28T14:52:24.963879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230747847899:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.963975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789230594002775:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.964263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789230594002779:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29270
2025-11-28T14:52:24.964259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789234308895667:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32254
2025-11-28T14:52:24.964402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789231666852828:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.964584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005474:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.964620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789234644005474:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.965215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789230747847830:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.965263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.965299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230747847830:2049] [node 3] ICP22 created new session: [2:7577789230747847922:2049]
2025-11-28T14:52:24.965347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230747847922:2049] [node 3] ICS09 handshake done sender: [2:7577789230747847899:2112] self: [2:7577789230747847898:0] peer: [3:7577789231666852837:0] socket: 505 qp: -1
2025-11-28T14:52:24.965382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230747847922:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:24.965421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:24.965566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789234644005474:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:24.966268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789230594002695:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.966268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789231666852750:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.966342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789230594002773:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.966326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.966356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789234644005406:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.966361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789231666852750:2049] [node 2] ICP22 created new session: [3:7577789231666852843:2049]
2025-11-28T14:52:24.966400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789231666852843:2049] [node 2] ICS09 handshake done sender: [3:7577789231666852828:2118] self: [3:7577789231666852837:0] peer: [2:7577789230747847898:0] socket: 507 qp: -1
2025-11-28T14:52:24.966478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.966463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789231666852843:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:24.966530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789234644005406:2049] [node 3] ICP22 created new session: [1:7577789234644005495:2049]
2025-11-28T14:52:24.966481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.966572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789234644005495:2049] [node 3] ICS09 handshake done sender: [1:7577789234644005474:2111] self: [1:7577789234644005473:0] peer: [3:7577789231666852842:0] socket: 522 qp: -1
2025-11-28T14:52:24.966582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789230594002696:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23692) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789230594002773:2115] held: no
2025-11-28T14:52:24.966596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789234644005495:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:24.966629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:24.966683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789230594002696:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.966649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789230747847829:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.966713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.966744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789230594002696:2049] [node 2] ICP22 created new session: [4:7577789230594002784:2049]
2025-11-28T14:52:24.966789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789230594002784:2049] [node 2] ICS09 handshake done sender: [4:7577789230594002773:2115] self: [4:7577789230594002777:0] peer: [2:7577789230747847900:0] socket: 508 qp: -1
2025-11-28T14:52:24.966829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789230594002784:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:24.966859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.967800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005472:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.967825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789234644005472:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.968215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789234308895667:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.968514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789234308895668:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32254
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.968911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005476:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.969073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789234308895669:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32254
2025-11-28T14:52:24.968948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789234644005476:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.970005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789234308895591:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.970166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789231666852749:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789231666852832:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.970325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789231666852845:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53000
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.970383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474861:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
2025-11-28T14:52:24.970432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789234644005472:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.970453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474862:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.970822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789234308895671:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32254
2025-11-28T14:52:24.970831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789234644005405:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.970920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.970974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789234644005405:2048] [node 2] ICP22 created new session: [1:7577789234644005500:2048]
2025-11-28T14:52:24.970954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789231666852822:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.971143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789234644005500:2048] [node 2] ICS09 handshake done sender: [1:7577789234644005472:2110] self: [1:7577789234644005471:0] peer: [2:7577789230747847925:0] socket: 521 qp: -1
2025-11-28T14:52:24.971338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789234644005500:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:24.970980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789231666852822:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.971371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.972157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789231666852832:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:24.972505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789234308895668:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.972522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789234644005476:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.972746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789232338474862:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.972870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789234308895669:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.972943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474863:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
2025-11-28T14:52:24.972930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789230747847927:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35932
2025-11-28T14:52:24.972986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789234308895590:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.972960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789230594002787:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29270
2025-11-28T14:52:24.973020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789231666852749:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.973077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789232338474861:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.973061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:24.973093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789231666852749:2048] [node 1] ICP22 created new session: [3:7577789231666852848:2048]
2025-11-28T14:52:24.973130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789231666852848:2048] [node 1] ICS09 handshake done sender: [3:7577789231666852832:2119] self: [3:7577789231666852842:0] peer: [1:7577789234644005473:0] socket: 526 qp: -1
2025-11-28T14:52:24.973158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789231666852848:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.973197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789232338474783:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.973180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.973614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789232338474846:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.973645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789230594002775:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:24.973718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789234308895652:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.973819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230747847903:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.973861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789230747847903:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.974240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789230594002769:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.974269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789230594002769:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.974256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789234308895673:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32254
2025-11-28T14:52:24.974423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789231666852822:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:24.974823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789230594002695:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.974965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:24.975018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789230594002695:2048] [node 1] ICP22 created new session: [4:7577789230594002788:2048]
2025-11-28T14:52:24.975212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789230594002788:2048] [node 1] ICS09 handshake done sender: [4:7577789230594002775:2117] self: [4:7577789230594002783:0] peer: [1:7577789234644005475:0] socket: 524 qp: -1
2025-11-28T14:52:24.975265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789230747847905:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.975264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789230594002788:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.975311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.975303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789234308895667:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:24.975306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789230747847905:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.975329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789234644005468:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.975377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.975419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789234644005468:2107] [node 4] ICP22 created new session: [1:7577789234644005502:2107]
2025-11-28T14:52:24.975949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789234644005502:2107] [node 4] ICS09 handshake done sender: [1:7577789234644005476:2112] self: [1:7577789234644005475:0] peer: [4:7577789230594002783:0] socket: 523 qp: -1
2025-11-28T14:52:24.976047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230747847913:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:24.976043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789234644005502:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:24.976082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:24.976124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789234308895591:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.976206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.976259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789234308895591:2050] [node 3] ICP22 created new session: [5:7577789234308895675:2050]
2025-11-28T14:52:24.976664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789234308895675:2050] [node 3] ICS09 handshake done sender: [5:7577789234308895667:2115] self: [5:7577789234308895670:0] peer: [3:7577789231666852821:0] socket: 534 qp: -1
2025-11-28T14:52:24.976699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789234308895675:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:24.976757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:24.976767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789231666852813:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.976774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789230594002767:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.976831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.976792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789230594002767:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.976955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789231666852813:2108] [node 5] ICP22 created new session: [3:7577789231666852850:2108]
2025-11-28T14:52:24.977443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789231666852850:2108] [node 5] ICS09 handshake done sender: [3:7577789231666852822:2113] self: [3:7577789231666852821:0] peer: [5:7577789234308895670:0] socket: 533 qp: -1
2025-11-28T14:52:24.977472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789231666852850:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:24.977506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:24.978483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789230747847829:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.978553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:24.978587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230747847829:2048] [node 1] ICP22 created new session: [2:7577789230747847930:2048]
2025-11-28T14:52:24.978630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230747847930:2048] [node 1] ICS09 handshake done sender: [2:7577789230747847913:2120] self: [2:7577789230747847925:0] peer: [1:7577789234644005471:0] socket: 527 qp: -1
2025-11-28T14:52:24.978658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230747847930:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.979487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.979863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789230594002769:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.979930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789230747847829:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:24.979974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474866:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
2025-11-28T14:52:24.980031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789234308895671:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.980187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789230594002767:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:24.980587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789230594002759:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.980625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:24.980687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789230594002759:2109] [node 6] ICP22 created new session: [4:7577789230594002792:2109]
2025-11-28T14:52:24.980712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789230594002792:2109] [node 6] ICS09 handshake done sender: [4:7577789230594002769:2114] self: [4:7577789230594002768:0] peer: [6:7577789232338474865:0] socket: 515 qp: -1
2025-11-28T14:52:24.980736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789230594002792:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:24.980769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:24.981022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789232338474863:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.981057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789230594002758:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.981142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230747847903:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:24.981081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.981153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789230594002758:2108] [node 5] ICP22 created new session: [4:7577789230594002794:2108]
2025-11-28T14:52:24.981187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789230594002794:2108] [node 5] ICS09 handshake done sender: [4:7577789230594002767:2113] self: [4:7577789230594002766:0] peer: [5:7577789234308895674:0] socket: 517 qp: -1
2025-11-28T14:52:24.981207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789230594002794:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:24.981238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:24.981331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789232338474861:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.981372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789234308895589:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.981708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789234308895677:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32254
2025-11-28T14:52:24.981746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789230747847905:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:24.982061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789232338474846:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.982109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.982344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789232338474846:2106] [node 4] ICP22 created new session: [6:7577789232338474869:2106]
2025-11-28T14:52:24.982404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789232338474869:2106] [node 4] ICS09 handshake done sender: [6:7577789232338474861:2116] self: [6:7577789232338474865:0] peer: [4:7577789230594002768:0] socket: 536 qp: -1
2025-11-28T14:52:24.982435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789232338474869:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:24.982457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789230747847894:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.982464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.982538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.982551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789234308895678:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32254
2025-11-28T14:52:24.982590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230747847894:2108] [node 5] ICP22 created new session: [2:7577789230747847932:2108]
2025-11-28T14:52:24.982653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230747847932:2108] [node 5] ICS09 handshake done sender: [2:7577789230747847903:2114] self: [2:7577789230747847902:0] peer: [5:7577789234308895672:0] socket: 498 qp: -1
2025-11-28T14:52:24.982684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230747847932:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:24.982642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789230594002796:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29270
2025-11-28T14:52:24.982713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:24.982660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005478:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.982693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789234644005478:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.982910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789234308895669:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:24.983411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789234308895652:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.983445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:24.983477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789234308895652:2108] [node 4] ICP22 created new session: [5:7577789234308895681:2108]
2025-11-28T14:52:24.983505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789234308895681:2108] [node 4] ICS09 handshake done sender: [5:7577789234308895669:2117] self: [5:7577789234308895674:0] peer: [4:7577789230594002766:0] socket: 535 qp: -1
2025-11-28T14:52:24.983526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789234308895681:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:24.983545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:24.983551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789234644005478:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:24.983577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789230747847895:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.983710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:24.983759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474867:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
2025-11-28T14:52:24.983792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789230747847895:2109] [node 6] ICP22 created new session: [2:7577789230747847934:2109]
2025-11-28T14:52:24.983844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789230747847934:2109] [node 6] ICS09 handshake done sender: [2:7577789230747847905:2115] self: [2:7577789230747847904:0] peer: [6:7577789232338474864:0] socket: 537 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.983886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789230747847934:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:24.983878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789234644005469:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.983938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:24.983918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.983943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789234644005469:2108] [node 5] ICP22 created new session: [1:7577789234644005506:2108]
2025-11-28T14:52:24.983946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474868:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
2025-11-28T14:52:24.983969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789234644005506:2108] [node 5] ICS09 handshake done sender: [1:7577789234644005478:2113] self: [1:7577789234644005477:0] peer: [5:7577789234308895680:0] socket: 514 qp: -1
2025-11-28T14:52:24.983986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789234644005506:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:24.984074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789234308895668:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.984008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:24.984795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789232338474784:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.985016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789232338474867:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.985256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789232338474862:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:24.985405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789231666852824:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.985441Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789231666852824:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.986479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789231666852824:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:24.987091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789234308895590:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.987173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.987203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789234308895590:2049] [node 2] ICP22 created new session: [5:7577789234308895684:2049]
2025-11-28T14:52:24.987335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789234308895684:2049] [node 2] ICS09 handshake done sender: [5:7577789234308895668:2116] self: [5:7577789234308895672:0] peer: [2:7577789230747847902:0] socket: 519 qp: -1
2025-11-28T14:52:24.987309Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789230594002758:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:24.987246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789231666852814:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.987360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789234308895684:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:24.987314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:24.987426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.987357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789231666852814:2109] [node 6] ICP22 created new session: [3:7577789231666852853:2109]
2025-11-28T14:52:24.987445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789231666852853:2109] [node 6] ICS09 handshake done sender: [3:7577789231666852824:2114] self: [3:7577789231666852823:0] peer: [6:7577789232338474871:0] socket: 544 qp: -1
2025-11-28T14:52:24.987481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789231666852853:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:24.987562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789232338474783:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.987517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:24.987610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:24.987654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789232338474783:2049] [node 2] ICP22 created new session: [6:7577789232338474873:2049]
2025-11-28T14:52:24.987695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789232338474873:2049] [node 2] ICS09 handshake done sender: [6:7577789232338474862:2117] self: [6:7577789232338474864:0] peer: [2:7577789230747847904:0] socket: 538 qp: -1
2025-11-28T14:52:24.987733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789232338474873:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:24.987771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:24.987974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789232338474783:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:24.989674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789234308895590:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:24.989775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789234308895589:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789234308895671:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.990132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789234308895683:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32254
2025-11-28T14:52:24.990469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789232338474782:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.990620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789234308895671:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.990782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789232338474784:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2255) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789232338474863:2118] held: no
2025-11-28T14:52:24.990802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789234308895686:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32254
2025-11-28T14:52:24.990849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789234308895589:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.990898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:24.990925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789234308895589:2048] [node 1] ICP22 created new session: [5:7577789234308895688:2048]
2025-11-28T14:52:24.990956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789234308895688:2048] [node 1] ICS09 handshake done sender: [5:7577789234308895671:2118] self: [5:7577789234308895680:0] peer: [1:7577789234644005477:0] socket: 539 qp: -1
2025-11-28T14:52:24.990977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789234308895688:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.990994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.991143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789234644005480:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.991176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789234644005480:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.991357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474875:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
2025-11-28T14:52:24.992153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789234644005480:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:24.992737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789234644005470:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.992780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:24.992902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789234644005470:2109] [node 6] ICP22 created new session: [1:7577789234644005509:2109]
2025-11-28T14:52:24.993083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789234644005509:2109] [node 6] ICS09 handshake done sender: [1:7577789234644005480:2114] self: [1:7577789234644005479:0] peer: [6:7577789232338474876:0] socket: 496 qp: -1
2025-11-28T14:52:24.993107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789234644005509:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:24.993132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.993304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474877:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
2025-11-28T14:52:24.993456Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789232338474863:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:24.994092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789232338474877:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.994142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789232338474784:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.994109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789234308895686:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.994184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:24.994226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789232338474784:2050] [node 3] ICP22 created new session: [6:7577789232338474879:2050]
2025-11-28T14:52:24.994271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789232338474879:2050] [node 3] ICS09 handshake done sender: [6:7577789232338474863:2118] self: [6:7577789232338474871:0] peer: [3:7577789231666852823:0] socket: 545 qp: -1
2025-11-28T14:52:24.994306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789232338474879:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:24.994338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:24.995727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789232338474782:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789232338474867:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.995899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474878:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
2025-11-28T14:52:24.996464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789232338474847:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:24.996533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789232338474867:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:24.996837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789232338474782:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.996881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:24.996923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789232338474782:2048] [node 1] ICP22 created new session: [6:7577789232338474882:2048]
2025-11-28T14:52:24.996964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789232338474882:2048] [node 1] ICS09 handshake done sender: [6:7577789232338474867:2120] self: [6:7577789232338474876:0] peer: [1:7577789234644005479:0] socket: 549 qp: -1
2025-11-28T14:52:24.996999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789232338474882:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:24.996962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789234308895663:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:24.997034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:24.996994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789234308895663:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:24.997959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789234308895663:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:24.998204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789232338474884:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57413
2025-11-28T14:52:24.998321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789234308895653:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:24.998377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:24.998425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789234308895653:2109] [node 6] ICP22 created new session: [5:7577789234308895692:2109]
2025-11-28T14:52:24.998503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789234308895692:2109] [node 6] ICS09 handshake done sender: [5:7577789234308895663:2114] self: [5:7577789234308895662:0] peer: [6:7577789232338474881:0] socket: 488 qp: -1
2025-11-28T14:52:24.998554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789234308895692:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:24.998590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:24.998611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789232338474877:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:24.998812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789232338474847:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:24.998851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:24.998889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789232338474847:2108] [node 5] ICP22 created new session: [6:7577789232338474885:2108]
2025-11-28T14:52:24.998926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789232338474885:2108] [node 5] ICS09 handshake done sender: [6:7577789232338474877:2123] self: [6:7577789232338474881:0] peer: [5:7577789234308895662:0] socket: 499 qp: -1
2025-11-28T14:52:24.998954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789232338474885:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:24.998981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.014017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:25.014263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789230747847930:2048] [node 1] ICS01 socket: 527 reason# EndOfStream
2025-11-28T14:52:25.014306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789230747847829:2048] [node 1] ICP30 unregister session Session# [2:7577789230747847930:2048] VirtualId# [2:7577789230747847925:0]
2025-11-28T14:52:25.014345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789230747847930:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.014421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.014478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.015196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:25.015400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789231666852848:2048] [node 1] ICS01 socket: 526 reason# EndOfStream
2025-11-28T14:52:25.015445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789231666852749:2048] [node 1] ICP30 unregister session Session# [3:7577789231666852848:2048] VirtualId# [3:7577789231666852842:0]
2025-11-28T14:52:25.015471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789231666852848:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.015538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.015589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.017232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:25.017408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789230594002788:2048] [node 1] ICS01 socket: 524 reason# EndOfStream
2025-11-28T14:52:25.017442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789230594002695:2048] [node 1] ICP30 unregister session Session# [4:7577789230594002788:2048] VirtualId# [4:7577789230594002783:0]
2025-11-28T14:52:25.017471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789230594002788:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.017534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.017575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.018167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:25.018380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789234308895688:2048] [node 1] ICS01 socket: 539 reason# EndOfStream
2025-11-28T14:52:25.018422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789234308895589:2048] [node 1] ICP30 unregister session Session# [5:7577789234308895688:2048] VirtualId# [5:7577789234308895680:0]
2025-11-28T14:52:25.018454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789234308895688:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.018530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.018593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.018989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:25.019123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789232338474882:2048] [node 1] ICS01 socket: 549 reason# EndOfStream
2025-11-28T14:52:25.019166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789232338474782:2048] [node 1] ICP30 unregister session Session# [6:7577789232338474882:2048] VirtualId# [6:7577789232338474876:0]
2025-11-28T14:52:25.019188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789232338474882:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.019235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.019273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.042442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:25.042676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789231666852843:2049] [node 2] ICS01 socket: 507 reason# EndOfStream
2025-11-28T14:52:25.042723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789231666852750:2049] [node 2] ICP30 unregister session Session# [3:7577789231666852843:2049] VirtualId# [3:7577789231666852837:0]
2025-11-28T14:52:25.042756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789231666852843:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.042829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.042887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.044926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:25.045168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789230594002784:2049] [node 2] ICS01 socket: 508 reason# EndOfStream
2025-11-28T14:52:25.045222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789230594002696:2049] [node 2] ICP30 unregister session Session# [4:7577789230594002784:2049] VirtualId# [4:7577789230594002777:0]
2025-11-28T14:52:25.045261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789230594002784:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.045372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.045443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.045819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:25.046009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789234308895684:2049] [node 2] ICS07 socket disconnect 519 reason# EndOfStream
2025-11-28T14:52:25.046037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789234308895684:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.046100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.046151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789234308895684:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:25.046658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:25.046836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789232338474873:2049] [node 2] ICS07 socket disconnect 538 reason# EndOfStream
2025-11-28T14:52:25.046861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789232338474873:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.046918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.046945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789232338474873:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:25.046936Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789234308895590:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23692) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.047001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789234308895684:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.047035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789234308895590:2049] [node 2] ICP30 unregister session Session# [5:7577789234308895684:2049] VirtualId# [5:7577789234308895672:0]
2025-11-28T14:52:25.047077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.047112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789234308895590:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23692) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.047130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23692) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.047503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789232338474783:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23692) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.047535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789232338474873:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.047555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789232338474783:2049] [node 2] ICP30 unregister session Session# [6:7577789232338474873:2049] VirtualId# [6:7577789232338474864:0]
2025-11-28T14:52:25.047581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.047597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789232338474783:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23692) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.047604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23692) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.056943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789234308895590:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:25.056991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789232338474783:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:25.084297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:25.084489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789230594002780:2050] [node 3] ICS07 socket disconnect 520 reason# EndOfStream
2025-11-28T14:52:25.084532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789230594002780:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.084606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:25.084667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789230594002780:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:25.085079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:25.085227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789234308895675:2050] [node 3] ICS01 socket: 534 reason# EndOfStream
2025-11-28T14:52:25.085263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789234308895591:2050] [node 3] ICP30 unregister session Session# [5:7577789234308895675:2050] VirtualId# [5:7577789234308895670:0]
2025-11-28T14:52:25.085286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789234308895675:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.085337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:25.085457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789230594002697:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2255) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.085375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:25.085509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789230594002780:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.085531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789230594002697:2050] [node 3] ICP30 unregister session Session# [4:7577789230594002780:2050] VirtualId# [4:7577789230594002776:0]
2025-11-28T14:52:25.085577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:25.085601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789230594002697:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2255) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.085614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2255) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.085876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:25.086006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789232338474879:2050] [node 3] ICS01 socket: 545 reason# EndOfStream
2025-11-28T14:52:25.086038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789232338474784:2050] [node 3] ICP30 unregister session Session# [6:7577789232338474879:2050] VirtualId# [6:7577789232338474871:0]
2025-11-28T14:52:25.086057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789232338474879:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.086102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:25.086141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:25.105917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:25.106125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789234308895681:2108] [node 4] ICS07 socket disconnect 535 reason# EndOfStream
2025-11-28T14:52:25.106164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789234308895681:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.106262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:25.106313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789234308895681:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:25.106667Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:25.106789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789232338474869:2106] [node 4] ICS07 socket disconnect 536 reason# EndOfStream
2025-11-28T14:52:25.106811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789232338474869:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.106860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:25.106896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789232338474869:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:25.107121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789234308895652:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22130) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.107179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789234308895681:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.107217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789234308895652:2108] [node 4] ICP30 unregister session Session# [5:7577789234308895681:2108] VirtualId# [5:7577789234308895674:0]
2025-11-28T14:52:25.107266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:25.107296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789234308895652:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.107310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.107658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789232338474846:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22130) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.107697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789232338474869:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.107722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789232338474846:2106] [node 4] ICP30 unregister session Session# [6:7577789232338474869:2106] VirtualId# [6:7577789232338474865:0]
2025-11-28T14:52:25.107747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:25.107769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789232338474846:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.107784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.117363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789234308895652:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:25.117373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789232338474846:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:25.139405Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:25.139630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789232338474885:2108] [node 5] ICS01 socket: 499 reason# EndOfStream
2025-11-28T14:52:25.139740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789232338474847:2108] [node 5] ICP30 unregister session Session# [6:7577789232338474885:2108] VirtualId# [6:7577789232338474881:0]
2025-11-28T14:52:25.139785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789232338474885:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.139881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:25.139943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:25.196318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789237759799595:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:25.196322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789237759799594:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:25.213493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789235225724610:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.213528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789235225724611:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:25.230104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789235087037074:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.230145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789235087037075:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:25.246743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789234817966579:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.246786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789234817966581:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:25.246815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789234817966580:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:25.263595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237297674651:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:25.263618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237297674650:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.263664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237297674652:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:25.280318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789237342214057:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.280351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789237342214058:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:25.280408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789237342214059:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:25.333840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789237342214118:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:25.350723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789237759799657:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:25.353741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789235087037137:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:25.356097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789234817966643:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:25.356276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789235087037138:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:25.356432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789235225724674:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:25.356699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789237759799658:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:25.356517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789237342214121:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:25.357203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237297674713:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:25.357072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789235087037139:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:25.356930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789235225724675:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:25.357291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789234817966644:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:25.357450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789237759799659:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:25.357166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789235225724676:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:25.358484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237297674714:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:25.360379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789237759799594:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46486214ns) {Configure:104(46479998ns) {ProcessConfigured:135(291026ns) {StartInitialHandshake:157(288746ns) {DropHandshakes:470(1952ns) {DropIncomingHandshake:442(619ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(444ns) PrepareNewSessionHandshake:477(1615ns) {SwitchToState:179(517ns)}}}}}
2025-11-28T14:52:25.360377Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789237297674651:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26492173ns) {Configure:104(26485260ns) {ProcessConfigured:135(239366ns) {StartInitialHandshake:157(237117ns) {DropHandshakes:470(1449ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(529ns) PrepareNewSessionHandshake:477(3792ns) {SwitchToState:179(581ns)}}}}}
2025-11-28T14:52:25.360775Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789234817966580:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43403746ns) {Configure:104(43399146ns) {ProcessConfigured:135(115167ns) {StartInitialHandshake:157(113069ns) {DropHandshakes:470(6348ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(762ns)} GenerateSessionVirtualId:988(16717ns) PrepareNewSessionHandshake:477(1196ns) {SwitchToState:179(333ns)}}}}}
2025-11-28T14:52:25.361377Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789237342214057:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27154985ns) {Configure:104(27146006ns) {ProcessConfigured:135(230192ns) {StartInitialHandshake:157(228172ns) {DropHandshakes:470(1229ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(354ns)} GenerateSessionVirtualId:988(520ns) PrepareNewSessionHandshake:477(1473ns) {SwitchToState:179(577ns)}}}}}
2025-11-28T14:52:25.361750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789235087037075:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45757561ns) {Configure:104(45754638ns) {ProcessConfigured:135(105258ns) {StartInitialHandshake:157(103808ns) {DropHandshakes:470(8700ns) {DropIncomingHandshake:442(462ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(560ns) PrepareNewSessionHandshake:477(1111ns) {SwitchToState:179(631ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.366229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789237759799673:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35444
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.369332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789235225724687:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29032
2025-11-28T14:52:25.368500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789234817966655:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51533
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.368970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237297674728:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23600
2025-11-28T14:52:25.370579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789234817966659:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51533
2025-11-28T14:52:25.369987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789235225724688:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29032
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.374905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789235225724693:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29032
2025-11-28T14:52:25.374707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789235087037148:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-28T14:52:25.375407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789235087037150:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.375788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789235225724692:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29032
2025-11-28T14:52:25.375818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789234817966655:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.376215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789237759799674:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35444
2025-11-28T14:52:25.376204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237297674728:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.376402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789235225724688:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.376218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789235087037150:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.376422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789234817966579:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.376554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789235225724687:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.376523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789235087037155:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-28T14:52:25.377171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799665:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.377204Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789237759799665:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.377438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789235087037156:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-28T14:52:25.379030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789234817966660:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51533
2025-11-28T14:52:25.379046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789237759799676:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35444
2025-11-28T14:52:25.378588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789235087037148:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.379057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789235087037156:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.379141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799673:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.379163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789235225724693:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.379194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789235087037074:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.379282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799674:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.379529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789237759799675:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35444
2025-11-28T14:52:25.379615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789237759799677:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35444
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.380503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789237759799665:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.380564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799675:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.380943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789235225724694:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29032
2025-11-28T14:52:25.380830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799676:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.381517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789235225724692:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.381532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789235225724610:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.381546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789235087037155:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.381806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789234817966662:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51533
2025-11-28T14:52:25.381945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799663:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.381841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789234817966660:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.381983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789235087037075:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.381989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789234817966659:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.381983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789237759799663:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.382776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789235225724679:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.382181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789237759799657:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.382816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789235225724679:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.382405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:25.383376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789234817966581:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.382818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789237759799657:2107] [node 4] ICP22 created new session: [1:7577789237759799683:2107]
2025-11-28T14:52:25.383484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789235225724694:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.383707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789234817966580:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.384378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789234817966655:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:25.382858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789237759799683:2107] [node 4] ICS09 handshake done sender: [1:7577789237759799665:2112] self: [1:7577789237759799664:0] peer: [4:7577789234817966661:0] socket: 494 qp: -1
2025-11-28T14:52:25.384641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789235087037146:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.384855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789235225724682:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.383265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789237759799683:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:25.384878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789235225724682:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.384677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789235087037146:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.385141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789234817966579:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.383362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.384986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789235225724679:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:25.383566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799661:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.385210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:25.383581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789237759799661:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.385609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789234817966579:2048] [node 1] ICP22 created new session: [4:7577789234817966665:2048]
2025-11-28T14:52:25.384384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799677:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.385835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789234817966665:2048] [node 1] ICS09 handshake done sender: [4:7577789234817966655:2116] self: [4:7577789234817966661:0] peer: [1:7577789237759799664:0] socket: 502 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.386732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789235225724682:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.386069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789234817966665:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:25.387015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789235087037146:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.386589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:25.387375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789237759799663:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:25.387254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789235087037161:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-28T14:52:25.387867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789235225724611:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.388098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789237759799661:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.387931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:25.388948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789237759799595:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.388212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789235225724611:2049] [node 3] ICP22 created new session: [2:7577789235225724702:2049]
2025-11-28T14:52:25.388993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:25.389021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789237759799595:2049] [node 3] ICP22 created new session: [1:7577789237759799689:2049]
2025-11-28T14:52:25.389033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237297674730:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23600
2025-11-28T14:52:25.389057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789237759799689:2049] [node 3] ICS09 handshake done sender: [1:7577789237759799663:2111] self: [1:7577789237759799662:0] peer: [3:7577789235087037158:0] socket: 493 qp: -1
2025-11-28T14:52:25.389059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789235087037137:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.388864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789235225724702:2049] [node 3] ICS09 handshake done sender: [2:7577789235225724679:2111] self: [2:7577789235225724677:0] peer: [3:7577789235087037160:0] socket: 512 qp: -1
2025-11-28T14:52:25.389080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789237759799689:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:25.389097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:25.389098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789235087037155:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:25.388903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789235225724702:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:25.389051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789235225724674:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.389213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:25.389272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789237759799594:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.389060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:25.389092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:25.389407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.389336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789235087037137:2107] [node 4] ICP22 created new session: [3:7577789235087037164:2107]
2025-11-28T14:52:25.389142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789235225724674:2107] [node 4] ICP22 created new session: [2:7577789235225724705:2107]
2025-11-28T14:52:25.389528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789237759799594:2048] [node 2] ICP22 created new session: [1:7577789237759799691:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.389233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789235225724705:2107] [node 4] ICS09 handshake done sender: [2:7577789235225724682:2113] self: [2:7577789235225724681:0] peer: [4:7577789234817966664:0] socket: 517 qp: -1
2025-11-28T14:52:25.389652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789237759799691:2048] [node 2] ICS09 handshake done sender: [1:7577789237759799661:2110] self: [1:7577789237759799660:0] peer: [2:7577789235225724696:0] socket: 492 qp: -1
2025-11-28T14:52:25.389503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789235087037164:2107] [node 4] ICS09 handshake done sender: [3:7577789235087037146:2112] self: [3:7577789235087037144:0] peer: [4:7577789234817966663:0] socket: 527 qp: -1
2025-11-28T14:52:25.389617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789235087037164:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:25.389420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789235225724705:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:25.389656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.389502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.389681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789237759799691:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:25.389959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789234817966667:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51533
2025-11-28T14:52:25.389813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.390122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789234817966668:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51533
2025-11-28T14:52:25.391413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789235087037075:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.391574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.391643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789235087037075:2049] [node 2] ICP22 created new session: [3:7577789235087037166:2049]
2025-11-28T14:52:25.391610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789234817966659:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:25.391737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789235087037166:2049] [node 2] ICS09 handshake done sender: [3:7577789235087037155:2117] self: [3:7577789235087037160:0] peer: [2:7577789235225724677:0] socket: 525 qp: -1
2025-11-28T14:52:25.391829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789235087037166:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:25.391851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.392153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789234817966660:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.393793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237297674731:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23600
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.394410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789234817966581:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11387) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789234817966660:2118] held: no
2025-11-28T14:52:25.394408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237297674730:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.394887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789237297674650:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.395347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214135:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.394565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789234817966581:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.395359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799667:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.394922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789234817966580:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.395387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789237759799667:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.394959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.395957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789235225724610:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789235225724687:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.396431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789235087037075:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:25.395299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.395327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789234817966581:2050] [node 3] ICP22 created new session: [4:7577789234817966670:2050]
2025-11-28T14:52:25.396653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214136:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.396698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789235225724703:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29032
2025-11-28T14:52:25.396665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789237342214135:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.396755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789237759799667:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:25.396730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237297674733:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23600
2025-11-28T14:52:25.395335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789234817966580:2049] [node 2] ICP22 created new session: [4:7577789234817966671:2049]
2025-11-28T14:52:25.396761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789237342214057:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.395924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789234817966670:2050] [node 3] ICS09 handshake done sender: [4:7577789234817966660:2118] self: [4:7577789234817966663:0] peer: [3:7577789235087037144:0] socket: 532 qp: -1
2025-11-28T14:52:25.395946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789234817966670:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:25.395966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789234817966671:2049] [node 2] ICS09 handshake done sender: [4:7577789234817966659:2117] self: [4:7577789234817966664:0] peer: [2:7577789235225724681:0] socket: 520 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.395997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789234817966671:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.396028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.396380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:25.397129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237297674734:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23600
2025-11-28T14:52:25.397241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789237759799669:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.397268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789237759799658:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.397230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789235087037163:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-28T14:52:25.397405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789234817966580:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:25.397276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789237759799669:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.397310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.397339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789237759799658:2108] [node 5] ICP22 created new session: [1:7577789237759799695:2108]
2025-11-28T14:52:25.397820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789237759799695:2108] [node 5] ICS09 handshake done sender: [1:7577789237759799667:2113] self: [1:7577789237759799666:0] peer: [5:7577789237297674732:0] socket: 541 qp: -1
2025-11-28T14:52:25.397866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789237759799695:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:25.397898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.397884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237297674730:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:25.398231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789235225724687:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:25.398596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789235087037074:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789235087037148:2114] held: no
2025-11-28T14:52:25.398923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789237297674650:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.398965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:25.398992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237297674650:2048] [node 1] ICP22 created new session: [5:7577789237297674736:2048]
2025-11-28T14:52:25.399076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237297674736:2048] [node 1] ICS09 handshake done sender: [5:7577789237297674730:2116] self: [5:7577789237297674732:0] peer: [1:7577789237759799666:0] socket: 542 qp: -1
2025-11-28T14:52:25.399095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237297674736:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:25.399212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789235087037148:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:25.399123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:25.399722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789237297674650:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:25.399611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789237759799669:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:25.399862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789237759799659:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.399900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.400812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789235225724610:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.400091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789237759799659:2109] [node 6] ICP22 created new session: [1:7577789237759799698:2109]
2025-11-28T14:52:25.401293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789237759799698:2109] [node 6] ICS09 handshake done sender: [1:7577789237759799669:2114] self: [1:7577789237759799668:0] peer: [6:7577789237342214137:0] socket: 522 qp: -1
2025-11-28T14:52:25.401466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789235087037074:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.401477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:25.401472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789237759799698:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:25.401531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789235225724610:2048] [node 1] ICP22 created new session: [2:7577789235225724709:2048]
2025-11-28T14:52:25.401499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:25.401560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789235225724709:2048] [node 1] ICS09 handshake done sender: [2:7577789235225724687:2116] self: [2:7577789235225724696:0] peer: [1:7577789237759799660:0] socket: 500 qp: -1
2025-11-28T14:52:25.401558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:25.401685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789235087037074:2048] [node 1] ICP22 created new session: [3:7577789235087037170:2048]
2025-11-28T14:52:25.401675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789235225724709:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.402004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789235087037170:2048] [node 1] ICS09 handshake done sender: [3:7577789235087037148:2114] self: [3:7577789235087037158:0] peer: [1:7577789237759799662:0] socket: 504 qp: -1
2025-11-28T14:52:25.402200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789235087037170:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:25.401992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: 2025-11-28T14:52:25.402260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
::ffff:127.1.0.0
2025-11-28T14:52:25.402278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237297674735:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23600
2025-11-28T14:52:25.402373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214138:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.402448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237297674733:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.402534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237297674731:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.402740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789237297674651:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.402801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214139:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.402827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789237297674713:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.402995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237297674735:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.403026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789237342214057:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789237342214135:2115] held: no
2025-11-28T14:52:25.403041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789234817966652:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.403013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789235225724686:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.403076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789234817966652:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.403170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789237342214136:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.403104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789237297674652:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.403043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789235225724686:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.403950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789234817966652:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.403361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789237342214139:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.404041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789235087037147:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.403380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789237342214135:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:25.404221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789234817966675:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51533
2025-11-28T14:52:25.404071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789235087037147:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.404043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214140:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.404908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789234817966643:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.404912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789235225724686:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:25.404691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789237342214057:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.405107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.405348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789234817966643:2108] [node 5] ICP22 created new session: [4:7577789234817966677:2108]
2025-11-28T14:52:25.405343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:25.405375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789237342214057:2048] [node 1] ICP22 created new session: [6:7577789237342214142:2048]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:25.405392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789234817966677:2108] [node 5] ICS09 handshake done sender: [4:7577789234817966652:2114] self: [4:7577789234817966651:0] peer: [5:7577789237297674740:0] socket: 529 qp: -1
2025-11-28T14:52:25.405427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789234817966677:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:25.405407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789235225724675:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.405544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789237342214142:2048] [node 1] ICS09 handshake done sender: [6:7577789237342214135:2115] self: [6:7577789237342214137:0] peer: [1:7577789237759799668:0] socket: 526 qp: -1
2025-11-28T14:52:25.405573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.405566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789235087037147:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:25.405558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789237342214140:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.405631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237297674742:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23600
2025-11-28T14:52:25.405648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.405705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789235225724675:2108] [node 5] ICP22 created new session: [2:7577789235225724712:2108]
2025-11-28T14:52:25.405594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789237342214142:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:25.405655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:25.405735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789235225724712:2108] [node 5] ICS09 handshake done sender: [2:7577789235225724686:2115] self: [2:7577789235225724683:0] peer: [5:7577789237297674739:0] socket: 516 qp: -1
2025-11-28T14:52:25.405794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789235087037138:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.405678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789237342214058:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.405756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789235225724712:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:25.405779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.405752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789237342214121:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.405896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789234817966675:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.405960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.406202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789235087037138:2108] [node 5] ICP22 created new session: [3:7577789235087037173:2108]
2025-11-28T14:52:25.406255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789235087037173:2108] [node 5] ICS09 handshake done sender: [3:7577789235087037147:2113] self: [3:7577789235087037145:0] peer: [5:7577789237297674741:0] socket: 547 qp: -1
2025-11-28T14:52:25.406292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789235087037173:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.406333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.406445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789234817966643:2108] [node 5] ICP18 handshake (actor [4:7577789234817966675:2122]) is too old
2025-11-28T14:52:25.406922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789235225724685:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.406981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789234817966654:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.406952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237297674743:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23600
2025-11-28T14:52:25.406993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789234817966654:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.406943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789235225724685:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.407610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789237342214059:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.407560Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237297674731:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.407998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789235225724685:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:25.408731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789237297674651:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26737) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789237297674733:2118] held: no
2025-11-28T14:52:25.408774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789235225724676:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.408799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789234817966654:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.408873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789235087037151:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.408928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.408904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789235087037151:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.409211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789234817966644:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.409548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214148:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.408991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789235225724676:2109] [node 6] ICP22 created new session: [2:7577789235225724715:2109]
2025-11-28T14:52:25.409243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.409735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214147:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.409231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789235225724715:2109] [node 6] ICS09 handshake done sender: [2:7577789235225724685:2114] self: [2:7577789235225724684:0] peer: [6:7577789237342214143:0] socket: 514 qp: -1
2025-11-28T14:52:25.409771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789235225724715:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:25.410786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789235087037151:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:25.409821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789234817966644:2110] [node 6] ICP22 created new session: [4:7577789234817966680:2110]
2025-11-28T14:52:25.410840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789237297674713:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.410068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789234817966680:2110] [node 6] ICS09 handshake done sender: [4:7577789234817966654:2115] self: [4:7577789234817966653:0] peer: [6:7577789237342214145:0] socket: 511 qp: -1
2025-11-28T14:52:25.410068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:25.410878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:25.410259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789234817966680:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:25.410916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237297674713:2108] [node 4] ICP22 created new session: [5:7577789237297674745:2108]
No RDMA context found for GID: 2025-11-28T14:52:25.410660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
2025-11-28T14:52:25.411179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789235087037139:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.410954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237297674745:2108] [node 4] ICS09 handshake done sender: [5:7577789237297674731:2117] self: [5:7577789237297674740:0] peer: [4:7577789234817966651:0] socket: 528 qp: -1
2025-11-28T14:52:25.411180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789237342214058:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26737) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789237342214136:2116] held: no
2025-11-28T14:52:25.411605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.411158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237297674745:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:25.411562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.411693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789235087037139:2109] [node 6] ICP22 created new session: [3:7577789235087037177:2109]
2025-11-28T14:52:25.411705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789235087037175:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-28T14:52:25.411731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789235087037177:2109] [node 6] ICS09 handshake done sender: [3:7577789235087037151:2116] self: [3:7577789235087037149:0] peer: [6:7577789237342214146:0] socket: 550 qp: -1
2025-11-28T14:52:25.411762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789235087037177:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:25.411793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:25.411958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789237297674652:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11387) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789237297674735:2120] held: no
2025-11-28T14:52:25.411954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789237342214139:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.412008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789237297674713:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:25.412160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237297674733:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:25.412100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789237342214136:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:25.412438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789235087037175:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.412470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789237342214121:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.412581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789237342214058:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.412597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:25.412655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789237342214121:2107] [node 4] ICP22 created new session: [6:7577789237342214150:2107]
2025-11-28T14:52:25.412678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237297674744:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23600
2025-11-28T14:52:25.412665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.412688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789237342214058:2049] [node 2] ICP22 created new session: [6:7577789237342214151:2049]
2025-11-28T14:52:25.412627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789235087037139:2109] [node 6] ICP18 handshake (actor [3:7577789235087037175:2121]) is too old
2025-11-28T14:52:25.412691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789237342214150:2107] [node 4] ICS09 handshake done sender: [6:7577789237342214139:2118] self: [6:7577789237342214145:0] peer: [4:7577789234817966653:0] socket: 496 qp: -1
2025-11-28T14:52:25.412717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789237342214150:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:25.412740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.412751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789237342214151:2049] [node 2] ICS09 handshake done sender: [6:7577789237342214136:2116] self: [6:7577789237342214143:0] peer: [2:7577789235225724684:0] socket: 530 qp: -1
2025-11-28T14:52:25.412834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789237297674651:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.412875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.412785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789237342214151:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:25.412908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237297674651:2049] [node 2] ICP22 created new session: [5:7577789237297674747:2049]
2025-11-28T14:52:25.412811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.413138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237297674747:2049] [node 2] ICS09 handshake done sender: [5:7577789237297674733:2118] self: [5:7577789237297674739:0] peer: [2:7577789235225724683:0] socket: 523 qp: -1
2025-11-28T14:52:25.413167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237297674747:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:25.413209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.413452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237297674735:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:25.413767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789237297674652:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:25.413796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
::ffff:127.1.0.0
2025-11-28T14:52:25.413815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237297674652:2050] [node 3] ICP22 created new session: [5:7577789237297674750:2050]
2025-11-28T14:52:25.413856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237297674750:2050] [node 3] ICS09 handshake done sender: [5:7577789237297674735:2120] self: [5:7577789237297674741:0] peer: [3:7577789235087037145:0] socket: 548 qp: -1
2025-11-28T14:52:25.413872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237297674750:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:25.413888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:25.413907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214149:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.414445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789237342214059:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11387) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789237342214140:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.414552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789237342214140:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:25.414582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214155:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.414622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789237342214059:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.414692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:25.414719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789237342214059:2050] [node 3] ICP22 created new session: [6:7577789237342214156:2050]
2025-11-28T14:52:25.414742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789237342214156:2050] [node 3] ICS09 handshake done sender: [6:7577789237342214140:2119] self: [6:7577789237342214146:0] peer: [3:7577789235087037149:0] socket: 510 qp: -1
2025-11-28T14:52:25.414760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789237342214156:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:25.414782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:25.415375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789237342214155:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.415451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789237342214118:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.415588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237297674724:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.415613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789237297674724:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.416251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237297674724:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.416346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789237342214159:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55292
2025-11-28T14:52:25.416409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789237297674714:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.416478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.416526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237297674714:2109] [node 6] ICP22 created new session: [5:7577789237297674753:2109]
2025-11-28T14:52:25.416556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237297674753:2109] [node 6] ICS09 handshake done sender: [5:7577789237297674724:2114] self: [5:7577789237297674723:0] peer: [6:7577789237342214158:0] socket: 495 qp: -1
2025-11-28T14:52:25.416555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789237342214155:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:25.416601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237297674753:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:25.416641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:25.416799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789237342214118:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.416862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.416899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789237342214118:2106] [node 5] ICP22 created new session: [6:7577789237342214160:2106]
2025-11-28T14:52:25.416935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789237342214160:2106] [node 5] ICS09 handshake done sender: [6:7577789237342214155:2123] self: [6:7577789237342214158:0] peer: [5:7577789237297674723:0] socket: 509 qp: -1
2025-11-28T14:52:25.416954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789237342214160:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:25.416971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.417128Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789237342214118:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:25.442629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:25.442792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789235225724709:2048] [node 1] ICS01 socket: 500 reason# EndOfStream
2025-11-28T14:52:25.442833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789235225724610:2048] [node 1] ICP30 unregister session Session# [2:7577789235225724709:2048] VirtualId# [2:7577789235225724696:0]
2025-11-28T14:52:25.442858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789235225724709:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.442922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.442957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.443821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:25.444017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789235087037170:2048] [node 1] ICS01 socket: 504 reason# EndOfStream
2025-11-28T14:52:25.444072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789235087037074:2048] [node 1] ICP30 unregister session Session# [3:7577789235087037170:2048] VirtualId# [3:7577789235087037158:0]
2025-11-28T14:52:25.444100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789235087037170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.444161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.444203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.445901Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:25.446041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789234817966665:2048] [node 1] ICS07 socket disconnect 502 reason# EndOfStream
2025-11-28T14:52:25.446058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789234817966665:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.446116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.446172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789234817966665:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:25.446694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789234817966579:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29834) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.446763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789234817966665:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.446789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789234817966579:2048] [node 1] ICP30 unregister session Session# [4:7577789234817966665:2048] VirtualId# [4:7577789234817966661:0]
2025-11-28T14:52:25.446821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.446859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789234817966579:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.446868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.446837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:25.447033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789237297674736:2048] [node 1] ICS07 socket disconnect 542 reason# EndOfStream
2025-11-28T14:52:25.447082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789237297674736:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.447145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.447189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789237297674736:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:25.447705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789237297674650:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29834) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.447738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789237297674736:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.447737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:25.447755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789237297674650:2048] [node 1] ICP30 unregister session Session# [5:7577789237297674736:2048] VirtualId# [5:7577789237297674732:0]
2025-11-28T14:52:25.447811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.447829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789237297674650:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.447845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.447923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789237342214142:2048] [node 1] ICS01 socket: 526 reason# EndOfStream
2025-11-28T14:52:25.447951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789237342214057:2048] [node 1] ICP30 unregister session Session# [6:7577789237342214142:2048] VirtualId# [6:7577789237342214137:0]
2025-11-28T14:52:25.447969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789237342214142:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.448034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.448074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.456293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789234817966579:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:25.457261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789237297674650:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:25.475977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:25.476192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789235087037166:2049] [node 2] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-28T14:52:25.476230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789235087037166:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.476317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.476385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789235087037166:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:25.477148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789235087037075:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26737) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.477215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789235087037166:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.477273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789235087037075:2049] [node 2] ICP30 unregister session Session# [3:7577789235087037166:2049] VirtualId# [3:7577789235087037160:0]
2025-11-28T14:52:25.477331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.477401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789235087037075:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26737) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.477428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26737) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.477995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:25.478116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789234817966671:2049] [node 2] ICS01 socket: 520 reason# EndOfStream
2025-11-28T14:52:25.478145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789234817966580:2049] [node 2] ICP30 unregister session Session# [4:7577789234817966671:2049] VirtualId# [4:7577789234817966664:0]
2025-11-28T14:52:25.478168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789234817966671:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.478218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.478253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.478768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:25.478902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789237297674747:2049] [node 2] ICS01 socket: 523 reason# EndOfStream
2025-11-28T14:52:25.478926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789237297674651:2049] [node 2] ICP30 unregister session Session# [5:7577789237297674747:2049] VirtualId# [5:7577789237297674739:0]
2025-11-28T14:52:25.478944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789237297674747:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.478990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.479014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.479241Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:25.479414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789237342214151:2049] [node 2] ICS01 socket: 530 reason# EndOfStream
2025-11-28T14:52:25.479464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789237342214058:2049] [node 2] ICP30 unregister session Session# [6:7577789237342214151:2049] VirtualId# [6:7577789237342214143:0]
2025-11-28T14:52:25.479489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789237342214151:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.479550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.479587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.487033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789235087037075:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:25.524122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:25.524272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789234817966670:2050] [node 3] ICS01 socket: 532 reason# EndOfStream
2025-11-28T14:52:25.524303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789234817966581:2050] [node 3] ICP30 unregister session Session# [4:7577789234817966670:2050] VirtualId# [4:7577789234817966663:0]
2025-11-28T14:52:25.524326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789234817966670:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.524410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:25.524446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:25.525089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:25.525262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789237297674750:2050] [node 3] ICS01 socket: 548 reason# EndOfStream
2025-11-28T14:52:25.525299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789237297674652:2050] [node 3] ICP30 unregister session Session# [5:7577789237297674750:2050] VirtualId# [5:7577789237297674741:0]
2025-11-28T14:52:25.525321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789237297674750:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.525378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:25.525434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:25.525925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:25.526046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789237342214156:2050] [node 3] ICS01 socket: 510 reason# EndOfStream
2025-11-28T14:52:25.526069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789237342214059:2050] [node 3] ICP30 unregister session Session# [6:7577789237342214156:2050] VirtualId# [6:7577789237342214146:0]
2025-11-28T14:52:25.526089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789237342214156:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.526135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:25.526158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:25.563858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:25.563999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789237297674745:2108] [node 4] ICS07 socket disconnect 528 reason# EndOfStream
2025-11-28T14:52:25.564025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789237297674745:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.564086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:25.564125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789237297674745:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:25.564727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789237297674713:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.564809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789237297674745:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.564869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789237297674713:2108] [node 4] ICP30 unregister session Session# [5:7577789237297674745:2108] VirtualId# [5:7577789237297674740:0]
2025-11-28T14:52:25.564911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:25.564940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789237297674713:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.564951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.564928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:25.565139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789237342214150:2107] [node 4] ICS07 socket disconnect 496 reason# EndOfStream
2025-11-28T14:52:25.565198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789237342214150:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.565272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:25.565324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789237342214150:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:25.565877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789237342214121:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.565960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789237342214150:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.566007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789237342214121:2107] [node 4] ICP30 unregister session Session# [6:7577789237342214150:2107] VirtualId# [6:7577789237342214145:0]
2025-11-28T14:52:25.566051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:25.566083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789237342214121:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.566098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.576073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789237342214121:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:25.596793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:25.596947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789237342214160:2106] [node 5] ICS07 socket disconnect 509 reason# EndOfStream
2025-11-28T14:52:25.596968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789237342214160:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.597023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:25.597073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789237342214160:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:25.597585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789237342214118:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12380) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.597623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789237342214160:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.597642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789237342214118:2106] [node 5] ICP30 unregister session Session# [6:7577789237342214160:2106] VirtualId# [6:7577789237342214158:0]
2025-11-28T14:52:25.597682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:25.597715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789237342214118:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12380) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.597726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12380) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.607826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789237342214118:2106] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:25.654689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789238669359766:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:25.654700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789238669359767:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:25.671419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789237780882169:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:25.671429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789237780882168:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.687905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789238393136555:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.688272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789238393136556:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:25.705523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789238836882171:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.705566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789238836882172:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:25.705616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789238836882173:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:25.722991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237159713650:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.723022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237159713651:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:25.723042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237159713652:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:25.741020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789236081950270:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:25.741060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789236081950272:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:25.741158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789236081950271:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:25.811382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789236081950330:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:25.823828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789238669359829:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:25.823900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789237780882231:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:25.823854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789236081950331:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:25.824459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237159713713:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:25.824484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789238669359830:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:25.824452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789238393136627:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:25.824101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789237780882232:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:25.824834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789238393136628:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:25.824701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789237159713714:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:25.825136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789237780882233:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:25.825256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789238836882234:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:25.824859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789238393136629:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:25.826043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789238836882235:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:25.825863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789238669359831:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:25.829835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789238836882172:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36063053ns) {Configure:104(36052223ns) {ProcessConfigured:135(46181ns) {StartInitialHandshake:157(44508ns) {DropHandshakes:470(1089ns) {DropIncomingHandshake:442(382ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(2569ns) PrepareNewSessionHandshake:477(731ns) {SwitchToState:179(375ns)}}}}}
2025-11-28T14:52:25.829871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789238669359766:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51241657ns) {Configure:104(51238257ns) {ProcessConfigured:135(225323ns) {StartInitialHandshake:157(223443ns) {DropHandshakes:470(1630ns) {DropIncomingHandshake:442(460ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(566ns) PrepareNewSessionHandshake:477(961ns) {SwitchToState:179(281ns)}}}}}
2025-11-28T14:52:25.829978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789238669359767:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51134622ns) {Configure:104(48088162ns) {ProcessConfigured:135(246832ns) {StartInitialHandshake:157(245292ns) {DropHandshakes:470(1594ns) {DropIncomingHandshake:442(455ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(536ns) PrepareNewSessionHandshake:477(963ns) {SwitchToState:179(354ns)}}}}}
2025-11-28T14:52:25.832304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789236081950270:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30266605ns) {Configure:104(30257420ns) {ProcessConfigured:135(472614ns) {StartInitialHandshake:157(470764ns) {DropHandshakes:470(1296ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:477(1589ns) {SwitchToState:179(529ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.837370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789237780882240:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48411
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.837661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789238393136643:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45870
2025-11-28T14:52:25.837676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789238669359841:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43289
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.837993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789237780882241:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48411
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.838122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789238669359840:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43289
2025-11-28T14:52:25.838132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789238669359847:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43289
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.838868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789238393136643:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.838561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359841:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.839064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789238393136644:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45870
2025-11-28T14:52:25.839257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789238836882246:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5893
2025-11-28T14:52:25.839547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359840:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.839645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789237780882241:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.839659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789237780882240:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:25.839806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789238836882246:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.839785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789238393136645:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45870
2025-11-28T14:52:25.840132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237159713728:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57204
2025-11-28T14:52:25.840021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789238836882250:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5893
2025-11-28T14:52:25.840055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789238836882173:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.840276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789238836882249:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5893
2025-11-28T14:52:25.840436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789238393136635:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.840478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789238393136635:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.840531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789238393136646:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45870
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.843486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789238393136635:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.843610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789237780882249:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48411
2025-11-28T14:52:25.843613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789238393136645:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.844093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789237780882251:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48411
2025-11-28T14:52:25.844077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789238669359850:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43289
2025-11-28T14:52:25.844356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789238393136555:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.844078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789238669359849:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43289
2025-11-28T14:52:25.845276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789237780882250:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48411
2025-11-28T14:52:25.845578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789238393136627:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.846311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237159713728:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.846458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789238836882250:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.846108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789237780882249:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.845567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359836:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.846622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789237780882168:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.845618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.845587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789238669359836:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.846842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789237780882251:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.846237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359847:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.848177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789238836882249:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.846027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789238393136644:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.846889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359850:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.846179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789238393136627:2107] [node 4] ICP22 created new session: [3:7577789238393136650:2107]
2025-11-28T14:52:25.847445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789237780882236:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.848567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789238836882171:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.847185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359849:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.847478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789237780882236:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.848732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359834:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.846434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789238393136556:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.848781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789238836882253:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5893
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.846704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789238393136650:2107] [node 4] ICS09 handshake done sender: [3:7577789238393136635:2121] self: [3:7577789238393136634:0] peer: [4:7577789238836882252:0] socket: 501 qp: -1
2025-11-28T14:52:25.849051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789238836882172:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.848753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789238669359834:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.848179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789237780882250:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.847123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789238393136650:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:25.850405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950348:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.849958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359837:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.850431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789238836882246:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:25.847420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.849739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789237780882236:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:25.851620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789238836882173:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.851698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:25.849735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789238393136646:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.849925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789237780882239:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.850000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789238669359837:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.849938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789237780882239:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.851870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789238836882173:2050] [node 3] ICP22 created new session: [4:7577789238836882256:2050]
2025-11-28T14:52:25.850241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789238669359836:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:25.852092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789238836882256:2050] [node 3] ICS09 handshake done sender: [4:7577789238836882246:2115] self: [4:7577789238836882252:0] peer: [3:7577789238393136634:0] socket: 510 qp: -1
2025-11-28T14:52:25.851750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789237780882169:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.852330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789238836882256:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:25.850405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789238669359834:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.853497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789236081950348:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.851793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:25.854188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789238393136654:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45870
2025-11-28T14:52:25.852487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:25.852479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789238669359837:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.854063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789236081950272:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.852055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789237780882169:2049] [node 3] ICP22 created new session: [2:7577789237780882260:2049]
2025-11-28T14:52:25.854197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789238393136655:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45870
2025-11-28T14:52:25.853202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789238669359767:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.852449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789237780882260:2049] [node 3] ICS09 handshake done sender: [2:7577789237780882236:2110] self: [2:7577789237780882234:0] peer: [3:7577789238393136652:0] socket: 512 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.853242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:25.852786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789237780882260:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:25.854210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789238393136627:2107] [node 4] ICP18 handshake (actor [3:7577789238393136646:2127]) is too old
2025-11-28T14:52:25.853722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789238669359767:2049] [node 3] ICP22 created new session: [1:7577789238669359858:2049]
2025-11-28T14:52:25.853790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789238669359766:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.856148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789238836882258:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5893
2025-11-28T14:52:25.852807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.853845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.854526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789237780882239:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.854211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789238669359858:2049] [node 3] ICS09 handshake done sender: [1:7577789238669359836:2111] self: [1:7577789238669359832:0] peer: [3:7577789238393136649:0] socket: 516 qp: -1
2025-11-28T14:52:25.854216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789238669359766:2048] [node 2] ICP22 created new session: [1:7577789238669359859:2048]
2025-11-28T14:52:25.856462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789238393136556:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789238393136644:2125] held: no
2025-11-28T14:52:25.854806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789237780882259:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48411
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.854321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789238669359859:2048] [node 2] ICS09 handshake done sender: [1:7577789238669359834:2110] self: [1:7577789238669359833:0] peer: [2:7577789237780882254:0] socket: 515 qp: -1
2025-11-28T14:52:25.856523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789238836882259:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5893
2025-11-28T14:52:25.855850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789237780882231:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.856534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789238393136637:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.856558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789238393136637:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.854349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789238669359858:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:25.856163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:25.858471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950350:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.854463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789238669359859:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:25.856261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789237780882231:2107] [node 4] ICP22 created new session: [2:7577789237780882263:2107]
2025-11-28T14:52:25.854489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:25.857444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789238393136644:2125] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:25.854501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.856472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789237780882263:2107] [node 4] ICS09 handshake done sender: [2:7577789237780882239:2112] self: [2:7577789237780882235:0] peer: [4:7577789238836882255:0] socket: 514 qp: -1
2025-11-28T14:52:25.855889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789238669359829:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.856514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789237780882263:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:25.856232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:25.858737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789238393136645:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:25.859569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789238836882250:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:25.856559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.859084Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789238393136555:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789238393136645:2126] held: no
2025-11-28T14:52:25.858250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789237780882249:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:25.856461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789238669359829:2107] [node 4] ICP22 created new session: [1:7577789238669359862:2107]
2025-11-28T14:52:25.859789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789238836882249:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:25.862860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789237780882168:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.859554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789238393136555:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.856512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789238669359862:2107] [node 4] ICS09 handshake done sender: [1:7577789238669359837:2112] self: [1:7577789238669359835:0] peer: [4:7577789238836882254:0] socket: 517 qp: -1
2025-11-28T14:52:25.862896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789238836882172:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789238836882249:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.861632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:25.862996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.856547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789238669359862:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:25.863162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789238836882171:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.862003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789238393136555:2048] [node 1] ICP22 created new session: [3:7577789238393136656:2048]
2025-11-28T14:52:25.856577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.863132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789237780882168:2048] [node 1] ICP22 created new session: [2:7577789237780882266:2048]
2025-11-28T14:52:25.863205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:25.862840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789238393136556:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.863392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789238836882171:2048] [node 1] ICP22 created new session: [4:7577789238836882261:2048]
2025-11-28T14:52:25.863291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789237780882266:2048] [node 1] ICS09 handshake done sender: [2:7577789237780882249:2117] self: [2:7577789237780882254:0] peer: [1:7577789238669359833:0] socket: 524 qp: -1
2025-11-28T14:52:25.862867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789238393136656:2048] [node 1] ICS09 handshake done sender: [3:7577789238393136645:2126] self: [3:7577789238393136649:0] peer: [1:7577789238669359832:0] socket: 521 qp: -1
2025-11-28T14:52:25.863476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789237780882266:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:25.863594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789238836882172:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.865996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237159713730:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57204
2025-11-28T14:52:25.863504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.863004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789238393136656:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:25.863628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.863057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:25.863852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789238836882261:2048] [node 1] ICS09 handshake done sender: [4:7577789238836882250:2117] self: [4:7577789238836882254:0] peer: [1:7577789238669359835:0] socket: 522 qp: -1
2025-11-28T14:52:25.866266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950351:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.863447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.866503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237159713731:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57204
2025-11-28T14:52:25.864818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789238836882172:2049] [node 2] ICP22 created new session: [4:7577789238836882262:2049]
2025-11-28T14:52:25.863898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789238393136556:2049] [node 2] ICP22 created new session: [3:7577789238393136659:2049]
2025-11-28T14:52:25.866303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789236081950350:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.865328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789238836882261:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:25.866541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789237780882168:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:25.865619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789238393136659:2049] [node 2] ICS09 handshake done sender: [3:7577789238393136644:2125] self: [3:7577789238393136652:0] peer: [2:7577789237780882234:0] socket: 519 qp: -1
2025-11-28T14:52:25.865385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:25.865664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789238393136659:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:25.866548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950352:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.865628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789238836882262:2049] [node 2] ICS09 handshake done sender: [4:7577789238836882249:2116] self: [4:7577789238836882255:0] peer: [2:7577789237780882235:0] socket: 520 qp: -1
2025-11-28T14:52:25.866568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789236081950271:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.865700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.866697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237159713730:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.865662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789238836882262:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:25.866043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.866820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237159713731:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.866801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789237780882246:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.866611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789238836882171:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:25.866862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789237159713651:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.866862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789238393136637:2122] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.866866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789236081950352:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.866831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789237780882246:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.866958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789237159713650:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.866985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950354:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.867061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789236081950270:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.867112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789237780882248:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.867137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789237780882248:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.867200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789238393136629:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.867176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359839:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.869476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237159713733:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57204
2025-11-28T14:52:25.869956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.870242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789236081950348:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:25.869832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789237780882248:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:25.867203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789238669359839:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.870016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789238393136629:2109] [node 6] ICP22 created new session: [3:7577789238393136662:2109]
2025-11-28T14:52:25.870331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789237780882233:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.869498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789238669359846:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.870673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789236081950272:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.870681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789238393136662:2109] [node 6] ICS09 handshake done sender: [3:7577789238393136637:2122] self: [3:7577789238393136636:0] peer: [6:7577789236081950349:0] socket: 542 qp: -1
2025-11-28T14:52:25.869530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789238669359846:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.870710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789236081950351:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.870450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.871241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789238393136662:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:25.870565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789237780882233:2108] [node 5] ICP22 created new session: [2:7577789237780882270:2108]
2025-11-28T14:52:25.871376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789238669359839:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:25.870718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:25.871299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:25.871234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789237780882270:2108] [node 5] ICS09 handshake done sender: [2:7577789237780882248:2116] self: [2:7577789237780882247:0] peer: [5:7577789237159713732:0] socket: 498 qp: -1
2025-11-28T14:52:25.872401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789238669359830:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.872278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237159713735:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57204
2025-11-28T14:52:25.872552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789238669359846:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:25.871262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789236081950272:2050] [node 3] ICP22 created new session: [6:7577789236081950356:2050]
2025-11-28T14:52:25.872599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.871253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789237780882270:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:25.872737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789238669359830:2108] [node 5] ICP22 created new session: [1:7577789238669359868:2108]
2025-11-28T14:52:25.872766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237159713733:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.871892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.873183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789238669359868:2108] [node 5] ICS09 handshake done sender: [1:7577789238669359839:2113] self: [1:7577789238669359838:0] peer: [5:7577789237159713734:0] socket: 533 qp: -1
2025-11-28T14:52:25.872056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789236081950356:2050] [node 3] ICS09 handshake done sender: [6:7577789236081950348:2115] self: [6:7577789236081950349:0] peer: [3:7577789238393136636:0] socket: 543 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 2025-11-28T14:52:25.871934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789237780882246:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:25.872968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789237159713651:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789237159713730:2116] held: no
::ffff:127.1.0.0
2025-11-28T14:52:25.873499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789238669359868:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.872797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789237780882232:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.872842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.872091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789236081950356:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:25.873558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237159713735:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.873529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.872139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:25.872885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789237780882232:2109] [node 6] ICP22 created new session: [2:7577789237780882272:2109]
2025-11-28T14:52:25.874135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237159713736:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57204
2025-11-28T14:52:25.873541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789238669359831:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.873949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789236081950331:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.873002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789237780882272:2109] [node 6] ICS09 handshake done sender: [2:7577789237780882246:2115] self: [2:7577789237780882245:0] peer: [6:7577789236081950353:0] socket: 506 qp: -1
2025-11-28T14:52:25.873761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.874134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237159713737:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57204
2025-11-28T14:52:25.874789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789238836882243:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.873169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789237780882272:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:25.874838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789238836882243:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.873937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789238669359831:2109] [node 6] ICP22 created new session: [1:7577789238669359870:2109]
2025-11-28T14:52:25.874134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950357:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.873209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:25.874106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789238669359870:2109] [node 6] ICS09 handshake done sender: [1:7577789238669359846:2116] self: [1:7577789238669359843:0] peer: [6:7577789236081950355:0] socket: 513 qp: -1
2025-11-28T14:52:25.874141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789238669359870:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:25.874450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:25.876379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789237159713650:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789237159713731:2117] held: no
2025-11-28T14:52:25.876557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789237159713652:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.876656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237159713730:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:25.876689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789238836882243:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:25.876672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237159713731:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:25.877587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789237159713651:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.877628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.878037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789237159713714:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.878051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789238836882235:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.878121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.878065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237159713651:2049] [node 2] ICP22 created new session: [5:7577789237159713739:2049]
2025-11-28T14:52:25.878309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789237159713650:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.878378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789238393136642:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.878380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789238836882235:2109] [node 6] ICP22 created new session: [4:7577789238836882267:2109]
2025-11-28T14:52:25.878325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789236081950350:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:25.878328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:25.878414Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789238393136642:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.879246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789238836882267:2109] [node 6] ICS09 handshake done sender: [4:7577789238836882243:2113] self: [4:7577789238836882242:0] peer: [6:7577789236081950359:0] socket: 499 qp: -1
2025-11-28T14:52:25.878388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237159713650:2048] [node 1] ICP22 created new session: [5:7577789237159713742:2048]
2025-11-28T14:52:25.879289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789236081950270:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789236081950352:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.879264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789238836882245:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.879455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789236081950271:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789236081950350:2116] held: no
2025-11-28T14:52:25.879349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789238393136642:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:25.879272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789238836882267:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:25.879772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789238393136628:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.878392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237159713739:2049] [node 2] ICS09 handshake done sender: [5:7577789237159713730:2116] self: [5:7577789237159713732:0] peer: [2:7577789237780882247:0] socket: 518 qp: -1
2025-11-28T14:52:25.879581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950360:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.879041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237159713742:2048] [node 1] ICS09 handshake done sender: [5:7577789237159713731:2117] self: [5:7577789237159713734:0] peer: [1:7577789238669359838:0] socket: 534 qp: -1
2025-11-28T14:52:25.879289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789238836882245:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.879068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237159713739:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:25.879292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:25.879593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789236081950271:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.879876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.879140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.879983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789238393136628:2108] [node 5] ICP22 created new session: [3:7577789238393136665:2108]
2025-11-28T14:52:25.879240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237159713742:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:25.879748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:25.879258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:25.880389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789238393136665:2108] [node 5] ICS09 handshake done sender: [3:7577789238393136642:2123] self: [3:7577789238393136638:0] peer: [5:7577789237159713738:0] socket: 554 qp: -1
2025-11-28T14:52:25.879810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789238836882268:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5893
2025-11-28T14:52:25.880799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789238393136665:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:25.879775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789236081950271:2049] [node 2] ICP22 created new session: [6:7577789236081950363:2049]
2025-11-28T14:52:25.880896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: 2025-11-28T14:52:25.879801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789236081950363:2049] [node 2] ICS09 handshake done sender: [6:7577789236081950350:2116] self: [6:7577789236081950353:0] peer: [2:7577789237780882245:0] socket: 509 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:25.879822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789236081950363:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:25.881062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789238836882245:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.879841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:25.881253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789238836882270:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5893
No RDMA context found for GID: 2025-11-28T14:52:25.881693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789238836882234:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:25.880899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789236081950352:2118] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:25.881642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950362:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.881721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.881933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789236081950270:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.882040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789238836882234:2108] [node 5] ICP22 created new session: [4:7577789238836882272:2108]
2025-11-28T14:52:25.881955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:25.882020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237159713746:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57204
2025-11-28T14:52:25.882087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789238836882272:2108] [node 5] ICS09 handshake done sender: [4:7577789238836882245:2114] self: [4:7577789238836882244:0] peer: [5:7577789237159713740:0] socket: 556 qp: -1
2025-11-28T14:52:25.882024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789236081950270:2048] [node 1] ICP22 created new session: [6:7577789236081950366:2048]
2025-11-28T14:52:25.882270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789238836882272:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:25.882091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789236081950366:2048] [node 1] ICS09 handshake done sender: [6:7577789236081950352:2118] self: [6:7577789236081950355:0] peer: [1:7577789238669359843:0] socket: 523 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.882123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789236081950366:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:25.882325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.882143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:25.882530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950365:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.882604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789237159713652:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11955) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789237159713733:2118] held: no
2025-11-28T14:52:25.882636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789236081950351:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.882786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789236081950331:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.882817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.883271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237159713733:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:25.883240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789236081950331:2106] [node 4] ICP22 created new session: [6:7577789236081950368:2106]
2025-11-28T14:52:25.883267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789236081950368:2106] [node 4] ICS09 handshake done sender: [6:7577789236081950351:2117] self: [6:7577789236081950359:0] peer: [4:7577789238836882242:0] socket: 527 qp: -1
2025-11-28T14:52:25.883339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789237159713747:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57204
2025-11-28T14:52:25.883286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789236081950368:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:25.883312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.883618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789237159713652:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.883648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:25.883672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237159713652:2050] [node 3] ICP22 created new session: [5:7577789237159713748:2050]
2025-11-28T14:52:25.883738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237159713748:2050] [node 3] ICS09 handshake done sender: [5:7577789237159713733:2118] self: [5:7577789237159713738:0] peer: [3:7577789238393136638:0] socket: 555 qp: -1
2025-11-28T14:52:25.884106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237159713748:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:25.884127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:25.884143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789236081950365:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.884258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237159713735:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:25.884344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789237159713714:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.884331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789236081950330:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:25.884407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:25.884423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789238836882270:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.884497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237159713714:2108] [node 4] ICP22 created new session: [5:7577789237159713750:2108]
2025-11-28T14:52:25.884527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789238836882234:2108] [node 5] ICP18 handshake (actor [4:7577789238836882270:2122]) is too old
2025-11-28T14:52:25.884586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237159713750:2108] [node 4] ICS09 handshake done sender: [5:7577789237159713735:2119] self: [5:7577789237159713740:0] peer: [4:7577789238836882244:0] socket: 557 qp: -1
2025-11-28T14:52:25.884637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237159713750:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:25.884662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:25.884746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789238836882235:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:25.884788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789237159713725:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:25.884826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789237159713725:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:25.886330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789237159713725:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:25.886458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789237159713713:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:25.886515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:25.886547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789237159713713:2109] [node 6] ICP22 created new session: [5:7577789237159713753:2109]
2025-11-28T14:52:25.886575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789237159713753:2109] [node 6] ICS09 handshake done sender: [5:7577789237159713725:2113] self: [5:7577789237159713724:0] peer: [6:7577789236081950370:0] socket: 528 qp: -1
2025-11-28T14:52:25.886594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789237159713753:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:25.886556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789236081950371:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-11-28T14:52:25.886616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:25.886768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789236081950365:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:25.887590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789236081950330:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29919) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789236081950365:2123] held: no
2025-11-28T14:52:25.887728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789236081950330:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:25.887766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:25.887802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789236081950330:2105] [node 5] ICP22 created new session: [6:7577789236081950372:2105]
2025-11-28T14:52:25.887829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789236081950372:2105] [node 5] ICS09 handshake done sender: [6:7577789236081950365:2123] self: [6:7577789236081950370:0] peer: [5:7577789237159713724:0] socket: 567 qp: -1
2025-11-28T14:52:25.887850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789236081950372:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:25.887878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:25.931824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:25.931997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789237780882266:2048] [node 1] ICS07 socket disconnect 524 reason# EndOfStream
2025-11-28T14:52:25.932025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789237780882266:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.932102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.932155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789237780882266:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:25.932863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:25.933103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789238393136656:2048] [node 1] ICS01 socket: 521 reason# EndOfStream
2025-11-28T14:52:25.933154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789238393136555:2048] [node 1] ICP30 unregister session Session# [3:7577789238393136656:2048] VirtualId# [3:7577789238393136649:0]
2025-11-28T14:52:25.933186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789238393136656:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.933257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.933323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.934099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789237780882168:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:25.934152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789237780882266:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:25.935545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:25.935793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789237780882168:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.935812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789238836882261:2048] [node 1] ICS07 socket disconnect 522 reason# EndOfStream
2025-11-28T14:52:25.935844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789237780882266:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.935847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789238836882261:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.935939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.935946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789237780882168:2048] [node 1] ICP30 unregister session Session# [2:7577789237780882266:2048] VirtualId# [2:7577789237780882254:0]
2025-11-28T14:52:25.936010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.935983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789238836882261:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:25.936035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789237780882168:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.936045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.936257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:25.936505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789237159713742:2048] [node 1] ICS01 socket: 534 reason# EndOfStream
2025-11-28T14:52:25.936552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789237159713650:2048] [node 1] ICP30 unregister session Session# [5:7577789237159713742:2048] VirtualId# [5:7577789237159713734:0]
2025-11-28T14:52:25.936610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789237159713742:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.936697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.936749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.936889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789238836882171:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:25.936964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789238836882261:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:25.937009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789238836882171:2048] [node 1] ICP30 unregister session Session# [4:7577789238836882261:2048] VirtualId# [4:7577789238836882254:0]
2025-11-28T14:52:25.937061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.937002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:25.937114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789238836882171:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.937132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:25.937148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789236081950366:2048] [node 1] ICS01 socket: 523 reason# EndOfStream
2025-11-28T14:52:25.937181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789236081950270:2048] [node 1] ICP30 unregister session Session# [6:7577789236081950366:2048] VirtualId# [6:7577789236081950355:0]
2025-11-28T14:52:25.937199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789236081950366:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.937256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:25.937329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:25.946778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789238836882171:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:25.978078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:25.978313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789238393136659:2049] [node 2] ICS01 socket: 519 reason# EndOfStream
2025-11-28T14:52:25.978358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789238393136556:2049] [node 2] ICP30 unregister session Session# [3:7577789238393136659:2049] VirtualId# [3:7577789238393136652:0]
2025-11-28T14:52:25.978395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789238393136659:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.978468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.978537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.980378Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:25.980526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789238836882262:2049] [node 2] ICS01 socket: 520 reason# EndOfStream
2025-11-28T14:52:25.980551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789238836882172:2049] [node 2] ICP30 unregister session Session# [4:7577789238836882262:2049] VirtualId# [4:7577789238836882255:0]
2025-11-28T14:52:25.980570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789238836882262:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.980630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.980662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.981104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:25.981343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789237159713739:2049] [node 2] ICS01 socket: 518 reason# EndOfStream
2025-11-28T14:52:25.981394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789237159713651:2049] [node 2] ICP30 unregister session Session# [5:7577789237159713739:2049] VirtualId# [5:7577789237159713732:0]
2025-11-28T14:52:25.981424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789237159713739:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.981510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.981570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:25.981636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:25.981763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789236081950363:2049] [node 2] ICS01 socket: 509 reason# EndOfStream
2025-11-28T14:52:25.981792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789236081950271:2049] [node 2] ICP30 unregister session Session# [6:7577789236081950363:2049] VirtualId# [6:7577789236081950353:0]
2025-11-28T14:52:25.981809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789236081950363:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:25.981882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:25.981929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:26.038841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:26.039006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789238836882256:2050] [node 3] ICS07 socket disconnect 510 reason# EndOfStream
2025-11-28T14:52:26.039037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789238836882256:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.039096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:26.039131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789238836882256:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:26.039703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:26.039879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789237159713748:2050] [node 3] ICS01 socket: 555 reason# EndOfStream
2025-11-28T14:52:26.039905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789238836882173:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11955) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:26.039944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789237159713652:2050] [node 3] ICP30 unregister session Session# [5:7577789237159713748:2050] VirtualId# [5:7577789237159713738:0]
2025-11-28T14:52:26.039988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789238836882256:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:26.039975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789237159713748:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.040031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:26.040155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789238836882173:2050] [node 3] ICP30 unregister session Session# [4:7577789238836882256:2050] VirtualId# [4:7577789238836882252:0]
2025-11-28T14:52:26.040153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:26.040210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:26.040242Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789238836882173:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11955) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.040259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11955) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.040670Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:26.040820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789236081950356:2050] [node 3] ICS01 socket: 543 reason# EndOfStream
2025-11-28T14:52:26.040857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789236081950272:2050] [node 3] ICP30 unregister session Session# [6:7577789236081950356:2050] VirtualId# [6:7577789236081950349:0]
2025-11-28T14:52:26.040880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789236081950356:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.040939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:26.040990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:26.050237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789238836882173:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:26.090094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:26.090249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789237159713750:2108] [node 4] ICS07 socket disconnect 557 reason# EndOfStream
2025-11-28T14:52:26.090282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789237159713750:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.090362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:26.090419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789237159713750:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:26.090711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:26.090905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789236081950368:2106] [node 4] ICS07 socket disconnect 527 reason# EndOfStream
2025-11-28T14:52:26.090965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789236081950368:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.091049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:26.091101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789236081950368:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:26.091061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789237159713714:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1303) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:26.091152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789237159713750:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:26.091200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789237159713714:2108] [node 4] ICP30 unregister session Session# [5:7577789237159713750:2108] VirtualId# [5:7577789237159713740:0]
2025-11-28T14:52:26.091239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:26.091278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789237159713714:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1303) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.091289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1303) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.091788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789236081950331:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1303) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:26.091835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789236081950368:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:26.091911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789236081950331:2106] [node 4] ICP30 unregister session Session# [6:7577789236081950368:2106] VirtualId# [6:7577789236081950359:0]
2025-11-28T14:52:26.091951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:26.091982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789236081950331:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1303) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.092001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1303) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.101403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789236081950331:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:26.129258Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:26.129409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789236081950372:2105] [node 5] ICS01 socket: 567 reason# EndOfStream
2025-11-28T14:52:26.129445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789236081950330:2105] [node 5] ICP30 unregister session Session# [6:7577789236081950372:2105] VirtualId# [6:7577789236081950370:0]
2025-11-28T14:52:26.129467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789236081950372:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.129559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:26.129620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:26.204800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789239819025999:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:26.205071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789239819026000:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:26.222636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789241900559929:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.222810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789241900559930:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:26.238989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789240179707144:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.239020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789240179707145:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:26.256479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789240639385902:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:26.256483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789240639385901:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.256568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789240639385903:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:26.275477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789239356909896:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:26.275497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789239356909895:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:26.275530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789239356909894:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.294513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789240570440681:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:26.294515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789240570440680:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.294573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789240570440682:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:26.368807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789239819026064:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:26.376585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789239356909957:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:26.374640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789239819026063:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:26.375852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789241900559994:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:26.375865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789240179707208:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:26.376384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789239819026065:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:26.376690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789241900559993:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:26.375859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789240179707207:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:26.376710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789240570440741:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:26.376811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789240179707209:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:26.376787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789241900559995:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:26.376835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789239356909958:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:26.376814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789240639385964:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:26.377242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789240570440744:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:26.378144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789240639385965:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:26.382375Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789240639385903:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36169520ns) {Configure:104(36159661ns) {ProcessConfigured:135(133289ns) {StartInitialHandshake:157(131755ns) {DropHandshakes:470(1373ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:477(996ns) {SwitchToState:179(497ns)}}}}}
2025-11-28T14:52:26.382399Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789240639385902:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41566943ns) {Configure:104(41560945ns) {ProcessConfigured:135(130762ns) {StartInitialHandshake:157(126089ns) {DropHandshakes:470(1396ns) {DropIncomingHandshake:442(409ns) DropOutgoingHandshake:457(258ns)} GenerateSessionVirtualId:988(906ns) PrepareNewSessionHandshake:477(3572ns) {SwitchToState:179(624ns)}}}}}
2025-11-28T14:52:26.383094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789241900559929:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46903287ns) {Configure:104(46896211ns) {ProcessConfigured:135(716995ns) {StartInitialHandshake:157(715413ns) {DropHandshakes:470(1715ns) {DropIncomingHandshake:442(552ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:477(1053ns) {SwitchToState:179(405ns)}}}}}
2025-11-28T14:52:26.382587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789240639385901:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36567572ns) {Configure:104(36556345ns) {ProcessConfigured:135(299583ns) {StartInitialHandshake:157(297431ns) {DropHandshakes:470(1033ns) {DropIncomingHandshake:442(267ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(1111ns) {SwitchToState:179(445ns)}}}}}
2025-11-28T14:52:26.383566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789239356909895:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(37036980ns) {Configure:104(37027037ns) {ProcessConfigured:135(148946ns) {StartInitialHandshake:157(146793ns) {DropHandshakes:470(1143ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(666ns) {SwitchToState:179(304ns)}}}}}
2025-11-28T14:52:26.384254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789240570440682:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(37316950ns) {Configure:104(37313352ns) {ProcessConfigured:135(641927ns) {StartInitialHandshake:157(640076ns) {DropHandshakes:470(1035ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(212ns)} GenerateSessionVirtualId:988(389ns) PrepareNewSessionHandshake:477(1089ns) {SwitchToState:179(623ns)}}}}}
2025-11-28T14:52:26.384282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789240570440680:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(37666937ns) {Configure:104(37662609ns) {ProcessConfigured:135(351508ns) {StartInitialHandshake:157(349740ns) {DropHandshakes:470(911ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(470ns) PrepareNewSessionHandshake:477(791ns) {SwitchToState:179(325ns)}}}}}
2025-11-28T14:52:26.384846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789240570440681:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(37864553ns) {Configure:104(37857713ns) {ProcessConfigured:135(934588ns) {StartInitialHandshake:157(933187ns) {DropHandshakes:470(9600ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:477(1009ns) {SwitchToState:179(143ns)}}}}}
2025-11-28T14:52:26.385232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789239819026000:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21432330ns) {Configure:104(21417557ns) {ProcessConfigured:135(353210ns) {StartInitialHandshake:157(351014ns) {DropHandshakes:470(1177ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(423ns) PrepareNewSessionHandshake:477(1326ns) {SwitchToState:179(538ns)}}}}}
2025-11-28T14:52:26.385287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789239819025999:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47734454ns) {Configure:104(47727844ns) {ProcessConfigured:135(767552ns) {StartInitialHandshake:157(765667ns) {DropHandshakes:470(1071ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(371ns)} GenerateSessionVirtualId:988(272ns) PrepareNewSessionHandshake:477(602ns) {SwitchToState:179(144ns)}}}}}
2025-11-28T14:52:26.385720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789239356909894:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(41532231ns) {Configure:104(41528487ns) {ProcessConfigured:135(1258431ns) {StartInitialHandshake:157(1257889ns) {DropHandshakes:470(174524ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(642ns) PrepareNewSessionHandshake:477(1247ns) {SwitchToState:179(528ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.390511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789241900560008:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63516
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:26.390550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440756:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
2025-11-28T14:52:26.390981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789239819026077:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48655
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.390867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789240639385979:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57108
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.390569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440757:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
2025-11-28T14:52:26.392911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789240179707221:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64639
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.392897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789241900560011:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63516
2025-11-28T14:52:26.391155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789239819026078:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48655
2025-11-28T14:52:26.392760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789240639385977:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57108
2025-11-28T14:52:26.394133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789241900560010:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63516
2025-11-28T14:52:26.394246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789239356909971:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39788
2025-11-28T14:52:26.394005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789240179707225:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64639
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.392916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789239819026080:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48655
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.394254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789240179707222:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64639
2025-11-28T14:52:26.394366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789239356909970:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39788
2025-11-28T14:52:26.395344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789240570440756:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.395290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789241900560010:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.394372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789239819026077:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.394257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789240179707223:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64639
2025-11-28T14:52:26.396049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789240570440757:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.396059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789239356909971:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.397535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789241900560011:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.398962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789240639385977:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.398204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789240179707221:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.399196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789240639385973:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.400449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789241900560000:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.400464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789239819026067:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.397560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789240570440681:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.396926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789239356909958:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.399226Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789240639385973:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.400496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789241900560000:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.399210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789240570440744:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.400497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789239819026067:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.397505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789239356909972:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39788
2025-11-28T14:52:26.398781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789240179707224:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64639
2025-11-28T14:52:26.401962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789239819026072:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.399621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789240639385901:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.399632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789239356909972:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.404348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789241900560008:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.400879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789240179707223:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.401999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789239819026072:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.403173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789240639385979:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.404400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789241900560001:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.399782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789239356909894:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.401748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789240179707224:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.404422Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789241900560001:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.403393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789240639385973:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:26.404316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789239819026080:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.399962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789239356909960:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.405115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789241900560000:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:26.403109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789240179707145:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.404611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789239819026078:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.399978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789239356909960:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.403157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789240179707222:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.403133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789239356909970:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.405541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789240639385964:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.404745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789239819026072:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:26.404336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789240179707144:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.404633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789239356909896:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.405253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789239819026071:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.406020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.404436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789240179707225:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.407174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789241900560001:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:26.405282Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789239819026071:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.405093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789239356909960:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:26.406163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789240639385964:2108] [node 5] ICP22 created new session: [4:7577789240639385984:2108]
2025-11-28T14:52:26.405352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789240179707215:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.407334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789241900559994:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.405372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789240179707215:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.407674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:26.407831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789239356909957:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.407855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440762:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
2025-11-28T14:52:26.407625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789240179707215:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:26.407054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789240639385984:2108] [node 5] ICS09 handshake done sender: [4:7577789240639385973:2113] self: [4:7577789240639385971:0] peer: [5:7577789239356909975:0] socket: 510 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.407133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789239819026063:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.407102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789240639385984:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:26.407895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789239356909978:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39788
2025-11-28T14:52:26.407341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789239819026067:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:26.407135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.407735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789241900559994:2109] [node 6] ICP22 created new session: [2:7577789241900560017:2109]
2025-11-28T14:52:26.408041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:26.408621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440763:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.407468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:26.407757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789241900559930:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.408813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789240639385985:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57108
2025-11-28T14:52:26.409347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789240570440756:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:26.408463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789239356909957:2109] [node 6] ICP22 created new session: [5:7577789239356909982:2109]
2025-11-28T14:52:26.408054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789239819026063:2109] [node 5] ICP22 created new session: [1:7577789239819026088:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.407938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:26.409772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789240570440757:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:26.408465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789239356909980:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39788
2025-11-28T14:52:26.410999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789240639385977:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:26.410780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789240179707234:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64639
2025-11-28T14:52:26.408510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789239819026088:2109] [node 5] ICS09 handshake done sender: [1:7577789239819026072:2113] self: [1:7577789239819026068:0] peer: [5:7577789239356909976:0] socket: 532 qp: -1
2025-11-28T14:52:26.407939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789241900560017:2109] [node 6] ICS09 handshake done sender: [2:7577789241900560000:2111] self: [2:7577789241900559999:0] peer: [6:7577789240570440760:0] socket: 505 qp: -1
2025-11-28T14:52:26.411914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789240639385901:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.408508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789239356909982:2109] [node 6] ICS09 handshake done sender: [5:7577789239356909960:2110] self: [5:7577789239356909959:0] peer: [6:7577789240570440761:0] socket: 515 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.410954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789240179707235:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64639
2025-11-28T14:52:26.410926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789240570440681:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.408539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789239819026071:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:26.411945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:26.410965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:26.408105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789241900560017:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:26.408616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789239356909982:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:26.408780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789239819026088:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:26.411858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789240179707208:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.413017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789240639385901:2048] [node 1] ICP22 created new session: [4:7577789240639385989:2048]
2025-11-28T14:52:26.408140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:26.408981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:26.411720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789240570440744:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.413651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:26.408813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:26.413663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789240639385989:2048] [node 1] ICS09 handshake done sender: [4:7577789240639385977:2114] self: [4:7577789240639385981:0] peer: [1:7577789239819026066:0] socket: 531 qp: -1
2025-11-28T14:52:26.407969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789241900559930:2049] [node 3] ICP22 created new session: [2:7577789241900560019:2049]
2025-11-28T14:52:26.411281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789239356909972:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:26.409516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789239819026064:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.411832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789240570440681:2049] [node 2] ICP22 created new session: [6:7577789240570440764:2049]
2025-11-28T14:52:26.411577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789239356909971:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:26.408516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789241900560019:2049] [node 3] ICS09 handshake done sender: [2:7577789241900560001:2112] self: [2:7577789241900559998:0] peer: [3:7577789240179707228:0] socket: 504 qp: -1
2025-11-28T14:52:26.414269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789240639385989:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:26.409787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:26.412987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789239356909981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39788
2025-11-28T14:52:26.413666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789240179707145:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789240179707221:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.411882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:26.408634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789241900560019:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:26.414306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:26.413893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789239356909894:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.413699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789240179707208:2108] [node 5] ICP22 created new session: [3:7577789240179707236:2108]
2025-11-28T14:52:26.408662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:26.412144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789240570440744:2107] [node 5] ICP22 created new session: [6:7577789240570440765:2107]
2025-11-28T14:52:26.410498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789239819026064:2107] [node 4] ICP22 created new session: [1:7577789239819026092:2107]
2025-11-28T14:52:26.415440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789240639385988:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57108
2025-11-28T14:52:26.413937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:26.413603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789240570440764:2049] [node 2] ICS09 handshake done sender: [6:7577789240570440756:2114] self: [6:7577789240570440760:0] peer: [2:7577789241900559999:0] socket: 516 qp: -1
2025-11-28T14:52:26.417464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789241900560023:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63516
2025-11-28T14:52:26.413756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789240179707236:2108] [node 5] ICS09 handshake done sender: [3:7577789240179707215:2111] self: [3:7577789240179707212:0] peer: [5:7577789239356909979:0] socket: 507 qp: -1
2025-11-28T14:52:26.417007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789240639385988:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.411001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789239819026092:2107] [node 4] ICS09 handshake done sender: [1:7577789239819026067:2111] self: [1:7577789239819026066:0] peer: [4:7577789240639385981:0] socket: 520 qp: -1
2025-11-28T14:52:26.413966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789239356909894:2048] [node 1] ICP22 created new session: [5:7577789239356909986:2048]
2025-11-28T14:52:26.413815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789240179707223:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:26.414079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789240570440764:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:26.417294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789240639385902:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.411027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789239819026092:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:26.414269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789240570440765:2107] [node 5] ICS09 handshake done sender: [6:7577789240570440757:2115] self: [6:7577789240570440761:0] peer: [5:7577789239356909959:0] socket: 530 qp: -1
2025-11-28T14:52:26.414113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789240179707236:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:26.414121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789239356909986:2048] [node 1] ICS09 handshake done sender: [5:7577789239356909972:2116] self: [5:7577789239356909976:0] peer: [1:7577789239819026068:0] socket: 536 qp: -1
2025-11-28T14:52:26.418136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789241900560003:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.414314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789240570440765:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:26.414821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:26.414281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789239356909986:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:26.411260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789239819026000:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.414865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:26.418158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789241900560003:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.415453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:26.415444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789240179707221:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:26.415521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:26.411597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:26.420885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789241900560003:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.415696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789240179707144:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.415586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789239356909958:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.421420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789241900559993:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.412109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:26.416047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:26.415619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:26.421662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:26.416085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789240179707144:2048] [node 1] ICP22 created new session: [3:7577789240179707240:2048]
2025-11-28T14:52:26.413041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789239819026000:2049] [node 3] ICP22 created new session: [1:7577789239819026094:2049]
2025-11-28T14:52:26.415834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789239356909958:2108] [node 4] ICP22 created new session: [5:7577789239356909988:2108]
No RDMA context found for GID: 2025-11-28T14:52:26.421832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789241900559993:2107] [node 4] ICP22 created new session: [2:7577789241900560025:2107]
2025-11-28T14:52:26.421674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789240639385993:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57108
2025-11-28T14:52:26.416262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789240179707240:2048] [node 1] ICS09 handshake done sender: [3:7577789240179707223:2116] self: [3:7577789240179707229:0] peer: [1:7577789239819026070:0] socket: 533 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:26.414525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789239819026094:2049] [node 3] ICS09 handshake done sender: [1:7577789239819026071:2112] self: [1:7577789239819026070:0] peer: [3:7577789240179707229:0] socket: 521 qp: -1
2025-11-28T14:52:26.416204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789239356909988:2108] [node 4] ICS09 handshake done sender: [5:7577789239356909971:2115] self: [5:7577789239356909975:0] peer: [4:7577789240639385971:0] socket: 529 qp: -1
2025-11-28T14:52:26.423101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440768:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
2025-11-28T14:52:26.422543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789241900560025:2107] [node 4] ICS09 handshake done sender: [2:7577789241900560003:2114] self: [2:7577789241900559996:0] peer: [4:7577789240639385992:0] socket: 564 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.416959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789240179707240:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.417206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789239356909988:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:26.422592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789241900560025:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:26.439173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440769:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
2025-11-28T14:52:26.417057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:26.422638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:26.414842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789239819026094:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:26.417241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:26.417792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789240179707145:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.444011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789240639385994:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57108
2025-11-28T14:52:26.421860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789239356909970:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:26.414872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:26.444320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440770:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
2025-11-28T14:52:26.417823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:26.415469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789239819026089:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48655
2025-11-28T14:52:26.419304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789240179707145:2049] [node 2] ICP22 created new session: [3:7577789240179707242:2049]
2025-11-28T14:52:26.445015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789240639385902:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789240639385988:2118] held: no
2025-11-28T14:52:26.420741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789239819026064:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:26.420428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789240179707242:2049] [node 2] ICS09 handshake done sender: [3:7577789240179707221:2114] self: [3:7577789240179707228:0] peer: [2:7577789241900559998:0] socket: 513 qp: -1
2025-11-28T14:52:26.444027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789239819026097:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48655
2025-11-28T14:52:26.422520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789239356909896:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789239356909970:2114] held: no
2025-11-28T14:52:26.420631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789240179707242:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:26.420880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:26.444000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789239356909984:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39788
2025-11-28T14:52:26.445382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789240639385988:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:26.445491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789240570440768:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.445184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789239356909896:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.463197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789240639385902:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.445354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:26.469450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789241900560023:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.469232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789240570440770:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.445523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789239356909896:2050] [node 3] ICP22 created new session: [5:7577789239356909990:2050]
2025-11-28T14:52:26.469375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:26.469449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789240570440682:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.454545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789239356909984:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.469568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789240639385902:2049] [node 2] ICP22 created new session: [4:7577789240639385995:2049]
2025-11-28T14:52:26.470258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789241900559993:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(23674338ns) {HandleHandshakeStatus:329(23671886ns) {DropIncomingHandshake:442(16255ns) DropOutgoingHandshake:457(367ns) SwitchToState:179(827ns) ProcessPendingSessionEvents:495(5867130ns) {ForwardSessionEventToSession:612(5116708ns) ForwardSessionEventToSession:612(694632ns) ForwardSessionEventToSession:612(9403ns) ForwardSessionEventToSession:612(2755ns) ForwardSessionEventToSession:612(2126ns) ForwardSessionEventToSession:612(13342ns) ForwardSessionEventToSession:612(2441ns) ForwardSessionEventToSession:612(2957ns) ForwardSessionEventToSession:612(5810ns) ForwardSessionEventToSession:612(4628ns)}}}
2025-11-28T14:52:26.470270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789240639385995:2049] [node 2] ICS09 handshake done sender: [4:7577789240639385988:2118] self: [4:7577789240639385992:0] peer: [2:7577789241900559996:0] socket: 524 qp: -1
2025-11-28T14:52:26.470318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789241900559929:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.469584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789240570440680:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.463316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789239356909990:2050] [node 3] ICS09 handshake done sender: [5:7577789239356909970:2114] self: [5:7577789239356909979:0] peer: [3:7577789240179707212:0] socket: 526 qp: -1
2025-11-28T14:52:26.470384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789240639385995:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:26.470382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789240179707227:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.470430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:26.470394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789241900560009:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.470423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789240179707227:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.469172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789239356909895:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.470558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789239819026081:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.470425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789241900560009:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.469235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789239356909990:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:26.470599Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789239819026081:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.469304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:26.472248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789239819026073:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.473410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789240179707227:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:26.472278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789239819026073:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.470483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789239356909896:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25945221ns) {HandleHandshakeStatus:329(25224349ns) {DropIncomingHandshake:442(1477ns) DropOutgoingHandshake:457(281ns) SwitchToState:179(506ns) ProcessPendingSessionEvents:495(691ns)}}
2025-11-28T14:52:26.482196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789241900560009:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.482330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789239819026081:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:26.490704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789240179707209:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.490722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789239819026073:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:26.495277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:26.495532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440773:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
2025-11-28T14:52:26.495259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789241900559995:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.495615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789239819025999:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.496265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789240179707209:2109] [node 6] ICP22 created new session: [3:7577789240179707245:2109]
2025-11-28T14:52:26.496642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789240570440682:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789240570440768:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.496294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789239819026065:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.495597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:26.497889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789240179707245:2109] [node 6] ICS09 handshake done sender: [3:7577789240179707227:2119] self: [3:7577789240179707226:0] peer: [6:7577789240570440771:0] socket: 525 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.496602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:26.498089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789240570440768:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.496597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789241900559995:2108] [node 5] ICP22 created new session: [2:7577789241900560030:2108]
2025-11-28T14:52:26.497944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789240179707245:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:26.496653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789239819025999:2048] [node 2] ICP22 created new session: [1:7577789239819026100:2048]
2025-11-28T14:52:26.499013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789239356909993:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39788
2025-11-28T14:52:26.499397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440774:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
2025-11-28T14:52:26.497826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789241900560030:2108] [node 5] ICS09 handshake done sender: [2:7577789241900560009:2116] self: [2:7577789241900560007:0] peer: [5:7577789239356909991:0] socket: 565 qp: -1
2025-11-28T14:52:26.498201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:26.497801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:26.499475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789240570440682:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.497943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789241900560030:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:26.499078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789239356909895:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789239356909984:2121] held: no
2025-11-28T14:52:26.497849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789239819026100:2048] [node 2] ICS09 handshake done sender: [1:7577789239819026073:2114] self: [1:7577789239819026069:0] peer: [2:7577789241900560027:0] socket: 567 qp: -1
2025-11-28T14:52:26.499627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:26.500874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789239356909984:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:26.497977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:26.497852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789239819026065:2110] [node 6] ICP22 created new session: [1:7577789239819026101:2110]
2025-11-28T14:52:26.501577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789239356909895:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.497895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789239819026100:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:26.500264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789240570440682:2050] [node 3] ICP22 created new session: [6:7577789240570440775:2050]
2025-11-28T14:52:26.498127Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789241900559929:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789241900560023:2119] held: no
2025-11-28T14:52:26.502053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:26.500574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789241900560029:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63516
2025-11-28T14:52:26.497907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789239819026101:2110] [node 6] ICS09 handshake done sender: [1:7577789239819026081:2118] self: [1:7577789239819026079:0] peer: [6:7577789240570440772:0] socket: 572 qp: -1
2025-11-28T14:52:26.502210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789239356909895:2049] [node 2] ICP22 created new session: [5:7577789239356909994:2049]
2025-11-28T14:52:26.497944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:26.501460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789241900560023:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:26.500577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789240570440680:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789240570440770:2121] held: no
2025-11-28T14:52:26.501608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789241900559929:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.501371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789240570440775:2050] [node 3] ICS09 handshake done sender: [6:7577789240570440768:2119] self: [6:7577789240570440771:0] peer: [3:7577789240179707226:0] socket: 558 qp: -1
2025-11-28T14:52:26.502407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789239356909994:2049] [node 2] ICS09 handshake done sender: [5:7577789239356909984:2121] self: [5:7577789239356909991:0] peer: [2:7577789241900560007:0] socket: 566 qp: -1
2025-11-28T14:52:26.497946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789239819026101:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:26.502006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:26.501545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789240570440770:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:26.502495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789239356909994:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:26.499033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:26.501569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789240570440775:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:26.502361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789241900559929:2048] [node 1] ICP22 created new session: [2:7577789241900560032:2048]
2025-11-28T14:52:26.502560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:26.501615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:26.502680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789240179707209:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(19065606ns) {HandleHandshakeStatus:329(19063245ns) {DropIncomingHandshake:442(21552ns) DropOutgoingHandshake:457(439ns) SwitchToState:179(843ns) ProcessPendingSessionEvents:495(1793423ns) {ForwardSessionEventToSession:612(1026542ns) ForwardSessionEventToSession:612(3966ns) ForwardSessionEventToSession:612(2903ns) ForwardSessionEventToSession:612(10005ns) ForwardSessionEventToSession:612(3433ns) ForwardSessionEventToSession:612(473192ns) ForwardSessionEventToSession:612(3157ns) ForwardSessionEventToSession:612(1907ns) ForwardSessionEventToSession:612(256376ns) ForwardSessionEventToSession:612(2859ns)}}}
2025-11-28T14:52:26.502399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789241900560032:2048] [node 1] ICS09 handshake done sender: [2:7577789241900560023:2119] self: [2:7577789241900560027:0] peer: [1:7577789239819026069:0] socket: 511 qp: -1
2025-11-28T14:52:26.502529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789240570440680:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.502493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789241900560032:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:26.502593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:26.502631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789240570440680:2048] [node 1] ICP22 created new session: [6:7577789240570440777:2048]
2025-11-28T14:52:26.502528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:26.502672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789240570440777:2048] [node 1] ICS09 handshake done sender: [6:7577789240570440770:2121] self: [6:7577789240570440772:0] peer: [1:7577789239819026079:0] socket: 573 qp: -1
2025-11-28T14:52:26.502723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789240570440777:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:26.502761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:26.503899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789240639385993:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.504019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789240639385903:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.504202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789240179707217:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.504227Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789240179707217:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.505050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789240179707217:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.505203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789240179707207:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.505274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789240639385998:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57108
2025-11-28T14:52:26.505283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:26.505315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789240179707207:2107] [node 4] ICP22 created new session: [3:7577789240179707248:2107]
2025-11-28T14:52:26.505353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789240179707248:2107] [node 4] ICS09 handshake done sender: [3:7577789240179707217:2113] self: [3:7577789240179707214:0] peer: [4:7577789240639385997:0] socket: 569 qp: -1
2025-11-28T14:52:26.505375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789240179707248:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:26.505405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:26.505472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789240639385993:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:26.505722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789240639385903:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789240639385993:2119] held: no
2025-11-28T14:52:26.505970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789240639385903:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.505998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:26.506020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789240639385903:2050] [node 3] ICP22 created new session: [4:7577789240639385999:2050]
2025-11-28T14:52:26.506042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789240639385999:2050] [node 3] ICS09 handshake done sender: [4:7577789240639385993:2119] self: [4:7577789240639385997:0] peer: [3:7577789240179707214:0] socket: 570 qp: -1
2025-11-28T14:52:26.506067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789240639385999:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:26.506097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:26.508083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789240570440769:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.508235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789240570440741:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:26.508411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789240639385980:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:26.508431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789240639385980:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:26.509093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789240639385980:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:26.509286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789240639385965:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:26.509348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:26.509391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789240639385965:2109] [node 6] ICP22 created new session: [4:7577789240639386002:2109]
2025-11-28T14:52:26.509361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789240570440780:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6984
2025-11-28T14:52:26.509439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789240639386002:2109] [node 6] ICS09 handshake done sender: [4:7577789240639385980:2116] self: [4:7577789240639385978:0] peer: [6:7577789240570440779:0] socket: 537 qp: -1
2025-11-28T14:52:26.509470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789240639386002:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:26.509500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:26.509571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789240570440769:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:26.509712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789240570440741:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5343) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789240570440769:2120] held: no
2025-11-28T14:52:26.510574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789240570440741:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:26.510656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:26.510692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789240570440741:2106] [node 4] ICP22 created new session: [6:7577789240570440781:2106]
2025-11-28T14:52:26.510794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789240570440781:2106] [node 4] ICS09 handshake done sender: [6:7577789240570440769:2120] self: [6:7577789240570440779:0] peer: [4:7577789240639385978:0] socket: 571 qp: -1
2025-11-28T14:52:26.510825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789240570440781:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:26.510887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:26.561726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:26.561883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789241900560032:2048] [node 1] ICS01 socket: 511 reason# EndOfStream
2025-11-28T14:52:26.561911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789241900559929:2048] [node 1] ICP30 unregister session Session# [2:7577789241900560032:2048] VirtualId# [2:7577789241900560027:0]
2025-11-28T14:52:26.561934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789241900560032:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.562024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:26.562071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:26.562808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:26.562961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789240179707240:2048] [node 1] ICS07 socket disconnect 533 reason# EndOfStream
2025-11-28T14:52:26.562986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789240179707240:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.563051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:26.563102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789240179707240:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:26.563743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789240179707144:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:26.563822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789240179707240:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:26.563865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789240179707144:2048] [node 1] ICP30 unregister session Session# [3:7577789240179707240:2048] VirtualId# [3:7577789240179707229:0]
2025-11-28T14:52:26.563907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:26.563931Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789240179707144:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.563939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.565427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:26.565543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789240639385989:2048] [node 1] ICS07 socket disconnect 531 reason# EndOfStream
2025-11-28T14:52:26.565564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789240639385989:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.565616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:26.565648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789240639385989:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:26.566170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789240639385901:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:26.566249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789240639385989:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:26.566282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789240639385901:2048] [node 1] ICP30 unregister session Session# [4:7577789240639385989:2048] VirtualId# [4:7577789240639385981:0]
2025-11-28T14:52:26.566229Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:26.566324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:26.566349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789240639385901:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.566356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.566383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789239356909986:2048] [node 1] ICS07 socket disconnect 536 reason# EndOfStream
2025-11-28T14:52:26.566417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789239356909986:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.566490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:26.566525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789239356909986:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:26.566936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:26.567150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789240570440777:2048] [node 1] ICS01 socket: 573 reason# EndOfStream
2025-11-28T14:52:26.567158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789239356909894:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:26.567202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789240570440680:2048] [node 1] ICP30 unregister session Session# [6:7577789240570440777:2048] VirtualId# [6:7577789240570440772:0]
2025-11-28T14:52:26.567230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789240570440777:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.567242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789239356909986:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:26.567272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789239356909894:2048] [node 1] ICP30 unregister session Session# [5:7577789239356909986:2048] VirtualId# [5:7577789239356909976:0]
2025-11-28T14:52:26.567310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:26.567328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789239356909894:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.567309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:26.567359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:26.567335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4030) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.573457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789240179707144:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:26.576483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789240639385901:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:26.577538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789239356909894:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:26.626038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:26.626198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789240179707242:2049] [node 2] ICS01 socket: 513 reason# EndOfStream
2025-11-28T14:52:26.626230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789240179707145:2049] [node 2] ICP30 unregister session Session# [3:7577789240179707242:2049] VirtualId# [3:7577789240179707228:0]
2025-11-28T14:52:26.626251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789240179707242:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.626325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:26.626368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:26.627869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:26.628028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789240639385995:2049] [node 2] ICS01 socket: 524 reason# EndOfStream
2025-11-28T14:52:26.628070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789240639385902:2049] [node 2] ICP30 unregister session Session# [4:7577789240639385995:2049] VirtualId# [4:7577789240639385992:0]
2025-11-28T14:52:26.628095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789240639385995:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.628174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:26.628227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:26.628605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:26.628774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789239356909994:2049] [node 2] ICS01 socket: 566 reason# EndOfStream
2025-11-28T14:52:26.628808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789239356909895:2049] [node 2] ICP30 unregister session Session# [5:7577789239356909994:2049] VirtualId# [5:7577789239356909991:0]
2025-11-28T14:52:26.628830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789239356909994:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.628886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:26.628914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:26.629564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:26.629828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789240570440764:2049] [node 2] ICS07 socket disconnect 516 reason# EndOfStream
2025-11-28T14:52:26.629862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789240570440764:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.629936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:26.629976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789240570440764:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:26.630588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789240570440681:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7416) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:26.630670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789240570440764:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:26.630692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789240570440681:2049] [node 2] ICP30 unregister session Session# [6:7577789240570440764:2049] VirtualId# [6:7577789240570440760:0]
2025-11-28T14:52:26.630721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:26.630744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789240570440681:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7416) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.630757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7416) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.641095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789240570440681:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:26.693890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:26.694066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789240639385999:2050] [node 3] ICS01 socket: 570 reason# EndOfStream
2025-11-28T14:52:26.694114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789240639385903:2050] [node 3] ICP30 unregister session Session# [4:7577789240639385999:2050] VirtualId# [4:7577789240639385997:0]
2025-11-28T14:52:26.694148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789240639385999:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.694219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:26.694264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:26.694645Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:26.694769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789239356909990:2050] [node 3] ICS01 socket: 526 reason# EndOfStream
2025-11-28T14:52:26.694795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789239356909896:2050] [node 3] ICP30 unregister session Session# [5:7577789239356909990:2050] VirtualId# [5:7577789239356909979:0]
2025-11-28T14:52:26.694811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789239356909990:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.694860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:26.694891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:26.695361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:26.695593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789240570440775:2050] [node 3] ICS01 socket: 558 reason# EndOfStream
2025-11-28T14:52:26.695641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789240570440682:2050] [node 3] ICP30 unregister session Session# [6:7577789240570440775:2050] VirtualId# [6:7577789240570440771:0]
2025-11-28T14:52:26.695670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789240570440775:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.695737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:26.695782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:26.760279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:26.760547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789239356909988:2108] [node 4] ICS07 socket disconnect 529 reason# EndOfStream
2025-11-28T14:52:26.760583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789239356909988:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.760700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:26.760756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789239356909988:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:26.761165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:26.761393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789240570440781:2106] [node 4] ICS01 socket: 571 reason# EndOfStream
2025-11-28T14:52:26.761382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789239356909958:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5343) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:26.761435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789239356909988:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:26.761444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789240570440741:2106] [node 4] ICP30 unregister session Session# [6:7577789240570440781:2106] VirtualId# [6:7577789240570440779:0]
2025-11-28T14:52:26.761574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789239356909958:2108] [node 4] ICP30 unregister session Session# [5:7577789239356909988:2108] VirtualId# [5:7577789239356909975:0]
2025-11-28T14:52:26.761486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789240570440781:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.761629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:26.761569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:26.761650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789239356909958:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.761641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:26.761666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.796415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:26.796672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789240570440765:2107] [node 5] ICS07 socket disconnect 530 reason# EndOfStream
2025-11-28T14:52:26.796707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789240570440765:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:26.796803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:26.796858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789240570440765:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:26.797504Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789240570440744:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27803) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:26.797549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789240570440765:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:26.797612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789240570440744:2107] [node 5] ICP30 unregister session Session# [6:7577789240570440765:2107] VirtualId# [6:7577789240570440761:0]
2025-11-28T14:52:26.797649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:26.797675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789240570440744:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.797685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:26.808009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789240570440744:2107] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:26.878134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789239895409316:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:26.878130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789239895409315:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:26.895587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789240001333632:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.895623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789240001333633:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:26.913710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789242259749743:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:26.913777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789242259749742:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.931288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789240985124179:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.931326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789240985124180:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:26.931358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789240985124181:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:26.953723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789239739868974:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.953755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789239739868975:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:26.953848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789239739868976:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:26.974740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789243106475743:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:26.974781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789243106475745:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:26.974855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789243106475744:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:27.065393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789247401443101:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:27.068153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789246554717102:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:27.068152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789244190376684:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:27.068537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789245280091538:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:27.069256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789244296301018:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:27.068778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789244034836333:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:27.068174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789246554717103:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:27.068578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789244190376685:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:27.068328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789245280091539:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:27.069418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789244296301020:2121] [node 6] ICP01 ready to work
2025-11-28T14:52:27.068805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789244190376686:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:27.070324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789247401443103:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:27.070043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789244296301019:2135] [node 5] ICP01 ready to work
2025-11-28T14:52:27.070222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789246554717104:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:27.070230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789244034836334:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:27.074899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789243106475743:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47797685ns) {Configure:104(47795146ns) {ProcessConfigured:135(896138ns) {StartInitialHandshake:157(894129ns) {DropHandshakes:470(79723ns) {DropIncomingHandshake:442(885ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(715ns) PrepareNewSessionHandshake:477(1579ns) {SwitchToState:179(757ns)}}}}}
2025-11-28T14:52:27.075768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789243106475744:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39777787ns) {Configure:104(19485195ns) {ProcessConfigured:135(830478ns) {StartInitialHandshake:157(828264ns) {DropHandshakes:470(994ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:477(1832ns) {SwitchToState:179(523ns)}}}}}
2025-11-28T14:52:27.078265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789240985124179:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42428015ns) {Configure:104(42419506ns) {ProcessConfigured:135(1062555ns) {StartInitialHandshake:157(1059902ns) {DropHandshakes:470(1509ns) {DropIncomingHandshake:442(596ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(594ns) PrepareNewSessionHandshake:477(1817ns) {SwitchToState:179(464ns)}}}}}
2025-11-28T14:52:27.078605Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789242259749743:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52210888ns) {Configure:104(52203938ns) {ProcessConfigured:135(792721ns) {StartInitialHandshake:157(790534ns) {DropHandshakes:470(661ns) {DropIncomingHandshake:442(178ns) DropOutgoingHandshake:457(135ns)} GenerateSessionVirtualId:988(2850ns) PrepareNewSessionHandshake:477(549ns) {SwitchToState:179(99ns)}}}}}
2025-11-28T14:52:27.078664Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789239895409315:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52163771ns) {Configure:104(42581697ns) {ProcessConfigured:135(236495ns) {StartInitialHandshake:157(234537ns) {DropHandshakes:470(1058ns) {DropIncomingHandshake:442(278ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(404ns) PrepareNewSessionHandshake:477(174805ns) {SwitchToState:179(885ns)}}}}}
2025-11-28T14:52:27.079045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789240985124180:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16439443ns) {Configure:104(11830395ns) {ProcessConfigured:135(1108672ns) {StartInitialHandshake:157(1106793ns) {DropHandshakes:470(1131ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(534ns) PrepareNewSessionHandshake:477(1210ns) {SwitchToState:179(554ns)}}}}}
2025-11-28T14:52:27.078867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789239895409316:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52854977ns) {Configure:104(52846380ns) {ProcessConfigured:135(484086ns) {StartInitialHandshake:157(483073ns) {DropHandshakes:470(628ns) {DropIncomingHandshake:442(181ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(358ns) PrepareNewSessionHandshake:477(3209ns) {SwitchToState:179(2510ns)}}}}}
2025-11-28T14:52:27.078887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789240001333632:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33384391ns) {Configure:104(22429011ns) {ProcessConfigured:135(1334974ns) {StartInitialHandshake:157(1333160ns) {DropHandshakes:470(169872ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(212ns)} GenerateSessionVirtualId:988(859ns) PrepareNewSessionHandshake:477(1767ns) {SwitchToState:179(599ns)}}}}}
2025-11-28T14:52:27.079419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789242259749742:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52384768ns) {Configure:104(52377966ns) {ProcessConfigured:135(381240ns) {StartInitialHandshake:157(380012ns) {DropHandshakes:470(997ns) {DropIncomingHandshake:442(341ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(568ns) {SwitchToState:179(121ns)}}}}}
2025-11-28T14:52:27.079373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789239739868974:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23457526ns) {Configure:104(23451429ns) {ProcessConfigured:135(1094810ns) {StartInitialHandshake:157(1092921ns) {DropHandshakes:470(1394ns) {DropIncomingHandshake:442(321ns) DropOutgoingHandshake:457(124ns)} GenerateSessionVirtualId:988(353ns) PrepareNewSessionHandshake:477(1566ns) {SwitchToState:179(700ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.082022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789244190376697:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46667
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.082021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244296301031:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9977
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.083855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789246554717115:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38707
2025-11-28T14:52:27.083376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244296301036:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9977
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.085112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789246554717119:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38707
2025-11-28T14:52:27.084238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789244190376701:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46667
2025-11-28T14:52:27.084045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244296301035:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9977
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.086508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244296301038:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9977
2025-11-28T14:52:27.085146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789244190376702:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46667
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.086665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789245280091553:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27479
2025-11-28T14:52:27.087213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443118:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-11-28T14:52:27.087916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789246554717120:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38707
2025-11-28T14:52:27.087923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443117:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.089693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789244190376697:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.087960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244296301031:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.088956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443119:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-11-28T14:52:27.090851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789245280091554:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27479
2025-11-28T14:52:27.088273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789246554717121:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38707
2025-11-28T14:52:27.091016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789244190376703:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46667
2025-11-28T14:52:27.091732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789247401443119:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.089629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789246554717115:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.091758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789245280091553:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.091674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789244190376701:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.088953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244296301037:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9977
2025-11-28T14:52:27.091794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789247401443117:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.090077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789246554717119:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.091878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789244190376694:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.092113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789245280091554:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.091601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244296301035:2143] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.092682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789247401443103:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.092397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789240985124179:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.090893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789242259749742:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.092807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789243106475745:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.091907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789244190376694:2122] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.091811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244296301036:2144] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.092053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789240001333632:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.092714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789240985124181:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.092657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789244190376690:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.092401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244296301037:2145] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.095062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789247401443118:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.094090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789245280091545:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.092680Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789244190376690:2120] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.092994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244296301038:2146] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.092694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789246554717120:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.094111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789245280091545:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.092813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789244190376703:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.095663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789243106475744:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.094428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244296301024:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.092963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789246554717109:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.096848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789245280091545:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:27.093815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789244190376702:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.094446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789244296301024:2137] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.092987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789246554717109:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.094086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789244190376692:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.096803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244296301024:2137] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:27.093004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789242259749743:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.099396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789243106475745:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789247401443119:2117] held: no
2025-11-28T14:52:27.094100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789244190376692:2121] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.095663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789244190376690:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.101477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789245280091538:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.097301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244296301027:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.093902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789246554717121:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.097316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244296301044:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9977
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.101509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.096413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789244190376694:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:27.095156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789246554717110:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.097330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789244296301027:2139] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.103416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443123:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.096810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789239895409316:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.100727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244296301027:2139] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:27.102607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789245280091538:2109] [node 6] ICP22 created new session: [4:7577789245280091560:2109]
2025-11-28T14:52:27.106668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443124:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-11-28T14:52:27.101180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789240001333632:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19382) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789244296301035:2143] held: no
2025-11-28T14:52:27.103387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789245280091560:2109] [node 6] ICS09 handshake done sender: [4:7577789245280091545:2110] self: [4:7577789245280091540:0] peer: [6:7577789247401443120:0] socket: 517 qp: -1
2025-11-28T14:52:27.095170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789246554717110:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.106678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443125:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-11-28T14:52:27.101427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789240001333633:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.103410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789245280091560:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:27.095659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789246554717109:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.096822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789239895409315:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.105247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:27.101478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.107182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789247401443119:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.102201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789240001333633:2049] [node 3] ICP22 created new session: [2:7577789244296301047:2049]
2025-11-28T14:52:27.097508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789246554717103:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.097011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.097722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789246554717110:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.105826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789245280091558:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27479
2025-11-28T14:52:27.102873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244296301047:2049] [node 3] ICS09 handshake done sender: [2:7577789244296301024:2137] self: [2:7577789244296301023:0] peer: [3:7577789246554717124:0] socket: 527 qp: -1
2025-11-28T14:52:27.109368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789244034836349:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27729
2025-11-28T14:52:27.102887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789244296301020:2121] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.106725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789245280091559:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27479
2025-11-28T14:52:27.110332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789243106475745:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.097027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789244190376692:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.097741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.110649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789244034836348:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27729
2025-11-28T14:52:27.097772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789246554717103:2109] [node 6] ICP22 created new session: [3:7577789246554717129:2109]
2025-11-28T14:52:27.111398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789244034836348:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:27.103186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.098820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789246554717129:2109] [node 6] ICS09 handshake done sender: [3:7577789246554717109:2111] self: [3:7577789246554717108:0] peer: [6:7577789247401443121:0] socket: 531 qp: -1
2025-11-28T14:52:27.107108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789240985124179:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19382) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789245280091553:2115] held: no
::ffff:127.1.0.0
2025-11-28T14:52:27.099365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789246554717129:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:27.111156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789243106475744:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789247401443118:2116] held: no
2025-11-28T14:52:27.097043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789239895409316:2049] [node 3] ICP22 created new session: [1:7577789244190376711:2049]
2025-11-28T14:52:27.110140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789245280091554:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:27.111695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789244034836350:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27729
2025-11-28T14:52:27.103201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244296301035:2143] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.111520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789240985124181:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.111172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.103203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244296301047:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:27.099404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:27.101172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789246554717104:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.097078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789244190376711:2049] [node 3] ICS09 handshake done sender: [1:7577789244190376694:2122] self: [1:7577789244190376693:0] peer: [3:7577789246554717123:0] socket: 523 qp: -1
2025-11-28T14:52:27.111545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.114133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789239739868974:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.111446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789247401443117:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.103449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789244296301020:2121] [node 6] ICP22 created new session: [2:7577789244296301048:2121]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.097101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789244190376711:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:27.114133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789244034836352:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27729
2025-11-28T14:52:27.111799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789240985124181:2050] [node 3] ICP22 created new session: [4:7577789245280091565:2050]
2025-11-28T14:52:27.101205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.111760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789243106475745:2050] [node 3] ICP22 created new session: [6:7577789247401443128:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.104456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.105937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789240001333632:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.097134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.102206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789246554717104:2107] [node 4] ICP22 created new session: [3:7577789246554717132:2107]
2025-11-28T14:52:27.114868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789244034836349:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.106649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244296301048:2121] [node 6] ICS09 handshake done sender: [2:7577789244296301027:2139] self: [2:7577789244296301025:0] peer: [6:7577789247401443122:0] socket: 528 qp: -1
2025-11-28T14:52:27.111932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789245280091553:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:27.116160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789244034836334:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.097364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:27.111890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789247401443118:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:27.102531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789246554717132:2107] [node 4] ICS09 handshake done sender: [3:7577789246554717110:2112] self: [3:7577789246554717105:0] peer: [4:7577789245280091556:0] socket: 526 qp: -1
2025-11-28T14:52:27.116380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789244034836350:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.107141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244296301048:2121] [node 6] ICS10 traffic start
2025-11-28T14:52:27.112390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789245280091565:2050] [node 3] ICS09 handshake done sender: [4:7577789245280091554:2116] self: [4:7577789245280091556:0] peer: [3:7577789246554717105:0] socket: 550 qp: -1
2025-11-28T14:52:27.098763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789239895409315:2048] [node 2] ICP22 created new session: [1:7577789244190376713:2048]
2025-11-28T14:52:27.112344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789247401443128:2050] [node 3] ICS09 handshake done sender: [6:7577789247401443119:2117] self: [6:7577789247401443121:0] peer: [3:7577789246554717108:0] socket: 545 qp: -1
2025-11-28T14:52:27.116543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789244034836351:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27729
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.102815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789246554717132:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:27.107191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:27.099024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789244190376713:2048] [node 2] ICS09 handshake done sender: [1:7577789244190376690:2120] self: [1:7577789244190376689:0] peer: [2:7577789244296301040:0] socket: 524 qp: -1
2025-11-28T14:52:27.112743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789245280091565:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:27.102819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789246554717128:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38707
2025-11-28T14:52:27.112732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789247401443128:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:27.116930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789244034836352:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.108487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:27.103675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.112772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.099354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789244190376713:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:27.108690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789240001333632:2048] [node 1] ICP22 created new session: [2:7577789244296301053:2048]
2025-11-28T14:52:27.117781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789239739868976:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.112758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.099376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:27.106673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789242259749743:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789246554717120:2118] held: no
2025-11-28T14:52:27.109387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244296301053:2048] [node 1] ICS09 handshake done sender: [2:7577789244296301035:2143] self: [2:7577789244296301040:0] peer: [1:7577789244190376689:0] socket: 530 qp: -1
2025-11-28T14:52:27.118045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789244034836351:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.107777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789246554717130:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38707
2025-11-28T14:52:27.114094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789245280091563:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27479
2025-11-28T14:52:27.099506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789244190376685:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.110714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244296301053:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.113447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443126:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-11-28T14:52:27.118771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789239739868975:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.100736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.111205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:27.114465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789240985124179:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.108547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789246554717119:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:27.114001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789243106475744:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.114499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:27.122363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789239739868974:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19382) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789244034836348:2115] held: no
2025-11-28T14:52:27.101153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789244190376685:2107] [node 4] ICP22 created new session: [1:7577789244190376716:2107]
2025-11-28T14:52:27.117452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244296301030:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.114043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:27.110940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789242259749742:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.101779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789244190376716:2107] [node 4] ICS09 handshake done sender: [1:7577789244190376692:2121] self: [1:7577789244190376687:0] peer: [4:7577789245280091555:0] socket: 522 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.114873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789240985124179:2048] [node 1] ICP22 created new session: [4:7577789245280091567:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.117491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789244296301030:2141] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.114402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789247401443103:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.126687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789244034836358:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27729
2025-11-28T14:52:27.110985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:27.115391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789245280091563:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.102456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789244190376716:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:27.111185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789246554717120:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:27.119950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244296301030:2141] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.111211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789242259749742:2048] [node 1] ICP22 created new session: [3:7577789246554717136:2048]
2025-11-28T14:52:27.102480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.121378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244296301026:2138] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.111792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789246554717136:2048] [node 1] ICS09 handshake done sender: [3:7577789246554717119:2117] self: [3:7577789246554717123:0] peer: [1:7577789244190376693:0] socket: 533 qp: -1
2025-11-28T14:52:27.115018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789243106475744:2049] [node 2] ICP22 created new session: [6:7577789247401443131:2049]
2025-11-28T14:52:27.114807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789244190376719:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46667
2025-11-28T14:52:27.115496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789245280091567:2048] [node 1] ICS09 handshake done sender: [4:7577789245280091553:2115] self: [4:7577789245280091555:0] peer: [1:7577789244190376687:0] socket: 546 qp: -1
2025-11-28T14:52:27.111975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789246554717136:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.121401Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789244296301026:2138] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.115459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.128003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789244034836359:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27729
2025-11-28T14:52:27.115419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789244190376695:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.112470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:27.128465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789244034836361:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27729
2025-11-28T14:52:27.116110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789245280091567:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.115751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789247401443131:2049] [node 2] ICS09 handshake done sender: [6:7577789247401443118:2116] self: [6:7577789247401443122:0] peer: [2:7577789244296301025:0] socket: 537 qp: -1
2025-11-28T14:52:27.114382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789242259749743:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.124110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244296301026:2138] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:27.115436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789244190376695:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.114994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:27.124447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789244296301018:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.115458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789242259749743:2049] [node 2] ICP22 created new session: [3:7577789246554717139:2049]
2025-11-28T14:52:27.117774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789244190376695:2123] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:27.130275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789239739868975:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789244034836351:2118] held: no
2025-11-28T14:52:27.116071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789246554717139:2049] [node 2] ICS09 handshake done sender: [3:7577789246554717120:2118] self: [3:7577789246554717124:0] peer: [2:7577789244296301023:0] socket: 538 qp: -1
2025-11-28T14:52:27.116247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:27.124763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.115761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789247401443103:2108] [node 4] ICP22 created new session: [6:7577789247401443132:2108]
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.119666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789244190376696:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.131686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789244034836351:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:27.116095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789246554717139:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:27.115779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789247401443131:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:27.116851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789240985124180:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.124800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789244296301018:2107] [node 4] ICP22 created new session: [2:7577789244296301057:2107]
2025-11-28T14:52:27.131709Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789239739868976:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789244034836350:2117] held: no
2025-11-28T14:52:27.119681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789244190376696:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.116555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:27.119746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789244190376686:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.116077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789247401443126:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.117531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789245280091549:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.117535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789246554717138:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38707
2025-11-28T14:52:27.125293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244296301057:2107] [node 4] ICS09 handshake done sender: [2:7577789244296301030:2141] self: [2:7577789244296301029:0] peer: [4:7577789245280091569:0] socket: 519 qp: -1
2025-11-28T14:52:27.119992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.117551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789245280091549:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.117519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789247401443132:2108] [node 4] ICS09 handshake done sender: [6:7577789247401443117:2115] self: [6:7577789247401443120:0] peer: [4:7577789245280091540:0] socket: 520 qp: -1
2025-11-28T14:52:27.119674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789246554717113:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.125366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789244296301019:2135] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.132192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789244034836349:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.120023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789244190376686:2109] [node 5] ICP22 created new session: [1:7577789244190376721:2109]
2025-11-28T14:52:27.117527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:27.125532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244296301057:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:27.118252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789245280091568:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27479
2025-11-28T14:52:27.121417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789244190376721:2109] [node 5] ICS09 handshake done sender: [1:7577789244190376695:2123] self: [1:7577789244190376688:0] peer: [5:7577789244034836353:0] socket: 518 qp: -1
2025-11-28T14:52:27.125808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.119694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789246554717113:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.126700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.117564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789247401443132:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:27.121251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789245280091549:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:27.132524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789244034836360:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27729
2025-11-28T14:52:27.122325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789244190376721:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:27.123747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789246554717113:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:27.127531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789244296301019:2135] [node 5] ICP22 created new session: [2:7577789244296301059:2135]
2025-11-28T14:52:27.118215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789243106475743:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.133090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789244034836348:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:27.122369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:27.128467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244296301059:2135] [node 5] ICS09 handshake done sender: [2:7577789244296301026:2138] self: [2:7577789244296301021:0] peer: [5:7577789244034836357:0] socket: 570 qp: -1
2025-11-28T14:52:27.124736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789246554717102:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.130266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244296301059:2135] [node 5] ICS10 traffic start
2025-11-28T14:52:27.122836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789245280091539:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.118384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.124118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789244190376696:2124] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:27.130323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:27.133314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789239739868975:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.125027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.122874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.128428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789243106475743:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19382) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789247401443126:2121] held: no
2025-11-28T14:52:27.125516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789244190376684:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.125051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789246554717102:2108] [node 5] ICP22 created new session: [3:7577789246554717142:2108]
2025-11-28T14:52:27.133358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:27.123479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789245280091539:2108] [node 5] ICP22 created new session: [4:7577789245280091572:2108]
2025-11-28T14:52:27.126111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.126730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789246554717142:2108] [node 5] ICS09 handshake done sender: [3:7577789246554717113:2114] self: [3:7577789246554717107:0] peer: [5:7577789244034836355:0] socket: 568 qp: -1
2025-11-28T14:52:27.124140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789245280091572:2108] [node 5] ICS09 handshake done sender: [4:7577789245280091549:2114] self: [4:7577789245280091543:0] peer: [5:7577789244034836354:0] socket: 567 qp: -1
2025-11-28T14:52:27.133404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789239739868975:2049] [node 2] ICP22 created new session: [5:7577789244034836363:2049]
2025-11-28T14:52:27.127083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789246554717142:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:27.126773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789244190376684:2110] [node 6] ICP22 created new session: [1:7577789244190376724:2110]
2025-11-28T14:52:27.130965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443136:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-11-28T14:52:27.127121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:27.133809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789244034836363:2049] [node 2] ICS09 handshake done sender: [5:7577789244034836351:2118] self: [5:7577789244034836357:0] peer: [2:7577789244296301021:0] socket: 525 qp: -1
2025-11-28T14:52:27.124435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789245280091572:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:27.132182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789247401443126:2121] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.124746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:27.128020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789244190376724:2110] [node 6] ICS09 handshake done sender: [1:7577789244190376696:2124] self: [1:7577789244190376691:0] peer: [6:7577789247401443134:0] socket: 536 qp: -1
2025-11-28T14:52:27.134077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789244034836334:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.126659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789240985124180:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789245280091563:2119] held: no
2025-11-28T14:52:27.130215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789244190376724:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:27.132511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789245280091574:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27479
2025-11-28T14:52:27.134126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.133593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789245280091563:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:27.130259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:27.134323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789240985124180:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.133415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789243106475743:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.135130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:27.134181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789244034836334:2108] [node 4] ICP22 created new session: [5:7577789244034836364:2108]
2025-11-28T14:52:27.135309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789240985124180:2049] [node 2] ICP22 created new session: [4:7577789245280091575:2049]
2025-11-28T14:52:27.134236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789239739868974:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.134462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:27.135348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789245280091575:2049] [node 2] ICS09 handshake done sender: [4:7577789245280091563:2119] self: [4:7577789245280091569:0] peer: [2:7577789244296301029:0] socket: 571 qp: -1
2025-11-28T14:52:27.134493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789243106475743:2048] [node 1] ICP22 created new session: [6:7577789247401443138:2048]
2025-11-28T14:52:27.134249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789244034836363:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:27.135378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789245280091575:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:27.135074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789247401443138:2048] [node 1] ICS09 handshake done sender: [6:7577789247401443126:2121] self: [6:7577789247401443134:0] peer: [1:7577789244190376691:0] socket: 510 qp: -1
2025-11-28T14:52:27.135440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:27.134281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:27.135333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789247401443138:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.134458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789244034836364:2108] [node 4] ICS09 handshake done sender: [5:7577789244034836349:2116] self: [5:7577789244034836354:0] peer: [4:7577789245280091543:0] socket: 521 qp: -1
2025-11-28T14:52:27.135445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:27.135668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443137:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-11-28T14:52:27.135052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789244034836364:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:27.135274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:27.135315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789239739868974:2048] [node 1] ICP22 created new session: [5:7577789244034836367:2048]
2025-11-28T14:52:27.135325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789244034836350:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:27.135343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789244034836367:2048] [node 1] ICS09 handshake done sender: [5:7577789244034836348:2115] self: [5:7577789244034836353:0] peer: [1:7577789244190376688:0] socket: 566 qp: -1
2025-11-28T14:52:27.135368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789244034836367:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.135442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:27.135500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.135962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789239739868976:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.136085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.136129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789239739868976:2050] [node 3] ICP22 created new session: [5:7577789244034836370:2050]
2025-11-28T14:52:27.136170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789244034836370:2050] [node 3] ICS09 handshake done sender: [5:7577789244034836350:2117] self: [5:7577789244034836355:0] peer: [3:7577789246554717107:0] socket: 569 qp: -1
2025-11-28T14:52:27.136201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789244034836370:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:27.136253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.137553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789247401443137:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.137713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789247401443101:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.137886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789244034836344:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.137916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789244034836344:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.138600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789244034836344:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:27.138764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789244034836333:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.138824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.138903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789244034836333:2109] [node 6] ICP22 created new session: [5:7577789244034836373:2109]
2025-11-28T14:52:27.138954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789244034836373:2109] [node 6] ICS09 handshake done sender: [5:7577789244034836344:2114] self: [5:7577789244034836340:0] peer: [6:7577789247401443140:0] socket: 512 qp: -1
2025-11-28T14:52:27.138999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789244034836373:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:27.139064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:27.138904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789247401443141:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-11-28T14:52:27.139157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789247401443137:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:27.139198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789247401443101:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20844) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789247401443137:2123] held: no
2025-11-28T14:52:27.139385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789247401443101:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.139416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.140258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789247401443101:2106] [node 5] ICP22 created new session: [6:7577789247401443142:2106]
2025-11-28T14:52:27.140316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789247401443142:2106] [node 5] ICS09 handshake done sender: [6:7577789247401443137:2123] self: [6:7577789247401443140:0] peer: [5:7577789244034836340:0] socket: 513 qp: -1
2025-11-28T14:52:27.140340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789247401443142:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:27.140416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:27.193379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:27.193592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789244296301053:2048] [node 1] ICS01 socket: 530 reason# EndOfStream
2025-11-28T14:52:27.193638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789240001333632:2048] [node 1] ICP30 unregister session Session# [2:7577789244296301053:2048] VirtualId# [2:7577789244296301040:0]
2025-11-28T14:52:27.193671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789244296301053:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.193760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.193815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.194720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:27.194925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789246554717136:2048] [node 1] ICS07 socket disconnect 533 reason# EndOfStream
2025-11-28T14:52:27.194959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789246554717136:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.195035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.195084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789246554717136:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:27.197271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:27.197373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789242259749742:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19382) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:27.197410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789246554717136:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:27.197526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789245280091567:2048] [node 1] ICS01 socket: 546 reason# EndOfStream
2025-11-28T14:52:27.197627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789240985124179:2048] [node 1] ICP30 unregister session Session# [4:7577789245280091567:2048] VirtualId# [4:7577789245280091555:0]
2025-11-28T14:52:27.197660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789245280091567:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.197743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.197798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.198292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789242259749742:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19382) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:27.198345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789246554717136:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:27.198370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789242259749742:2048] [node 1] ICP30 unregister session Session# [3:7577789246554717136:2048] VirtualId# [3:7577789246554717123:0]
2025-11-28T14:52:27.198414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.198434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789242259749742:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19382) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.198443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19382) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.198614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:27.198810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789244034836367:2048] [node 1] ICS01 socket: 566 reason# EndOfStream
2025-11-28T14:52:27.198845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789239739868974:2048] [node 1] ICP30 unregister session Session# [5:7577789244034836367:2048] VirtualId# [5:7577789244034836353:0]
2025-11-28T14:52:27.198875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789244034836367:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.198927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.198960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.199650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:27.199859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789247401443138:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-11-28T14:52:27.199906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789243106475743:2048] [node 1] ICP30 unregister session Session# [6:7577789247401443138:2048] VirtualId# [6:7577789247401443134:0]
2025-11-28T14:52:27.199940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789247401443138:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.200004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.200055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.208826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789242259749742:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:27.261042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:27.261218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789246554717139:2049] [node 2] ICS01 socket: 538 reason# EndOfStream
2025-11-28T14:52:27.261249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789242259749743:2049] [node 2] ICP30 unregister session Session# [3:7577789246554717139:2049] VirtualId# [3:7577789246554717124:0]
2025-11-28T14:52:27.261271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789246554717139:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.261341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:27.261382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:27.263454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:27.263684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789245280091575:2049] [node 2] ICS01 socket: 571 reason# EndOfStream
2025-11-28T14:52:27.263727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789240985124180:2049] [node 2] ICP30 unregister session Session# [4:7577789245280091575:2049] VirtualId# [4:7577789245280091569:0]
2025-11-28T14:52:27.263764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789245280091575:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.263831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:27.263870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:27.264523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:27.264722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789247401443131:2049] [node 2] ICS01 socket: 537 reason# EndOfStream
2025-11-28T14:52:27.264762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789243106475744:2049] [node 2] ICP30 unregister session Session# [6:7577789247401443131:2049] VirtualId# [6:7577789247401443122:0]
2025-11-28T14:52:27.264790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789247401443131:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.264862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:27.264910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:27.265552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:27.265725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789244034836363:2049] [node 2] ICS01 socket: 525 reason# EndOfStream
2025-11-28T14:52:27.265760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789239739868975:2049] [node 2] ICP30 unregister session Session# [5:7577789244034836363:2049] VirtualId# [5:7577789244034836357:0]
2025-11-28T14:52:27.265785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789244034836363:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.265842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:27.265879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:27.333743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:27.333968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789245280091565:2050] [node 3] ICS07 socket disconnect 550 reason# EndOfStream
2025-11-28T14:52:27.334007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789245280091565:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.334090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:27.334139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789245280091565:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:27.335115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789240985124181:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13207) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:27.335224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789245280091565:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:27.335134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:27.335267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789240985124181:2050] [node 3] ICP30 unregister session Session# [4:7577789245280091565:2050] VirtualId# [4:7577789245280091556:0]
2025-11-28T14:52:27.335300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789244034836370:2050] [node 3] ICS01 socket: 569 reason# EndOfStream
2025-11-28T14:52:27.335330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789239739868976:2050] [node 3] ICP30 unregister session Session# [5:7577789244034836370:2050] VirtualId# [5:7577789244034836355:0]
2025-11-28T14:52:27.335326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:27.335352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789244034836370:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.335378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789240985124181:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.335418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.335421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:27.335463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:27.336084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:27.336283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789247401443128:2050] [node 3] ICS01 socket: 545 reason# EndOfStream
2025-11-28T14:52:27.336330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789243106475745:2050] [node 3] ICP30 unregister session Session# [6:7577789247401443128:2050] VirtualId# [6:7577789247401443121:0]
2025-11-28T14:52:27.336384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789247401443128:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.336462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:27.336515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:27.377647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:27.377828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789244034836364:2108] [node 4] ICS07 socket disconnect 521 reason# EndOfStream
2025-11-28T14:52:27.377853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789244034836364:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.377925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:27.377970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789244034836364:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:27.378561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789244034836334:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22379) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:27.378493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:27.378602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789244034836364:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:27.378665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789244034836334:2108] [node 4] ICP30 unregister session Session# [5:7577789244034836364:2108] VirtualId# [5:7577789244034836354:0]
2025-11-28T14:52:27.378711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:27.378741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789244034836334:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22379) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.378738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789247401443132:2108] [node 4] ICS07 socket disconnect 520 reason# EndOfStream
2025-11-28T14:52:27.378750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22379) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.378783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789247401443132:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.378871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:27.378932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789247401443132:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:27.379576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789247401443103:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22379) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:27.379623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789247401443132:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:27.379717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789247401443103:2108] [node 4] ICP30 unregister session Session# [6:7577789247401443132:2108] VirtualId# [6:7577789247401443120:0]
2025-11-28T14:52:27.379775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:27.379809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789247401443103:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22379) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.379844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22379) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.388600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789244034836334:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:27.389612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789247401443103:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:27.442756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:27.443014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789247401443142:2106] [node 5] ICS01 socket: 513 reason# EndOfStream
2025-11-28T14:52:27.443070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789247401443101:2106] [node 5] ICP30 unregister session Session# [6:7577789247401443142:2106] VirtualId# [6:7577789247401443140:0]
2025-11-28T14:52:27.443126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789247401443142:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.443206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:27.443258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:27.559351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789246871053607:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:27.559393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789246871053608:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:27.575781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789244862588087:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:27.575813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789244862588088:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:27.599107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789245356866410:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:27.599149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789245356866411:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:27.620603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789246152106567:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:27.620654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789246152106568:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:27.620699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789246152106569:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:27.641054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789245606999976:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:27.641092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789245606999977:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:27.641114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789245606999978:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:27.661019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789245080031150:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:27.661057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789245080031151:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:27.661082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789245080031152:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:27.730821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789245080031212:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:27.744029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789246152106630:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:27.744103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789244862588168:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:27.744526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789245356866474:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:27.744093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789246871053697:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:27.744651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789244862588169:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:27.744560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789245607000039:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:27.744837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789246152106631:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:27.744759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789245080031215:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:27.744948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789245607000040:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:27.745178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789245356866475:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:27.745211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789244862588170:2126] [node 6] ICP01 ready to work
2025-11-28T14:52:27.745235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789246871053698:2114] [node 5] ICP01 ready to work
2025-11-28T14:52:27.745412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789245356866476:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:27.745643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789246871053699:2123] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.753748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789245356866490:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15896
2025-11-28T14:52:27.754313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789245356866491:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15896
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.755000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789245607000054:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57133
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.755562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244862588184:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12158
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.756931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789245356866492:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15896
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.756895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244862588185:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12158
2025-11-28T14:52:27.757014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244862588186:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12158
2025-11-28T14:52:27.757071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244862588187:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12158
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.760606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789246152106645:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27924
2025-11-28T14:52:27.760654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789246152106646:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27924
2025-11-28T14:52:27.760718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789246871053710:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10796
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.760916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789246871053711:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10796
2025-11-28T14:52:27.761012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789246152106647:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27924
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.761159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789246871053712:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10796
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.761380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789246871053713:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10796
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.761499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789246871053714:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10796
2025-11-28T14:52:27.761705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053711:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.761892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053713:2145] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.762471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789245607000054:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.763798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053710:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.765234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789246152106645:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.765251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789245356866490:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.765324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789246152106569:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.765315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789245356866491:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.765400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789245356866493:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15896
2025-11-28T14:52:27.765507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789245356866482:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.765523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789245356866482:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.766375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789245356866482:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.766430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789246152106649:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27924
2025-11-28T14:52:27.766779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789245356866474:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.766883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.766916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789245356866474:2107] [node 4] ICP22 created new session: [3:7577789245356866497:2107]
2025-11-28T14:52:27.766920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789246152106645:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:27.766974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789245356866497:2107] [node 4] ICS09 handshake done sender: [3:7577789245356866482:2113] self: [3:7577789245356866481:0] peer: [4:7577789246152106648:0] socket: 533 qp: -1
2025-11-28T14:52:27.767006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789245356866497:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:27.767038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.767154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789246152106569:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.767245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.767659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789246152106569:2050] [node 3] ICP22 created new session: [4:7577789246152106650:2050]
2025-11-28T14:52:27.767693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789246152106650:2050] [node 3] ICS09 handshake done sender: [4:7577789246152106645:2115] self: [4:7577789246152106648:0] peer: [3:7577789245356866481:0] socket: 542 qp: -1
2025-11-28T14:52:27.767721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789246152106650:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:27.767772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.767779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053712:2144] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.767953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244862588187:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.768386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053714:2146] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.768454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244862588185:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.768961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789245356866493:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.769098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244862588188:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12158
2025-11-28T14:52:27.769210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789245356866492:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.769240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244862588186:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.769405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244862588184:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.769399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789245356866410:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.769503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244862588188:2137] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.769520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789246152106647:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.769583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789246152106646:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.769595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789245356866411:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.769657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789244862588087:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.769653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789245607000056:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57133
2025-11-28T14:52:27.769681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053703:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.769716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789246871053703:2138] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.769795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244862588174:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.769814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789244862588174:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.769860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789246152106567:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.770125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789246152106568:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.770549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053701:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.770582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789246871053701:2137] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.770617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789245607000057:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57133
2025-11-28T14:52:27.770650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244862588176:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.770671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789244862588176:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.771894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053705:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.771921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789246871053705:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.771934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244862588174:2129] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.772211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789245607000056:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.772699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789245607000057:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.772708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789245356866502:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15896
2025-11-28T14:52:27.772809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789245606999978:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.772842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789244862588088:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.772868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789245607000039:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.772908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.772936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789244862588088:2049] [node 3] ICP22 created new session: [2:7577789244862588196:2049]
2025-11-28T14:52:27.773005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244862588196:2049] [node 3] ICS09 handshake done sender: [2:7577789244862588174:2129] self: [2:7577789244862588173:0] peer: [3:7577789245356866501:0] socket: 539 qp: -1
2025-11-28T14:52:27.773053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244862588196:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:27.773139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789246871053703:2138] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:27.773077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.773406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789245356866484:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.773421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789246152106639:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.773425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789245356866484:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.773446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789246152106639:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.773625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789246871053701:2137] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:27.773882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789245356866492:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:27.774411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789246871053608:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.774484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.774477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789245356866411:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.774529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789246871053608:2049] [node 3] ICP22 created new session: [1:7577789246871053726:2049]
2025-11-28T14:52:27.774514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:27.774511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789246152106655:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27924
2025-11-28T14:52:27.774551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789245356866411:2049] [node 2] ICP22 created new session: [3:7577789245356866505:2049]
2025-11-28T14:52:27.774533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244862588176:2130] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.774573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789246871053726:2049] [node 3] ICS09 handshake done sender: [1:7577789246871053703:2138] self: [1:7577789246871053702:0] peer: [3:7577789245356866500:0] socket: 552 qp: -1
2025-11-28T14:52:27.774584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789245356866505:2049] [node 2] ICS09 handshake done sender: [3:7577789245356866492:2118] self: [3:7577789245356866501:0] peer: [2:7577789244862588173:0] socket: 541 qp: -1
2025-11-28T14:52:27.774608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789246871053726:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:27.774637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.774611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789245356866505:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:27.774633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.775134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789244862588168:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.775168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789246871053607:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.775175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789246152106656:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27924
2025-11-28T14:52:27.775166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.775215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789244862588168:2107] [node 4] ICP22 created new session: [2:7577789244862588200:2107]
2025-11-28T14:52:27.775225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:27.775239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244862588200:2107] [node 4] ICS09 handshake done sender: [2:7577789244862588176:2130] self: [2:7577789244862588175:0] peer: [4:7577789246152106654:0] socket: 540 qp: -1
2025-11-28T14:52:27.775246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789246152106646:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:27.775261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244862588200:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:27.775268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789246871053607:2048] [node 2] ICP22 created new session: [1:7577789246871053728:2048]
2025-11-28T14:52:27.775304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.775316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789246871053728:2048] [node 2] ICS09 handshake done sender: [1:7577789246871053701:2137] self: [1:7577789246871053700:0] peer: [2:7577789244862588193:0] socket: 551 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:27.775385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789246871053728:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:27.775419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:27.775477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789245607000060:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57133
2025-11-28T14:52:27.775529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789246152106639:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:27.776088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789246152106568:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.776213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:27.776260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789246152106568:2049] [node 2] ICP22 created new session: [4:7577789246152106658:2049]
2025-11-28T14:52:27.776308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789246152106658:2049] [node 2] ICS09 handshake done sender: [4:7577789246152106646:2116] self: [4:7577789246152106654:0] peer: [2:7577789244862588175:0] socket: 543 qp: -1
2025-11-28T14:52:27.776338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789246152106658:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:27.776337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789245356866503:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15896
2025-11-28T14:52:27.776389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789245607000057:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.776412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789246871053705:2139] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.776416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:27.776953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789245356866411:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: 2025-11-28T14:52:27.777167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789245607000039:2108] [node 4] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:27.777183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789245356866484:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:27.777226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.777258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789244862588198:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12158
2025-11-28T14:52:27.777286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789245607000039:2108] [node 4] ICP22 created new session: [5:7577789245607000063:2108]
2025-11-28T14:52:27.777417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789245607000063:2108] [node 4] ICS09 handshake done sender: [5:7577789245607000057:2117] self: [5:7577789245607000059:0] peer: [4:7577789246152106638:0] socket: 561 qp: -1
2025-11-28T14:52:27.777444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789245607000063:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:27.777597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789245356866493:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:27.777479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.777668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789246152106630:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.777783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.777844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789246152106630:2108] [node 5] ICP22 created new session: [4:7577789246152106660:2108]
2025-11-28T14:52:27.777968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789246152106660:2108] [node 5] ICS09 handshake done sender: [4:7577789246152106639:2113] self: [4:7577789246152106638:0] peer: [5:7577789245607000059:0] socket: 560 qp: -1
2025-11-28T14:52:27.778005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789246152106660:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:27.778041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.778376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789245607000061:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57133
2025-11-28T14:52:27.778404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789246871053697:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.778484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.778526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789246871053697:2107] [node 4] ICP22 created new session: [1:7577789246871053730:2107]
2025-11-28T14:52:27.778567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789246871053730:2107] [node 4] ICS09 handshake done sender: [1:7577789246871053705:2139] self: [1:7577789246871053704:0] peer: [4:7577789246152106653:0] socket: 553 qp: -1
2025-11-28T14:52:27.779143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789246871053730:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:27.779199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.779396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789245356866475:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.779916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.779952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789245356866475:2108] [node 5] ICP22 created new session: [3:7577789245356866507:2108]
2025-11-28T14:52:27.780414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789245356866507:2108] [node 5] ICS09 handshake done sender: [3:7577789245356866484:2114] self: [3:7577789245356866483:0] peer: [5:7577789245607000058:0] socket: 520 qp: -1
2025-11-28T14:52:27.780512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789245356866507:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.780560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:27.780675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789245607000062:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57133
2025-11-28T14:52:27.780778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789245607000056:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:27.781090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789245606999978:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.781184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244862588188:2137] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:27.781144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.781174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789245606999978:2050] [node 3] ICP22 created new session: [5:7577789245607000066:2050]
2025-11-28T14:52:27.781232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789245356866410:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.781225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789245607000066:2050] [node 3] ICS09 handshake done sender: [5:7577789245607000056:2116] self: [5:7577789245607000058:0] peer: [3:7577789245356866483:0] socket: 559 qp: -1
2025-11-28T14:52:27.781288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.781251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789245607000066:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:27.781384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789245356866410:2048] [node 1] ICP22 created new session: [3:7577789245356866510:2048]
2025-11-28T14:52:27.781299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.781493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789245356866510:2048] [node 1] ICS09 handshake done sender: [3:7577789245356866493:2119] self: [3:7577789245356866500:0] peer: [1:7577789246871053702:0] socket: 556 qp: -1
2025-11-28T14:52:27.781625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031229:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-28T14:52:27.781697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789244862588087:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789244862588188:2137] held: no
2025-11-28T14:52:27.781528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789245356866510:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.781696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:27.781809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789244862588087:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.781856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:27.781898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789244862588087:2048] [node 1] ICP22 created new session: [2:7577789244862588203:2048]
2025-11-28T14:52:27.781871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789245356866410:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:27.781945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244862588203:2048] [node 1] ICS09 handshake done sender: [2:7577789244862588188:2137] self: [2:7577789244862588193:0] peer: [1:7577789246871053700:0] socket: 529 qp: -1
2025-11-28T14:52:27.782022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244862588203:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.782684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.783063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789246152106647:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.783352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031230:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-28T14:52:27.783462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789246152106567:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789246152106647:2117] held: no
2025-11-28T14:52:27.783587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031231:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-28T14:52:27.783637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789246152106567:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.783666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:27.783696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789246152106567:2048] [node 1] ICP22 created new session: [4:7577789246152106663:2048]
2025-11-28T14:52:27.783725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789246152106663:2048] [node 1] ICS09 handshake done sender: [4:7577789246152106647:2117] self: [4:7577789246152106653:0] peer: [1:7577789246871053704:0] socket: 554 qp: -1
2025-11-28T14:52:27.783743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789246152106663:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.783788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.784005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031232:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.784342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031233:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.785798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789245607000068:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57133
2025-11-28T14:52:27.785957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789245080031233:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.786049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789245080031232:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.786145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789245080031231:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.786276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789245080031215:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.786332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789245080031152:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.786644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789246152106641:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.786675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789246152106641:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.786653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789245080031212:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.786848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789245356866486:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.787385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789245607000050:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.786874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789245356866486:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.787646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789246152106641:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:27.787412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789245607000050:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.788240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789245607000062:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.788264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789245356866486:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:27.788358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789246152106631:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.788426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789245607000050:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:27.788417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.788451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789246152106631:2109] [node 6] ICP22 created new session: [4:7577789246152106666:2109]
2025-11-28T14:52:27.788485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789246152106666:2109] [node 6] ICS09 handshake done sender: [4:7577789246152106641:2114] self: [4:7577789246152106640:0] peer: [6:7577789245080031234:0] socket: 576 qp: -1
2025-11-28T14:52:27.788508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789246152106666:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:27.788606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789245356866476:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.788537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:27.788664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.788727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789245080031229:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.788772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789245356866476:2109] [node 6] ICP22 created new session: [3:7577789245356866513:2109]
2025-11-28T14:52:27.788820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789245356866513:2109] [node 6] ICS09 handshake done sender: [3:7577789245356866486:2115] self: [3:7577789245356866485:0] peer: [6:7577789245080031235:0] socket: 516 qp: -1
2025-11-28T14:52:27.788823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789245607000068:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.788850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789245356866513:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:27.788871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:27.788860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789245607000040:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.788892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.788911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789245080031230:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.788926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789245607000040:2109] [node 6] ICP22 created new session: [5:7577789245607000070:2109]
2025-11-28T14:52:27.788976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789245607000070:2109] [node 6] ICS09 handshake done sender: [5:7577789245607000050:2114] self: [5:7577789245607000049:0] peer: [6:7577789245080031236:0] socket: 548 qp: -1
2025-11-28T14:52:27.789005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789245607000070:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:27.789040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.789263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031237:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-28T14:52:27.789325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789245606999977:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.789893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789245606999976:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.789979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244862588178:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.790011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789244862588178:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.790437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053707:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.790470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789246871053707:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.791014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244862588178:2131] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.791872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031238:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-28T14:52:27.791902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789245607000074:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57133
2025-11-28T14:52:27.791967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789244862588169:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.792023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.792058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789244862588169:2108] [node 5] ICP22 created new session: [2:7577789244862588206:2108]
2025-11-28T14:52:27.792065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789246871053707:2140] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:27.792103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244862588206:2108] [node 5] ICS09 handshake done sender: [2:7577789244862588178:2131] self: [2:7577789244862588177:0] peer: [5:7577789245607000072:0] socket: 574 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.792248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244862588206:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:27.792417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:27.792482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031239:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-28T14:52:27.792568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789246871053698:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.792619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.792655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789246871053698:2114] [node 5] ICP22 created new session: [1:7577789246871053735:2114]
2025-11-28T14:52:27.792686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789246871053735:2114] [node 5] ICS09 handshake done sender: [1:7577789246871053707:2140] self: [1:7577789246871053706:0] peer: [5:7577789245607000073:0] socket: 537 qp: -1
2025-11-28T14:52:27.792674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789245607000062:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:27.792713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789246871053735:2114] [node 5] ICS10 traffic start
2025-11-28T14:52:27.792734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:27.792984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789245080031151:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.793069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789245080031150:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:27.793128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789245080031233:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:27.793112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789245606999977:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.793164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:27.793192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789245606999977:2049] [node 2] ICP22 created new session: [5:7577789245607000076:2049]
2025-11-28T14:52:27.793220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789245607000076:2049] [node 2] ICS09 handshake done sender: [5:7577789245607000062:2120] self: [5:7577789245607000072:0] peer: [2:7577789244862588177:0] socket: 526 qp: -1
2025-11-28T14:52:27.793296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789245607000076:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:27.793377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:27.793548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789245080031212:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11743) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789245080031231:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.793864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789245607000075:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57133
2025-11-28T14:52:27.793863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789245080031215:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.793934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:27.793978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789245080031215:2107] [node 4] ICP22 created new session: [6:7577789245080031242:2107]
2025-11-28T14:52:27.793962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789244862588180:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.793979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789246871053709:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:27.793994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789244862588180:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.794006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789246871053709:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:27.794022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789245080031242:2107] [node 4] ICS09 handshake done sender: [6:7577789245080031233:2120] self: [6:7577789245080031234:0] peer: [4:7577789246152106640:0] socket: 577 qp: -1
2025-11-28T14:52:27.794139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789245080031242:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:27.794167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:27.795213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789244862588180:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:27.795481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789246871053709:2141] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:27.795765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789244862588170:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.795806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.795839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789244862588170:2126] [node 6] ICP22 created new session: [2:7577789244862588209:2126]
2025-11-28T14:52:27.795908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789244862588209:2126] [node 6] ICS09 handshake done sender: [2:7577789244862588180:2132] self: [2:7577789244862588179:0] peer: [6:7577789245080031240:0] socket: 518 qp: -1
2025-11-28T14:52:27.795928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789244862588209:2126] [node 6] ICS10 traffic start
2025-11-28T14:52:27.795955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:27.796011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789246871053699:2123] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:27.796065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:27.796101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789246871053699:2123] [node 6] ICP22 created new session: [1:7577789246871053738:2123]
2025-11-28T14:52:27.796123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789245080031231:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:27.796147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789246871053738:2123] [node 6] ICS09 handshake done sender: [1:7577789246871053709:2141] self: [1:7577789246871053708:0] peer: [6:7577789245080031241:0] socket: 547 qp: -1
2025-11-28T14:52:27.796188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789246871053738:2123] [node 6] ICS10 traffic start
2025-11-28T14:52:27.796247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:27.796659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789245080031232:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:27.796977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789245606999976:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789245607000068:2121] held: no
2025-11-28T14:52:27.797204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789245607000068:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:27.797520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789245606999976:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.797542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789245080031212:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.797562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:27.797601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789245606999976:2048] [node 1] ICP22 created new session: [5:7577789245607000078:2048]
2025-11-28T14:52:27.797607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:27.797632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789245607000078:2048] [node 1] ICS09 handshake done sender: [5:7577789245607000068:2121] self: [5:7577789245607000073:0] peer: [1:7577789246871053706:0] socket: 531 qp: -1
2025-11-28T14:52:27.797659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789245607000078:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.797652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789245080031212:2105] [node 5] ICP22 created new session: [6:7577789245080031246:2105]
2025-11-28T14:52:27.797675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:27.797703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789245080031246:2105] [node 5] ICS09 handshake done sender: [6:7577789245080031231:2118] self: [6:7577789245080031236:0] peer: [5:7577789245607000049:0] socket: 575 qp: -1
2025-11-28T14:52:27.797736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789245080031246:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:27.797768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:27.798115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789245080031152:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.798191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:27.798237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789245080031152:2050] [node 3] ICP22 created new session: [6:7577789245080031249:2050]
2025-11-28T14:52:27.798279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789245080031249:2050] [node 3] ICS09 handshake done sender: [6:7577789245080031232:2119] self: [6:7577789245080031235:0] peer: [3:7577789245356866485:0] socket: 521 qp: -1
2025-11-28T14:52:27.798309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789245080031249:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:27.798354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:27.801238Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789245080031151:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32303) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789245080031229:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.801354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789245080031150:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789245080031230:2117] held: no
2025-11-28T14:52:27.801346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789245356866515:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15896
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.801568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031244:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:27.801968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789245080031245:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-28T14:52:27.802499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789245080031229:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:27.802697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789245080031230:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:27.802850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789245080031151:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.802908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:27.802952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789245080031151:2049] [node 2] ICP22 created new session: [6:7577789245080031251:2049]
2025-11-28T14:52:27.802990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789245080031251:2049] [node 2] ICS09 handshake done sender: [6:7577789245080031229:2116] self: [6:7577789245080031240:0] peer: [2:7577789244862588179:0] socket: 523 qp: -1
2025-11-28T14:52:27.803023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789245080031251:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:27.803075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:27.803398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789245080031150:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:27.803430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:27.803460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789245080031150:2048] [node 1] ICP22 created new session: [6:7577789245080031253:2048]
2025-11-28T14:52:27.803497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789245080031253:2048] [node 1] ICS09 handshake done sender: [6:7577789245080031230:2117] self: [6:7577789245080031241:0] peer: [1:7577789246871053708:0] socket: 550 qp: -1
2025-11-28T14:52:27.803527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789245080031253:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:27.803570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:27.820271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:27.820447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789244862588203:2048] [node 1] ICS01 socket: 529 reason# EndOfStream
2025-11-28T14:52:27.820476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789244862588087:2048] [node 1] ICP30 unregister session Session# [2:7577789244862588203:2048] VirtualId# [2:7577789244862588193:0]
2025-11-28T14:52:27.820503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789244862588203:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.820562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.820609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.842629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:27.842800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789245356866510:2048] [node 1] ICS01 socket: 556 reason# EndOfStream
2025-11-28T14:52:27.842832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789245356866410:2048] [node 1] ICP30 unregister session Session# [3:7577789245356866510:2048] VirtualId# [3:7577789245356866500:0]
2025-11-28T14:52:27.842859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789245356866510:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.842928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.842968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.846197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:27.846377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789246152106663:2048] [node 1] ICS01 socket: 554 reason# EndOfStream
2025-11-28T14:52:27.846417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789246152106567:2048] [node 1] ICP30 unregister session Session# [4:7577789246152106663:2048] VirtualId# [4:7577789246152106653:0]
2025-11-28T14:52:27.846447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789246152106663:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.846513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.846567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.847149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:27.847291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789245607000078:2048] [node 1] ICS01 socket: 531 reason# EndOfStream
2025-11-28T14:52:27.847318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789245606999976:2048] [node 1] ICP30 unregister session Session# [5:7577789245607000078:2048] VirtualId# [5:7577789245607000073:0]
2025-11-28T14:52:27.847338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789245607000078:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.847390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.847421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.847990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:27.848210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789245080031253:2048] [node 1] ICS01 socket: 550 reason# EndOfStream
2025-11-28T14:52:27.848258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789245080031150:2048] [node 1] ICP30 unregister session Session# [6:7577789245080031253:2048] VirtualId# [6:7577789245080031241:0]
2025-11-28T14:52:27.848290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789245080031253:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.848357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:27.848447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:27.865116Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:27.865275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789245356866505:2049] [node 2] ICS07 socket disconnect 541 reason# EndOfStream
2025-11-28T14:52:27.865296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789245356866505:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.865369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:27.865415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789245356866505:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:27.867244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:27.867419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789246152106658:2049] [node 2] ICS01 socket: 543 reason# EndOfStream
2025-11-28T14:52:27.867452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789246152106568:2049] [node 2] ICP30 unregister session Session# [4:7577789246152106658:2049] VirtualId# [4:7577789246152106654:0]
2025-11-28T14:52:27.867483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789246152106658:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.867537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:27.867574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:27.868043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789245356866411:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32303) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:27.868071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789245356866505:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:27.868380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:27.868512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789245607000076:2049] [node 2] ICS07 socket disconnect 526 reason# EndOfStream
2025-11-28T14:52:27.868537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789245607000076:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.868635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:27.868649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789245356866411:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32303) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:27.868682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789245607000076:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:27.868688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789245356866505:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:27.868707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789245356866411:2049] [node 2] ICP30 unregister session Session# [3:7577789245356866505:2049] VirtualId# [3:7577789245356866501:0]
2025-11-28T14:52:27.868740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:27.868784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789245356866411:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32303) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.868807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32303) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.869226Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789245606999977:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32303) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:27.869261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789245607000076:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:27.869281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789245606999977:2049] [node 2] ICP30 unregister session Session# [5:7577789245607000076:2049] VirtualId# [5:7577789245607000072:0]
2025-11-28T14:52:27.869313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:27.869351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789245606999977:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32303) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.869364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32303) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.869380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:27.869617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789245080031251:2049] [node 2] ICS01 socket: 523 reason# EndOfStream
2025-11-28T14:52:27.869658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789245080031151:2049] [node 2] ICP30 unregister session Session# [6:7577789245080031251:2049] VirtualId# [6:7577789245080031240:0]
2025-11-28T14:52:27.869694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789245080031251:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.869758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:27.869797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:27.879055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789245606999977:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:27.879055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789245356866411:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:27.901829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:27.901988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789246152106650:2050] [node 3] ICS01 socket: 542 reason# EndOfStream
2025-11-28T14:52:27.902017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789246152106569:2050] [node 3] ICP30 unregister session Session# [4:7577789246152106650:2050] VirtualId# [4:7577789246152106648:0]
2025-11-28T14:52:27.902042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789246152106650:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.902094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:27.902127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:27.902822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:27.902969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789245607000066:2050] [node 3] ICS01 socket: 559 reason# EndOfStream
2025-11-28T14:52:27.902996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789245606999978:2050] [node 3] ICP30 unregister session Session# [5:7577789245607000066:2050] VirtualId# [5:7577789245607000058:0]
2025-11-28T14:52:27.903026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789245607000066:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.903096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:27.903137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:27.903635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:27.903764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789245080031249:2050] [node 3] ICS07 socket disconnect 521 reason# EndOfStream
2025-11-28T14:52:27.903786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789245080031249:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.903835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:27.903863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789245080031249:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:27.904443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789245080031152:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6206) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:27.904476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789245080031249:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:27.904497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789245080031152:2050] [node 3] ICP30 unregister session Session# [6:7577789245080031249:2050] VirtualId# [6:7577789245080031235:0]
2025-11-28T14:52:27.904525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:27.904545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789245080031152:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6206) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.904554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6206) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.913858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789245080031152:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:27.923105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:27.923251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789245607000063:2108] [node 4] ICS01 socket: 561 reason# EndOfStream
2025-11-28T14:52:27.923281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789245607000039:2108] [node 4] ICP30 unregister session Session# [5:7577789245607000063:2108] VirtualId# [5:7577789245607000059:0]
2025-11-28T14:52:27.923301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789245607000063:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.923369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:27.923422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:27.924116Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:27.924251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789245080031242:2107] [node 4] ICS07 socket disconnect 577 reason# EndOfStream
2025-11-28T14:52:27.924273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789245080031242:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.924322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:27.924355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789245080031242:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:27.924981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789245080031215:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5229) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:27.925014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789245080031242:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:27.925038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789245080031215:2107] [node 4] ICP30 unregister session Session# [6:7577789245080031242:2107] VirtualId# [6:7577789245080031234:0]
2025-11-28T14:52:27.925063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:27.925086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789245080031215:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5229) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.925094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5229) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:27.935364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789245080031215:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:27.966532Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:27.966704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789245080031246:2105] [node 5] ICS01 socket: 575 reason# EndOfStream
2025-11-28T14:52:27.966744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789245080031212:2105] [node 5] ICP30 unregister session Session# [6:7577789245080031246:2105] VirtualId# [6:7577789245080031236:0]
2025-11-28T14:52:27.966775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789245080031246:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:27.966855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:27.966899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:28.021080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789248290273305:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:28.021079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789248290273304:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:28.044068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789251740083927:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.044138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789251740083928:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:28.068761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789250392623944:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.068863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789250392623945:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:28.093435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789249940919014:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.093462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789249940919015:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:28.093479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789249940919016:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:28.126290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249300060197:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:28.126302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249300060196:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.126322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249300060198:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:28.149662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789249822428430:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:28.149662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789249822428429:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.149693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789249822428431:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:28.243756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249300060259:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:28.244307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789249940919077:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:28.244466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789249822428491:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:28.244464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789249940919078:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:28.244744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249300060260:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:28.244663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789249822428490:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:28.247518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789249300060196:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(37697209ns) {Configure:104(37687812ns) {ProcessConfigured:135(173848ns) {StartInitialHandshake:157(171440ns) {DropHandshakes:470(1172ns) {DropIncomingHandshake:442(275ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:477(1717ns) {SwitchToState:179(1108ns)}}}}}
2025-11-28T14:52:28.247905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789251740083928:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(53554648ns) {Configure:104(38330390ns) {ProcessConfigured:135(162494ns) {StartInitialHandshake:157(160524ns) {DropHandshakes:470(1376ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(612ns) PrepareNewSessionHandshake:477(1051ns) {SwitchToState:179(474ns)}}}}}
2025-11-28T14:52:28.248026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789251740083927:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50658312ns) {Configure:104(50649988ns) {ProcessConfigured:135(102179ns) {StartInitialHandshake:157(100257ns) {DropHandshakes:470(1190ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(907ns) PrepareNewSessionHandshake:477(691ns) {SwitchToState:179(261ns)}}}}}
2025-11-28T14:52:28.247973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789250392623945:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51022698ns) {Configure:104(51018256ns) {ProcessConfigured:135(42801ns) {StartInitialHandshake:157(41358ns) {DropHandshakes:470(1081ns) {DropIncomingHandshake:442(292ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(546ns) PrepareNewSessionHandshake:477(765ns) {SwitchToState:179(311ns)}}}}}
2025-11-28T14:52:28.248052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789250392623944:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50659821ns) {Configure:104(50652158ns) {ProcessConfigured:135(101349ns) {StartInitialHandshake:157(99136ns) {DropHandshakes:470(1028ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(2902ns) PrepareNewSessionHandshake:477(764ns) {SwitchToState:179(317ns)}}}}}
2025-11-28T14:52:28.248266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789251740083995:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:28.248268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789250392624011:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:28.248448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789251740083996:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:28.248542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789251740083997:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:28.248482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789250392624012:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:28.248753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789250392624013:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:28.249012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789248290273304:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56350146ns) {Configure:104(56345872ns) {ProcessConfigured:135(287466ns) {StartInitialHandshake:157(285000ns) {DropHandshakes:470(1226ns) {DropIncomingHandshake:442(405ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(684ns) PrepareNewSessionHandshake:477(981ns) {SwitchToState:179(560ns)}}}}}
2025-11-28T14:52:28.249233Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789248290273305:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(56427507ns) {Configure:104(56424156ns) {ProcessConfigured:135(127188ns) {StartInitialHandshake:157(125374ns) {DropHandshakes:470(1183ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(529ns) PrepareNewSessionHandshake:477(5338ns) {SwitchToState:179(335ns)}}}}}
2025-11-28T14:52:28.249484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789248290273371:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:28.249725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789248290273372:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:28.249912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789248290273373:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.253578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789250392624022:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50275
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.253558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789251740084006:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55597
2025-11-28T14:52:28.253811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789251740084007:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55597
2025-11-28T14:52:28.254116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789250392624024:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50275
2025-11-28T14:52:28.259245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789250392624022:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.259999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249300060274:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39693
2025-11-28T14:52:28.260917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789250392624025:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50275
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.261739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789248290273381:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
2025-11-28T14:52:28.261746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789251740084006:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:28.262017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789249940919090:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
::ffff:127.1.0.0
2025-11-28T14:52:28.261882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789248290273380:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:28.262201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789251740084009:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55597
2025-11-28T14:52:28.262305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789251740084010:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55597
2025-11-28T14:52:28.262320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789248290273382:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
2025-11-28T14:52:28.262308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789249940919089:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-28T14:52:28.262390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789251740084007:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.262387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789250392624024:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.262534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789250392623945:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.262557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273381:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.262720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273380:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.262976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789249940919089:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.264024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789249940919090:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.263443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273382:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.264657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789251740083992:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.264467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789248290273385:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.264685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789251740083992:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.265121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789251740084013:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55597
2025-11-28T14:52:28.265118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789249940919016:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.265192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789250392624028:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50275
2025-11-28T14:52:28.265617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789249940919015:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.265617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789251740084009:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.265836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249300060274:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.266452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789251740083992:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.266748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789250392624016:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.266778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789248290273387:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
2025-11-28T14:52:28.267532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789249940919094:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-28T14:52:28.266779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789250392624016:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.267545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789251740083999:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.268529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789250392624029:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50275
2025-11-28T14:52:28.267571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789251740083999:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.268210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789251740083928:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.268258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:28.268832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789250392624016:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:28.268583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789251740083928:2049] [node 3] ICP22 created new session: [2:7577789251740084016:2049]
2025-11-28T14:52:28.268689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789251740084016:2049] [node 3] ICS09 handshake done sender: [2:7577789251740083992:2111] self: [2:7577789251740083991:0] peer: [3:7577789250392624027:0] socket: 532 qp: -1
2025-11-28T14:52:28.268789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789251740084016:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:28.268847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:28.271503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789250392623945:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789250392624024:2116] held: no
2025-11-28T14:52:28.271624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273387:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.271707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789250392624011:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.271750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273385:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.271751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:28.271810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789250392624011:2107] [node 4] ICP22 created new session: [3:7577789250392624031:2107]
2025-11-28T14:52:28.271818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789249940919094:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.271888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789250392624031:2107] [node 4] ICS09 handshake done sender: [3:7577789250392624016:2112] self: [3:7577789250392624014:0] peer: [4:7577789249940919095:0] socket: 527 qp: -1
2025-11-28T14:52:28.271896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789251740083999:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:28.272003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789250392624031:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:28.271999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789249940919097:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-28T14:52:28.271908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789251740084010:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.272048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.272055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789251740084013:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:28.272050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789250392624024:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.272144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789251740083995:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.272180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:28.272226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789251740083995:2107] [node 4] ICP22 created new session: [2:7577789251740084020:2107]
2025-11-28T14:52:28.272400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789251740084020:2107] [node 4] ICS09 handshake done sender: [2:7577789251740083999:2113] self: [2:7577789251740083998:0] peer: [4:7577789249940919096:0] socket: 536 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:28.272437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249300060276:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39693
::ffff:127.1.0.0
2025-11-28T14:52:28.272431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789251740084020:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:28.272459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.272596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789249940919098:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-28T14:52:28.272697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789251740083927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.272732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789250392624025:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.272767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789250392623945:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.272795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.272825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789250392623945:2049] [node 2] ICP22 created new session: [3:7577789250392624033:2049]
2025-11-28T14:52:28.272862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789250392624033:2049] [node 2] ICS09 handshake done sender: [3:7577789250392624024:2116] self: [3:7577789250392624027:0] peer: [2:7577789251740083991:0] socket: 534 qp: -1
2025-11-28T14:52:28.272855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789251740083995:2107] [node 4] ICP18 handshake (actor [2:7577789251740084013:2120]) is too old
2025-11-28T14:52:28.272889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789250392624033:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:28.272875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789249940919014:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.272917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.272947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789250392624028:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.273124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789249940919089:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:28.273168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789249940919090:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.273267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273375:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.273293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789248290273375:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.273467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789249940919016:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.273343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273368:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.273363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789248290273368:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.275030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789250392623944:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.273508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:28.275353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789250392624011:2107] [node 4] ICP18 handshake (actor [3:7577789250392624028:2118]) is too old
2025-11-28T14:52:28.273549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789249940919016:2050] [node 3] ICP22 created new session: [4:7577789249940919100:2050]
2025-11-28T14:52:28.275082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789249940919100:2050] [node 3] ICS09 handshake done sender: [4:7577789249940919089:2115] self: [4:7577789249940919095:0] peer: [3:7577789250392624014:0] socket: 548 qp: -1
2025-11-28T14:52:28.275200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789249940919015:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-28T14:52:28.275841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789248290273368:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.275237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789249940919100:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:28.275342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:28.276009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273370:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.275823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.276037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789248290273370:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.275872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789249940919015:2049] [node 2] ICP22 created new session: [4:7577789249940919102:2049]
2025-11-28T14:52:28.276034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428507:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
2025-11-28T14:52:28.275919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789249940919102:2049] [node 2] ICS09 handshake done sender: [4:7577789249940919090:2116] self: [4:7577789249940919096:0] peer: [2:7577789251740083998:0] socket: 550 qp: -1
2025-11-28T14:52:28.276052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249300060277:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39693
2025-11-28T14:52:28.276059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789249940919102:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:28.276282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249300060276:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.276093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.276065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428508:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
2025-11-28T14:52:28.276606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789249300060197:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.276582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789248290273304:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.277618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789251740084001:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.277647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789251740084001:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.277730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.278139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789251740084024:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55597
2025-11-28T14:52:28.278126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789248290273304:2048] [node 2] ICP22 created new session: [1:7577789248290273395:2048]
2025-11-28T14:52:28.278184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789248290273395:2048] [node 2] ICS09 handshake done sender: [1:7577789248290273368:2110] self: [1:7577789248290273367:0] peer: [2:7577789251740084022:0] socket: 545 qp: -1
2025-11-28T14:52:28.278217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789248290273395:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:28.278566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.278766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789249822428507:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.278774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249300060277:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.278906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789249300060198:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.278914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789249822428508:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.278965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789249822428430:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.278977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789248290273375:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:28.279591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789249822428431:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.280352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789250392624017:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.281571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789248290273371:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.280409Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789250392624017:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.281725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:28.281843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789248290273370:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:28.281841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789250392624023:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.281853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789248290273371:2107] [node 4] ICP22 created new session: [1:7577789248290273399:2107]
2025-11-28T14:52:28.281863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789250392624023:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.281891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789251740083927:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20292) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789251740084010:2119] held: no
2025-11-28T14:52:28.282069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789248290273399:2107] [node 4] ICS09 handshake done sender: [1:7577789248290273375:2112] self: [1:7577789248290273374:0] peer: [4:7577789249940919099:0] socket: 560 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.282073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789250392624038:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50275
2025-11-28T14:52:28.282024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789249940919015:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:28.282099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789248290273399:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:28.282725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789251740084003:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.282754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789251740084003:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.282213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.283440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249300060280:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39693
2025-11-28T14:52:28.282784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789251740084010:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:28.283707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789251740084001:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.283655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789250392624025:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:28.284056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789251740083927:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:28.284097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:28.284162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789250392623944:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.284139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789251740083927:2048] [node 1] ICP22 created new session: [2:7577789251740084027:2048]
2025-11-28T14:52:28.284195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:28.284226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789250392623944:2048] [node 1] ICP22 created new session: [3:7577789250392624041:2048]
2025-11-28T14:52:28.284472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249300060282:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39693
No RDMA context found for GID: 2025-11-28T14:52:28.284477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789251740084027:2048] [node 1] ICS09 handshake done sender: [2:7577789251740084010:2119] self: [2:7577789251740084022:0] peer: [1:7577789248290273367:0] socket: 546 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:28.284480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249300060281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39693
2025-11-28T14:52:28.284240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789250392624017:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.284538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789251740084027:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:28.284566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.284691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789249940919104:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-28T14:52:28.284505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789250392624041:2048] [node 1] ICS09 handshake done sender: [3:7577789250392624025:2117] self: [3:7577789250392624037:0] peer: [1:7577789248290273369:0] socket: 551 qp: -1
2025-11-28T14:52:28.284533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789250392624041:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:28.284583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789248290273305:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.284825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789251740083996:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.284556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.284862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:28.285147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:28.285108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789251740083996:2108] [node 5] ICP22 created new session: [2:7577789251740084029:2108]
2025-11-28T14:52:28.285178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789248290273305:2049] [node 3] ICP22 created new session: [1:7577789248290273401:2049]
2025-11-28T14:52:28.285145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789251740084029:2108] [node 5] ICS09 handshake done sender: [2:7577789251740084001:2114] self: [2:7577789251740084000:0] peer: [5:7577789249300060278:0] socket: 568 qp: -1
2025-11-28T14:52:28.285448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789248290273401:2049] [node 3] ICS09 handshake done sender: [1:7577789248290273370:2111] self: [1:7577789248290273369:0] peer: [3:7577789250392624037:0] socket: 549 qp: -1
2025-11-28T14:52:28.285222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789251740084029:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:28.286520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789248290273401:2049] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:28.286541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:28.285364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.287074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249300060277:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:28.287330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789249940919014:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20292) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789249940919094:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.287205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249300060276:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.287577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428511:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.287955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789250392624012:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.287883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428512:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
2025-11-28T14:52:28.287903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789249300060197:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.288076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428513:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
2025-11-28T14:52:28.287999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:28.288079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.288028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789250392624012:2108] [node 5] ICP22 created new session: [3:7577789250392624043:2108]
2025-11-28T14:52:28.288084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789249300060198:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.288116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249300060197:2049] [node 2] ICP22 created new session: [5:7577789249300060284:2049]
2025-11-28T14:52:28.288059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789250392624043:2108] [node 5] ICS09 handshake done sender: [3:7577789250392624017:2113] self: [3:7577789250392624015:0] peer: [5:7577789249300060279:0] socket: 571 qp: -1
2025-11-28T14:52:28.288083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789250392624043:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:28.288167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:28.288104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.288168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249300060284:2049] [node 2] ICS09 handshake done sender: [5:7577789249300060276:2116] self: [5:7577789249300060278:0] peer: [2:7577789251740084000:0] socket: 569 qp: -1
2025-11-28T14:52:28.288191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249300060198:2050] [node 3] ICP22 created new session: [5:7577789249300060285:2050]
2025-11-28T14:52:28.288196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249300060284:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:28.288226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.288230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249300060285:2050] [node 3] ICS09 handshake done sender: [5:7577789249300060277:2117] self: [5:7577789249300060279:0] peer: [3:7577789250392624015:0] socket: 575 qp: -1
2025-11-28T14:52:28.288343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789250392624023:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:28.288247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249300060285:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:28.288505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789251740084003:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:28.288265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:28.289594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789249940919094:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:28.290020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789251740083997:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.290357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:28.290105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789249940919014:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.290369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:28.290460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789251740083997:2109] [node 6] ICP22 created new session: [2:7577789251740084031:2109]
2025-11-28T14:52:28.290423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789249940919014:2048] [node 1] ICP22 created new session: [4:7577789249940919107:2048]
2025-11-28T14:52:28.290413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789249822428508:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:28.290725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789249940919107:2048] [node 1] ICS09 handshake done sender: [4:7577789249940919094:2117] self: [4:7577789249940919099:0] peer: [1:7577789248290273374:0] socket: 561 qp: -1
2025-11-28T14:52:28.290822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789251740084031:2109] [node 6] ICS09 handshake done sender: [2:7577789251740084003:2115] self: [2:7577789251740084002:0] peer: [6:7577789249822428509:0] socket: 570 qp: -1
2025-11-28T14:52:28.290860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789251740084031:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:28.290860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789249822428507:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.290755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789249940919107:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:28.291024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.291042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789250392624013:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.290801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.291083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:28.291427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789250392624013:2109] [node 6] ICP22 created new session: [3:7577789250392624045:2109]
2025-11-28T14:52:28.291496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789249822428431:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.291527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789250392624045:2109] [node 6] ICS09 handshake done sender: [3:7577789250392624023:2115] self: [3:7577789250392624021:0] peer: [6:7577789249822428510:0] socket: 572 qp: -1
2025-11-28T14:52:28.291507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789249822428512:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.291568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789250392624045:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:28.291540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789249822428430:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.291974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.291592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:28.291903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789249822428431:2050] [node 3] ICP22 created new session: [6:7577789249822428515:2050]
2025-11-28T14:52:28.292156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.292265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789249822428515:2050] [node 3] ICS09 handshake done sender: [6:7577789249822428508:2116] self: [6:7577789249822428510:0] peer: [3:7577789250392624021:0] socket: 574 qp: -1
2025-11-28T14:52:28.292463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789249822428515:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:28.292464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789249822428430:2049] [node 2] ICP22 created new session: [6:7577789249822428516:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.292694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:28.293092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789249822428516:2049] [node 2] ICS09 handshake done sender: [6:7577789249822428507:2115] self: [6:7577789249822428509:0] peer: [2:7577789251740084002:0] socket: 573 qp: -1
2025-11-28T14:52:28.293109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789249822428516:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:28.293137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.293526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249300060283:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39693
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.293754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249300060280:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.293850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249300060283:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.293835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428514:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
2025-11-28T14:52:28.293992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789249300060196:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.294159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789249300060259:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.294230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273377:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.294257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789248290273377:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.294585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789249940919109:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-28T14:52:28.294896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789249822428514:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.295191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789248290273377:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.295667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789249822428490:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.295727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789248290273372:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.295894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:28.295977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789248290273372:2108] [node 5] ICP22 created new session: [1:7577789248290273405:2108]
2025-11-28T14:52:28.295949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428519:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
2025-11-28T14:52:28.296061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789248290273405:2108] [node 5] ICS09 handshake done sender: [1:7577789248290273377:2113] self: [1:7577789248290273376:0] peer: [5:7577789249300060290:0] socket: 578 qp: -1
2025-11-28T14:52:28.295886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789249940919086:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.296109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789249822428429:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.296152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789248290273405:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.295921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789249940919086:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.296198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.296695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249300060292:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39693
2025-11-28T14:52:28.296147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789249940919088:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.296170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789249940919088:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.296695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789249822428519:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.297415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789249940919109:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.297986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789249822428491:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.297994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789248290273384:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.298030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789248290273384:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.298246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789249300060196:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20292) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789249300060280:2118] held: no
2025-11-28T14:52:28.298374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789249940919086:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.298850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249300060273:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.298977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789249940919088:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.298867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789249300060273:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.299417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789248290273384:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:28.299450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249300060280:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:28.299619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428524:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
2025-11-28T14:52:28.299911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789249940919078:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.300002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789248290273373:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.299969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:28.300329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:28.300160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789249300060196:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.300102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789249940919078:2108] [node 5] ICP22 created new session: [4:7577789249940919114:2108]
2025-11-28T14:52:28.300122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789249940919077:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.300460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789248290273373:2109] [node 6] ICP22 created new session: [1:7577789248290273408:2109]
2025-11-28T14:52:28.300493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:28.300154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:28.300525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789249822428512:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:28.300533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789248290273408:2109] [node 6] ICS09 handshake done sender: [1:7577789248290273384:2117] self: [1:7577789248290273383:0] peer: [6:7577789249822428522:0] socket: 526 qp: -1
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.300570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789248290273408:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:28.300332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789249940919114:2108] [node 5] ICS09 handshake done sender: [4:7577789249940919086:2113] self: [4:7577789249940919085:0] peer: [5:7577789249300060291:0] socket: 583 qp: -1
2025-11-28T14:52:28.300606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.300572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249300060196:2048] [node 1] ICP22 created new session: [5:7577789249300060295:2048]
2025-11-28T14:52:28.300417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789249940919114:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:28.300456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789249940919077:2109] [node 6] ICP22 created new session: [4:7577789249940919115:2109]
2025-11-28T14:52:28.300630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428525:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
2025-11-28T14:52:28.300628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249300060295:2048] [node 1] ICS09 handshake done sender: [5:7577789249300060280:2118] self: [5:7577789249300060290:0] peer: [1:7577789248290273376:0] socket: 579 qp: -1
2025-11-28T14:52:28.300472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.300668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249300060295:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.300501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789249940919115:2109] [node 6] ICS09 handshake done sender: [4:7577789249940919088:2114] self: [4:7577789249940919087:0] peer: [6:7577789249822428520:0] socket: 584 qp: -1
2025-11-28T14:52:28.300738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249300060293:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39693
2025-11-28T14:52:28.300539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789249940919115:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:28.300764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.301186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789249822428490:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.300611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.301202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789249940919112:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-28T14:52:28.301259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:28.301298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789249822428490:2106] [node 4] ICP22 created new session: [6:7577789249822428527:2106]
2025-11-28T14:52:28.301322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789249822428527:2106] [node 4] ICS09 handshake done sender: [6:7577789249822428512:2118] self: [6:7577789249822428520:0] peer: [4:7577789249940919087:0] socket: 558 qp: -1
2025-11-28T14:52:28.301392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789249822428527:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:28.301420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.301615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249300060273:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:28.302071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249300060283:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:28.302553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789249300060260:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.303094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:28.303168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249300060260:2109] [node 6] ICP22 created new session: [5:7577789249300060297:2109]
2025-11-28T14:52:28.303158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789249822428429:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20292) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789249822428514:2120] held: no
2025-11-28T14:52:28.303204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789249300060259:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.303255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249300060297:2109] [node 6] ICS09 handshake done sender: [5:7577789249300060273:2114] self: [5:7577789249300060272:0] peer: [6:7577789249822428523:0] socket: 525 qp: -1
2025-11-28T14:52:28.303276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:28.303301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249300060297:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:28.303301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249300060259:2108] [node 4] ICP22 created new session: [5:7577789249300060298:2108]
2025-11-28T14:52:28.303323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249300060298:2108] [node 4] ICS09 handshake done sender: [5:7577789249300060283:2121] self: [5:7577789249300060291:0] peer: [4:7577789249940919085:0] socket: 588 qp: -1
2025-11-28T14:52:28.303329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.303344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249300060298:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:28.303364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.303429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789249822428514:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:28.303546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789249300060259:2108] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.304756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789249822428526:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9822
2025-11-28T14:52:28.304774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789249822428429:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.304841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:28.304888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789249822428429:2048] [node 1] ICP22 created new session: [6:7577789249822428529:2048]
2025-11-28T14:52:28.304925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789249822428529:2048] [node 1] ICS09 handshake done sender: [6:7577789249822428514:2120] self: [6:7577789249822428522:0] peer: [1:7577789248290273383:0] socket: 529 qp: -1
2025-11-28T14:52:28.304934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789249940919077:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:28.305038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789249822428529:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:28.305248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.305633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789249822428519:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.306394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789249822428491:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3483) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789249822428519:2121] held: no
2025-11-28T14:52:28.306487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789249822428491:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.306563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:28.306610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789249822428491:2105] [node 5] ICP22 created new session: [6:7577789249822428531:2105]
2025-11-28T14:52:28.306650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789249822428531:2105] [node 5] ICS09 handshake done sender: [6:7577789249822428519:2121] self: [6:7577789249822428523:0] peer: [5:7577789249300060272:0] socket: 538 qp: -1
2025-11-28T14:52:28.306679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789249822428531:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:28.306730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.341984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:28.342230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789251740084027:2048] [node 1] ICS01 socket: 546 reason# EndOfStream
2025-11-28T14:52:28.342283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789251740083927:2048] [node 1] ICP30 unregister session Session# [2:7577789251740084027:2048] VirtualId# [2:7577789251740084022:0]
2025-11-28T14:52:28.342330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789251740084027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.342420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.342479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.342935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:28.343197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789250392624041:2048] [node 1] ICS01 socket: 551 reason# EndOfStream
2025-11-28T14:52:28.343253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789250392623944:2048] [node 1] ICP30 unregister session Session# [3:7577789250392624041:2048] VirtualId# [3:7577789250392624037:0]
2025-11-28T14:52:28.343290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789250392624041:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.343401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.343468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.344807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:28.344936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789249940919107:2048] [node 1] ICS01 socket: 561 reason# EndOfStream
2025-11-28T14:52:28.344965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789249940919014:2048] [node 1] ICP30 unregister session Session# [4:7577789249940919107:2048] VirtualId# [4:7577789249940919099:0]
2025-11-28T14:52:28.344986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789249940919107:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.345047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.345103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.345692Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:28.345821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789249300060295:2048] [node 1] ICS01 socket: 579 reason# EndOfStream
2025-11-28T14:52:28.345858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789249300060196:2048] [node 1] ICP30 unregister session Session# [5:7577789249300060295:2048] VirtualId# [5:7577789249300060290:0]
2025-11-28T14:52:28.345873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789249300060295:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.345934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.345971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.346512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:28.346654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789249822428529:2048] [node 1] ICS01 socket: 529 reason# EndOfStream
2025-11-28T14:52:28.346681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789249822428429:2048] [node 1] ICP30 unregister session Session# [6:7577789249822428529:2048] VirtualId# [6:7577789249822428522:0]
2025-11-28T14:52:28.346699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789249822428529:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.346759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.346796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.390359Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:28.390590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789250392624033:2049] [node 2] ICS01 socket: 534 reason# EndOfStream
2025-11-28T14:52:28.390629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789250392623945:2049] [node 2] ICP30 unregister session Session# [3:7577789250392624033:2049] VirtualId# [3:7577789250392624027:0]
2025-11-28T14:52:28.390659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789250392624033:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.390751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:28.390812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:28.392843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:28.392999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789249940919102:2049] [node 2] ICS07 socket disconnect 550 reason# EndOfStream
2025-11-28T14:52:28.393041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789249940919102:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.393109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:28.393163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789249940919102:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:28.393745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:28.393942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789249300060284:2049] [node 2] ICS07 socket disconnect 569 reason# EndOfStream
2025-11-28T14:52:28.393975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789249300060284:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.394057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789249940919015:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.394165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789249940919102:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.394065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:28.394191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789249940919015:2049] [node 2] ICP30 unregister session Session# [4:7577789249940919102:2049] VirtualId# [4:7577789249940919096:0]
2025-11-28T14:52:28.394105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789249300060284:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:28.394236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:28.394261Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789249940919015:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.394276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.394486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:28.394685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789249822428516:2049] [node 2] ICS07 socket disconnect 573 reason# EndOfStream
2025-11-28T14:52:28.394716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789249822428516:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.394800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:28.394847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789249822428516:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:28.394890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789249300060197:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.394927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789249300060284:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.394985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789249300060197:2049] [node 2] ICP30 unregister session Session# [5:7577789249300060284:2049] VirtualId# [5:7577789249300060278:0]
2025-11-28T14:52:28.395027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:28.395049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789249300060197:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.395063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.395589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789249822428430:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.395694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789249822428516:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.395737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789249822428430:2049] [node 2] ICP30 unregister session Session# [6:7577789249822428516:2049] VirtualId# [6:7577789249822428509:0]
2025-11-28T14:52:28.395776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:28.395807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789249822428430:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.395820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11737) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.404397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789249940919015:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:28.404402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789249300060197:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:28.405372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789249822428430:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:28.442182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:28.442341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789249940919100:2050] [node 3] ICS01 socket: 548 reason# EndOfStream
2025-11-28T14:52:28.442377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789249940919016:2050] [node 3] ICP30 unregister session Session# [4:7577789249940919100:2050] VirtualId# [4:7577789249940919095:0]
2025-11-28T14:52:28.442398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789249940919100:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.442445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:28.442480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:28.443012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:28.443164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789249300060285:2050] [node 3] ICS07 socket disconnect 575 reason# EndOfStream
2025-11-28T14:52:28.443225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789249300060285:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.443300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:28.443336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789249300060285:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:28.443606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:28.443744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789249822428515:2050] [node 3] ICS07 socket disconnect 574 reason# EndOfStream
2025-11-28T14:52:28.443775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789249822428515:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.443849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:28.443919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789249300060198:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25540) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.443911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789249822428515:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:28.444005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789249300060285:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.444031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789249300060198:2050] [node 3] ICP30 unregister session Session# [5:7577789249300060285:2050] VirtualId# [5:7577789249300060279:0]
2025-11-28T14:52:28.444092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:28.444122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789249300060198:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.444132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.444618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789249822428431:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25540) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.444678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789249822428515:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.444721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789249822428431:2050] [node 3] ICP30 unregister session Session# [6:7577789249822428515:2050] VirtualId# [6:7577789249822428510:0]
2025-11-28T14:52:28.444758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:28.444794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789249822428431:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.444807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.453547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789249300060198:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:28.454516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789249822428431:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:28.487140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:28.487412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789249300060298:2108] [node 4] ICS07 socket disconnect 588 reason# EndOfStream
2025-11-28T14:52:28.487462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789249300060298:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.487564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:28.487615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789249300060298:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:28.487962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:28.488107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789249822428527:2106] [node 4] ICS07 socket disconnect 558 reason# EndOfStream
2025-11-28T14:52:28.488132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789249822428527:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.488197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:28.488231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789249822428527:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:28.488353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789249300060259:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17705) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.488482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789249300060298:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.488531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789249300060259:2108] [node 4] ICP30 unregister session Session# [5:7577789249300060298:2108] VirtualId# [5:7577789249300060291:0]
2025-11-28T14:52:28.488581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:28.488626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789249300060259:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.488646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.488930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789249822428490:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17705) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.488994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789249822428527:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.489037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789249822428490:2106] [node 4] ICP30 unregister session Session# [6:7577789249822428527:2106] VirtualId# [6:7577789249822428520:0]
2025-11-28T14:52:28.489069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:28.489087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789249822428490:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.489103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.498614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789249822428490:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:28.516898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:28.517107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789249822428531:2105] [node 5] ICS01 socket: 538 reason# EndOfStream
2025-11-28T14:52:28.517156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789249822428491:2105] [node 5] ICP30 unregister session Session# [6:7577789249822428531:2105] VirtualId# [6:7577789249822428523:0]
2025-11-28T14:52:28.517214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789249822428531:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.517305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:28.517355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:28.566793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789250890491248:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:28.566795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789250890491247:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:28.584236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789249610918701:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.584329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789249610918702:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:28.601059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789248228769083:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.601180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789248228769084:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:28.619264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789247953142449:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.619278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789247953142450:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:28.619311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789247953142451:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:28.636606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249213213370:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:28.636633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249213213369:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.636643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249213213371:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:28.657361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789250933276343:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:28.657363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789250933276342:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:28.657368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789250933276344:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:28.736750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789248228769147:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:28.738356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249213213432:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:28.738223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789250890491312:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:28.738218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789249610918764:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:28.739697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789249213213433:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:28.739612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789248228769148:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:28.739847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789250890491313:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:28.739377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789249610918765:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:28.740763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789249610918766:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:28.740781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789248228769149:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:28.740861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789250890491314:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:28.743476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789250890491247:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48179450ns) {Configure:104(48172478ns) {ProcessConfigured:135(68496ns) {StartInitialHandshake:157(64579ns) {DropHandshakes:470(1488ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(820ns) PrepareNewSessionHandshake:477(3667ns) {SwitchToState:179(325ns)}}}}}
2025-11-28T14:52:28.743783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789249213213369:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39324900ns) {Configure:104(39317377ns) {ProcessConfigured:135(117782ns) {StartInitialHandshake:157(115236ns) {DropHandshakes:470(2155ns) {DropIncomingHandshake:442(565ns) DropOutgoingHandshake:457(481ns)} GenerateSessionVirtualId:988(719ns) PrepareNewSessionHandshake:477(1609ns) {SwitchToState:179(750ns)}}}}}
2025-11-28T14:52:28.743700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789250890491248:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49529689ns) {Configure:104(49524824ns) {ProcessConfigured:135(184971ns) {StartInitialHandshake:157(177084ns) {DropHandshakes:470(1839ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(300ns)} GenerateSessionVirtualId:988(934ns) PrepareNewSessionHandshake:477(48250ns) {SwitchToState:179(499ns)}}}}}
2025-11-28T14:52:28.744203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789248228769084:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40349465ns) {Configure:104(40343888ns) {ProcessConfigured:135(72103ns) {StartInitialHandshake:157(70659ns) {DropHandshakes:470(1027ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(347ns) PrepareNewSessionHandshake:477(721ns) {SwitchToState:179(338ns)}}}}}
2025-11-28T14:52:28.744206Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789248228769083:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46824590ns) {Configure:104(46818477ns) {ProcessConfigured:135(180281ns) {StartInitialHandshake:157(178946ns) {DropHandshakes:470(1165ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(419ns) PrepareNewSessionHandshake:477(1240ns) {SwitchToState:179(547ns)}}}}}
2025-11-28T14:52:28.744930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789249610918701:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49334241ns) {Configure:104(41548012ns) {ProcessConfigured:135(183211ns) {StartInitialHandshake:157(181428ns) {DropHandshakes:470(1210ns) {DropIncomingHandshake:442(378ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(516ns) PrepareNewSessionHandshake:477(980ns) {SwitchToState:179(446ns)}}}}}
2025-11-28T14:52:28.745011Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789249610918702:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41597912ns) {Configure:104(41592336ns) {ProcessConfigured:135(241045ns) {StartInitialHandshake:157(239163ns) {DropHandshakes:470(1397ns) {DropIncomingHandshake:442(482ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:477(1403ns) {SwitchToState:179(145ns)}}}}}
2025-11-28T14:52:28.745984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789250933276342:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38941526ns) {Configure:104(38935401ns) {ProcessConfigured:135(255521ns) {StartInitialHandshake:157(253478ns) {DropHandshakes:470(1109ns) {DropIncomingHandshake:442(277ns) DropOutgoingHandshake:457(231ns)} GenerateSessionVirtualId:988(430ns) PrepareNewSessionHandshake:477(832ns) {SwitchToState:179(346ns)}}}}}
2025-11-28T14:52:28.746385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789247953142451:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42229418ns) {Configure:104(36336368ns) {ProcessConfigured:135(269337ns) {StartInitialHandshake:157(267629ns) {DropHandshakes:470(1305ns) {DropIncomingHandshake:442(473ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:477(1264ns) {SwitchToState:179(464ns)}}}}}
2025-11-28T14:52:28.745993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789250933276344:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25031484ns) {Configure:104(25023210ns) {ProcessConfigured:135(261144ns) {StartInitialHandshake:157(259069ns) {DropHandshakes:470(997ns) {DropIncomingHandshake:442(310ns) DropOutgoingHandshake:457(96ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:477(133582ns) {SwitchToState:179(1217ns)}}}}}
2025-11-28T14:52:28.746418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789247953142449:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36479828ns) {Configure:104(36469060ns) {ProcessConfigured:135(400058ns) {StartInitialHandshake:157(399067ns) {DropHandshakes:470(992ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(863ns) {SwitchToState:179(312ns)}}}}}
2025-11-28T14:52:28.746345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789250933276343:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(35905916ns) {Configure:104(35897545ns) {ProcessConfigured:135(278664ns) {StartInitialHandshake:157(276840ns) {DropHandshakes:470(1054ns) {DropIncomingHandshake:442(218ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(1159ns) {SwitchToState:179(449ns)}}}}}
2025-11-28T14:52:28.746466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789247953142450:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36498216ns) {Configure:104(36493403ns) {ProcessConfigured:135(392441ns) {StartInitialHandshake:157(390574ns) {DropHandshakes:470(1088ns) {DropIncomingHandshake:442(308ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:477(1334ns) {SwitchToState:179(557ns)}}}}}
2025-11-28T14:52:28.746746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789250933276426:2105] [node 4] ICP01 ready to work
2025-11-28T14:52:28.746798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789250933276427:2115] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.747348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789247953142518:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:28.747583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789247953142519:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:28.748401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789250890491327:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28213
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.750415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789250890491328:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28213
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.752264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789248228769159:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-11-28T14:52:28.753570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789248228769164:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.754199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491327:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.754390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789249610918777:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60190
2025-11-28T14:52:28.754370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789250890491330:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28213
2025-11-28T14:52:28.755353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789249610918779:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60190
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.755684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789249610918778:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60190
No RDMA context found for GID: 2025-11-28T14:52:28.756280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789248228769159:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.756507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789250890491331:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28213
2025-11-28T14:52:28.756042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789249610918777:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.756653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789248228769165:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-11-28T14:52:28.756320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789249610918779:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.757019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789248228769164:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.756734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789249610918783:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60190
2025-11-28T14:52:28.757205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491316:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.756766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789249610918701:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.757246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789250890491316:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.756989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789249610918778:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.757220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789247953142520:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2581
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.757299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789248228769083:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.758000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789247953142521:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2581
2025-11-28T14:52:28.757390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491328:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.758001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789248228769166:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-11-28T14:52:28.758650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491318:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.759839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789247953142520:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.758677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789250890491318:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.760985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789249610918783:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.760124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789247953142521:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.760476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789250890491318:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:28.761139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789247953142451:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.761191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789247953142526:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2581
2025-11-28T14:52:28.761213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789248228769165:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.760594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789250890491316:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.761874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789248228769169:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-11-28T14:52:28.761277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789247953142449:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.761599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491330:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.762579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789247953142526:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.761951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789248228769155:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.762226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491331:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.762966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789247953142450:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.762688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491320:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.761972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789248228769155:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.763363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789249610918787:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60190
2025-11-28T14:52:28.762698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789250890491248:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.762208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789248228769170:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-11-28T14:52:28.762715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789250890491320:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.763437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789249610918770:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.763452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789249610918770:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.762571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789248228769166:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.762765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789250890491247:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.763681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789249610918772:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.763700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789249610918772:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.762903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789248228769084:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.762796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.762821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789250890491247:2048] [node 2] ICP22 created new session: [1:7577789250890491337:2048]
2025-11-28T14:52:28.764516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789249610918788:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60190
2025-11-28T14:52:28.763633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789248228769155:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:28.762953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:28.765798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789249610918770:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:28.763697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789248228769164:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:28.766245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789249610918772:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:28.763342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789250890491337:2048] [node 2] ICS09 handshake done sender: [1:7577789250890491316:2112] self: [1:7577789250890491315:0] peer: [2:7577789249610918785:0] socket: 535 qp: -1
2025-11-28T14:52:28.765221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789248228769083:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.763363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789250890491248:2049] [node 3] ICP22 created new session: [1:7577789250890491338:2049]
2025-11-28T14:52:28.766964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789249610918779:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.765965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:28.767218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789249610918764:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.763443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789250890491337:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:28.766197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789248228769083:2048] [node 1] ICP22 created new session: [3:7577789248228769174:2048]
2025-11-28T14:52:28.767241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:28.763460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789250890491335:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28213
2025-11-28T14:52:28.767433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789249610918702:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.767073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789248228769174:2048] [node 1] ICS09 handshake done sender: [3:7577789248228769164:2117] self: [3:7577789248228769168:0] peer: [1:7577789250890491317:0] socket: 541 qp: -1
2025-11-28T14:52:28.768649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789247953142533:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2581
2025-11-28T14:52:28.767100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789248228769174:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:28.767437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789249610918764:2107] [node 4] ICP22 created new session: [2:7577789249610918792:2107]
2025-11-28T14:52:28.767582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.763471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789250890491338:2049] [node 3] ICS09 handshake done sender: [1:7577789250890491318:2113] self: [1:7577789250890491317:0] peer: [3:7577789248228769168:0] socket: 537 qp: -1
2025-11-28T14:52:28.763481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.769809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276435:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
2025-11-28T14:52:28.767841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789249610918702:2049] [node 3] ICP22 created new session: [2:7577789249610918793:2049]
2025-11-28T14:52:28.763494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789250890491338:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:28.767439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.764516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:28.768016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789249610918792:2107] [node 4] ICS09 handshake done sender: [2:7577789249610918772:2112] self: [2:7577789249610918771:0] peer: [4:7577789247953142532:0] socket: 557 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.767834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789248228769147:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.768172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789249610918792:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:28.768741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789250890491320:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:28.770815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789247953142520:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:28.768403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.767871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-28T14:52:28.768915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789250890491247:2048] [node 2] ICP18 handshake (actor [1:7577789250890491331:2120]) is too old
::ffff:127.1.0.0
2025-11-28T14:52:28.768822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789249610918793:2049] [node 3] ICS09 handshake done sender: [2:7577789249610918770:2111] self: [2:7577789249610918768:0] peer: [3:7577789248228769172:0] socket: 559 qp: -1
2025-11-28T14:52:28.770892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789247953142534:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2581
No RDMA context found for GID: 2025-11-28T14:52:28.767917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789248228769147:2107] [node 4] ICP22 created new session: [3:7577789248228769177:2107]
::ffff:127.1.0.0
2025-11-28T14:52:28.770494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789250890491312:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.768853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789249610918793:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:28.768576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789248228769177:2107] [node 4] ICS09 handshake done sender: [3:7577789248228769155:2113] self: [3:7577789248228769154:0] peer: [4:7577789247953142530:0] socket: 538 qp: -1
2025-11-28T14:52:28.770544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:28.768884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:28.770947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789247953142535:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2581
2025-11-28T14:52:28.772024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789247953142451:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.771415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249213213447:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41065
2025-11-28T14:52:28.770866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789250890491312:2107] [node 4] ICP22 created new session: [1:7577789250890491345:2107]
2025-11-28T14:52:28.770031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789249610918701:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.768637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789248228769177:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.770909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789250890491345:2107] [node 4] ICS09 handshake done sender: [1:7577789250890491320:2114] self: [1:7577789250890491319:0] peer: [4:7577789247953142531:0] socket: 540 qp: -1
2025-11-28T14:52:28.772069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:28.768863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789248228769083:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:28.772767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789247953142451:2050] [node 3] ICP22 created new session: [4:7577789247953142536:2050]
2025-11-28T14:52:28.770500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:28.770933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789250890491345:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:28.768879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.770545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789249610918701:2048] [node 1] ICP22 created new session: [2:7577789249610918796:2048]
2025-11-28T14:52:28.772982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789247953142536:2050] [node 3] ICS09 handshake done sender: [4:7577789247953142520:2113] self: [4:7577789247953142530:0] peer: [3:7577789248228769154:0] socket: 544 qp: -1
2025-11-28T14:52:28.771160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.772959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789248228769169:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.770814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789249610918796:2048] [node 1] ICS09 handshake done sender: [2:7577789249610918779:2117] self: [2:7577789249610918785:0] peer: [1:7577789250890491315:0] socket: 546 qp: -1
2025-11-28T14:52:28.773014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789247953142536:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:28.773287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789248228769084:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7915) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789248228769166:2119] held: no
2025-11-28T14:52:28.771185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491335:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.770850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789249610918796:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:28.775754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789250933276435:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.773236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.777162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491324:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.776160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789250933276342:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.770895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.773698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789247953142526:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.777012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249213213447:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.773400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789248228769176:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-11-28T14:52:28.777029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276437:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.777194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789250890491324:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.775348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789247953142521:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:28.774153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789248228769147:2107] [node 4] ICP18 handshake (actor [3:7577789248228769169:2120]) is too old
2025-11-28T14:52:28.775747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789249610918787:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.777832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789250890491312:2107] [node 4] ICP18 handshake (actor [1:7577789250890491335:2121]) is too old
2025-11-28T14:52:28.777967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276436:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
2025-11-28T14:52:28.777117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789247953142449:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.778022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789249213213369:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.775169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789248228769166:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.778230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789250890491324:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:28.777144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789247953142450:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.778816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789250933276436:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.778209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789249610918701:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:28.777963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789248228769084:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.778518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249213213449:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41065
2025-11-28T14:52:28.778263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:28.778704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789250890491322:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.778466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789249610918764:2107] [node 4] ICP18 handshake (actor [2:7577789249610918787:2119]) is too old
2025-11-28T14:52:28.778442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.778532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249213213448:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41065
2025-11-28T14:52:28.779094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789250933276344:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.778735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789250890491322:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.778478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789247953142450:2049] [node 2] ICP22 created new session: [4:7577789247953142539:2049]
2025-11-28T14:52:28.778450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.779054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249213213448:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.779002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789250890491314:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.778553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789247953142449:2048] [node 1] ICP22 created new session: [4:7577789247953142538:2048]
2025-11-28T14:52:28.778479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789248228769084:2049] [node 2] ICP22 created new session: [3:7577789248228769181:2049]
2025-11-28T14:52:28.779332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789249213213371:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.778610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789247953142538:2048] [node 1] ICS09 handshake done sender: [4:7577789247953142521:2114] self: [4:7577789247953142531:0] peer: [1:7577789250890491319:0] socket: 547 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:28.779279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-11-28T14:52:28.779341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789250890491314:2109] [node 6] ICP22 created new session: [1:7577789250890491348:2109]
2025-11-28T14:52:28.780035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789250933276437:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.779697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249213213449:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.778626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789248228769181:2049] [node 2] ICS09 handshake done sender: [3:7577789248228769166:2119] self: [3:7577789248228769172:0] peer: [2:7577789249610918768:0] socket: 560 qp: -1
2025-11-28T14:52:28.778638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789247953142538:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:28.778654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789248228769181:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:28.779681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789250890491348:2109] [node 6] ICS09 handshake done sender: [1:7577789250890491324:2116] self: [1:7577789250890491323:0] peer: [6:7577789250933276438:0] socket: 581 qp: -1
2025-11-28T14:52:28.778640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789247953142539:2049] [node 2] ICS09 handshake done sender: [4:7577789247953142526:2116] self: [4:7577789247953142532:0] peer: [2:7577789249610918771:0] socket: 562 qp: -1
2025-11-28T14:52:28.780011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789249213213370:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.780398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276439:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
2025-11-28T14:52:28.778672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789247953142539:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:28.779716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789250890491348:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:28.779246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.778678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.779888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.779040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.780689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789250933276343:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.779733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789248228769163:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.779755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789248228769163:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.781313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789250933276342:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13678) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789250933276435:2130] held: no
2025-11-28T14:52:28.779935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789248228769162:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.779952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789248228769162:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.781353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789249610918774:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.781448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789248228769163:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:28.781389Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789249610918774:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.781523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789248228769162:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.782893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789250890491322:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.783058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789248228769149:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.783378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789250890491313:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.783366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789249610918776:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.783197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:28.784173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789250933276435:2130] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:28.784392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.783234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789248228769149:2110] [node 6] ICP22 created new session: [3:7577789248228769185:2110]
2025-11-28T14:52:28.783387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789249610918776:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.783414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789248228769148:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.783654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789249610918774:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.784563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789250890491313:2108] [node 5] ICP22 created new session: [1:7577789250890491351:2108]
2025-11-28T14:52:28.783421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789248228769185:2110] [node 6] ICS09 handshake done sender: [3:7577789248228769163:2116] self: [3:7577789248228769161:0] peer: [6:7577789250933276440:0] socket: 533 qp: -1
2025-11-28T14:52:28.784540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789249610918765:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.785719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789250890491351:2108] [node 5] ICS09 handshake done sender: [1:7577789250890491322:2115] self: [1:7577789250890491321:0] peer: [5:7577789249213213450:0] socket: 579 qp: -1
2025-11-28T14:52:28.786441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789250933276344:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789250933276436:2131] held: no
2025-11-28T14:52:28.784142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789248228769185:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:28.786099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249213213454:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41065
2025-11-28T14:52:28.784844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:28.784177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.785465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789249610918765:2108] [node 5] ICP22 created new session: [2:7577789249610918802:2108]
2025-11-28T14:52:28.785754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789250890491351:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:28.785543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789249610918776:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:28.784843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:28.787117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789250933276342:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.786469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249213213453:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41065
2025-11-28T14:52:28.785798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.785767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789249610918802:2108] [node 5] ICS09 handshake done sender: [2:7577789249610918774:2113] self: [2:7577789249610918773:0] peer: [5:7577789249213213452:0] socket: 556 qp: -1
2025-11-28T14:52:28.787297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:28.785716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789248228769148:2109] [node 5] ICP22 created new session: [3:7577789248228769187:2109]
2025-11-28T14:52:28.785799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789249610918802:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.787519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789250933276342:2048] [node 1] ICP22 created new session: [6:7577789250933276445:2048]
2025-11-28T14:52:28.786206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789249610918766:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.785755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789248228769187:2109] [node 5] ICS09 handshake done sender: [3:7577789248228769162:2115] self: [3:7577789248228769160:0] peer: [5:7577789249213213451:0] socket: 554 qp: -1
2025-11-28T14:52:28.787805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789250933276445:2048] [node 1] ICS09 handshake done sender: [6:7577789250933276435:2130] self: [6:7577789250933276438:0] peer: [1:7577789250890491323:0] socket: 582 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.788420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249213213455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41065
2025-11-28T14:52:28.787903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789250933276445:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:28.786026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789248228769187:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:28.786252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:28.787921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.786134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.788175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276442:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
2025-11-28T14:52:28.786437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.789256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789249213213369:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13678) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789249213213447:2115] held: no
2025-11-28T14:52:28.787063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789249610918766:2109] [node 6] ICP22 created new session: [2:7577789249610918804:2109]
2025-11-28T14:52:28.790676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789249213213371:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789249213213448:2116] held: no
2025-11-28T14:52:28.787172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789249610918804:2109] [node 6] ICS09 handshake done sender: [2:7577789249610918776:2114] self: [2:7577789249610918775:0] peer: [6:7577789250933276441:0] socket: 585 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:28.787195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789249610918804:2109] [node 6] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:28.791116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789249213213370:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7915) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789249213213449:2117] held: no
2025-11-28T14:52:28.791135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249213213447:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:28.787507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.791228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789247953142544:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2581
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.791293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249213213448:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:28.791284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789250933276343:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7915) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789250933276437:2132] held: no
2025-11-28T14:52:28.791412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276444:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
2025-11-28T14:52:28.791444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789250933276436:2131] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:28.791569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789249213213369:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.791592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249213213449:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.791702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.791997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276446:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
2025-11-28T14:52:28.791862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249213213369:2048] [node 1] ICP22 created new session: [5:7577789249213213457:2048]
2025-11-28T14:52:28.791879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789249213213371:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.791996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:28.792074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789250933276344:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.792015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249213213457:2048] [node 1] ICS09 handshake done sender: [5:7577789249213213447:2115] self: [5:7577789249213213450:0] peer: [1:7577789250890491321:0] socket: 580 qp: -1
2025-11-28T14:52:28.792124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:28.792030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249213213371:2050] [node 3] ICP22 created new session: [5:7577789249213213460:2050]
2025-11-28T14:52:28.792041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249213213457:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:28.792158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789250933276344:2050] [node 3] ICP22 created new session: [6:7577789250933276449:2050]
2025-11-28T14:52:28.792166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789250933276437:2132] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:28.792077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249213213460:2050] [node 3] ICS09 handshake done sender: [5:7577789249213213448:2116] self: [5:7577789249213213451:0] peer: [3:7577789248228769160:0] socket: 584 qp: -1
2025-11-28T14:52:28.792181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789250933276449:2050] [node 3] ICS09 handshake done sender: [6:7577789250933276436:2131] self: [6:7577789250933276440:0] peer: [3:7577789248228769161:0] socket: 545 qp: -1
2025-11-28T14:52:28.792083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:28.792201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789250933276449:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:28.792094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249213213456:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41065
2025-11-28T14:52:28.792236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:28.792111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249213213460:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:28.792142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:28.792284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789249213213370:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.792411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.792453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249213213370:2049] [node 2] ICP22 created new session: [5:7577789249213213464:2049]
2025-11-28T14:52:28.792504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249213213464:2049] [node 2] ICS09 handshake done sender: [5:7577789249213213449:2117] self: [5:7577789249213213452:0] peer: [2:7577789249610918773:0] socket: 586 qp: -1
2025-11-28T14:52:28.792533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249213213464:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:28.792565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.792623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789250933276446:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.792651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789250933276343:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.792702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:28.792724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789250933276343:2049] [node 2] ICP22 created new session: [6:7577789250933276451:2049]
2025-11-28T14:52:28.792749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789250933276451:2049] [node 2] ICS09 handshake done sender: [6:7577789250933276437:2132] self: [6:7577789250933276441:0] peer: [2:7577789249610918775:0] socket: 587 qp: -1
2025-11-28T14:52:28.792765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789250933276451:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:28.792780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:28.793243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249213213456:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.793353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789247953142545:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2581
2025-11-28T14:52:28.793397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249213213463:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41065
2025-11-28T14:52:28.793466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789249213213432:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:28.793442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789250933276426:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.793681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789247953142544:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.793687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789247953142523:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.793701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789247953142523:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.793750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276453:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
2025-11-28T14:52:28.794003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249213213463:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.793836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789247953142527:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.793855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789247953142527:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.795079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789247953142527:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.795166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789247953142523:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.795192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789250933276453:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.795282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789250933276427:2115] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:28.795252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789247953142545:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.795289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276455:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
2025-11-28T14:52:28.795274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789247953142518:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.795467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789249213213446:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:28.795330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:28.795491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789249213213468:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41065
2025-11-28T14:52:28.795491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789247953142519:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.795537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789250933276446:2136] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:28.795496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789249213213446:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:28.795500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789247953142518:2109] [node 6] ICP22 created new session: [4:7577789247953142549:2109]
2025-11-28T14:52:28.795513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:28.795635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789250933276426:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.795552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789247953142519:2108] [node 5] ICP22 created new session: [4:7577789247953142550:2108]
2025-11-28T14:52:28.796079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249213213456:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:28.796134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:28.796245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789249213213446:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:28.795554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789247953142549:2109] [node 6] ICS09 handshake done sender: [4:7577789247953142527:2117] self: [4:7577789247953142525:0] peer: [6:7577789250933276454:0] socket: 532 qp: -1
2025-11-28T14:52:28.796613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789250933276426:2105] [node 4] ICP22 created new session: [6:7577789250933276457:2105]
2025-11-28T14:52:28.796831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789249213213432:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.795577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789247953142550:2108] [node 5] ICS09 handshake done sender: [4:7577789247953142523:2115] self: [4:7577789247953142522:0] peer: [5:7577789249213213466:0] socket: 531 qp: -1
2025-11-28T14:52:28.796638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789250933276457:2105] [node 4] ICS09 handshake done sender: [6:7577789250933276446:2136] self: [6:7577789250933276454:0] peer: [4:7577789247953142525:0] socket: 558 qp: -1
2025-11-28T14:52:28.797038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789249213213433:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:28.797135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:28.796657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789250933276457:2105] [node 4] ICS10 traffic start
2025-11-28T14:52:28.795580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789247953142549:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:28.797233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:28.796673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.797337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249213213432:2108] [node 4] ICP22 created new session: [5:7577789249213213470:2108]
2025-11-28T14:52:28.795603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789247953142550:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:28.795634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.797351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789249213213433:2109] [node 6] ICP22 created new session: [5:7577789249213213471:2109]
::ffff:127.1.0.0
2025-11-28T14:52:28.795949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.797394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249213213471:2109] [node 6] ICS09 handshake done sender: [5:7577789249213213446:2114] self: [5:7577789249213213445:0] peer: [6:7577789250933276456:0] socket: 567 qp: -1
2025-11-28T14:52:28.797418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249213213471:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:28.797422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789249213213470:2108] [node 4] ICS09 handshake done sender: [5:7577789249213213456:2121] self: [5:7577789249213213466:0] peer: [4:7577789247953142522:0] socket: 565 qp: -1
2025-11-28T14:52:28.797449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:28.797454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789250933276459:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40032
2025-11-28T14:52:28.797099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789247953142519:2108] [node 5] ICP18 handshake (actor [4:7577789247953142545:2122]) is too old
2025-11-28T14:52:28.797448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789249213213470:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:28.797480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:28.797610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789250933276453:2137] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:28.798280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789250933276427:2115] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:28.798415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:28.798455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789250933276427:2115] [node 5] ICP22 created new session: [6:7577789250933276460:2115]
2025-11-28T14:52:28.798492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789250933276460:2115] [node 5] ICS09 handshake done sender: [6:7577789250933276453:2137] self: [6:7577789250933276456:0] peer: [5:7577789249213213445:0] socket: 593 qp: -1
2025-11-28T14:52:28.798515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789250933276460:2115] [node 5] ICS10 traffic start
2025-11-28T14:52:28.798542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:28.834986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:28.835203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789249610918796:2048] [node 1] ICS07 socket disconnect 546 reason# EndOfStream
2025-11-28T14:52:28.835238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789249610918796:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.835334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.835408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789249610918796:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:28.835870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:28.836013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789248228769174:2048] [node 1] ICS01 socket: 541 reason# EndOfStream
2025-11-28T14:52:28.836046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789248228769083:2048] [node 1] ICP30 unregister session Session# [3:7577789248228769174:2048] VirtualId# [3:7577789248228769168:0]
2025-11-28T14:52:28.836066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789248228769174:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.836121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.836164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.837480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789249610918701:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13678) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:28.837511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789249610918796:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:28.838270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:28.838507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789247953142538:2048] [node 1] ICS01 socket: 547 reason# EndOfStream
2025-11-28T14:52:28.838561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789247953142449:2048] [node 1] ICP30 unregister session Session# [4:7577789247953142538:2048] VirtualId# [4:7577789247953142531:0]
2025-11-28T14:52:28.838589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789247953142538:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.838714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789249610918701:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13678) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.838727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.838761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789249610918796:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.838841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789249610918701:2048] [node 1] ICP30 unregister session Session# [2:7577789249610918796:2048] VirtualId# [2:7577789249610918785:0]
2025-11-28T14:52:28.838788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.838879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.838903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789249610918701:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.838922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.839195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:28.839409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789249213213457:2048] [node 1] ICS01 socket: 580 reason# EndOfStream
2025-11-28T14:52:28.839461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789249213213369:2048] [node 1] ICP30 unregister session Session# [5:7577789249213213457:2048] VirtualId# [5:7577789249213213450:0]
2025-11-28T14:52:28.839499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789249213213457:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.839562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.839603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.839789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:28.839961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789250933276445:2048] [node 1] ICS01 socket: 582 reason# EndOfStream
2025-11-28T14:52:28.840012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789250933276342:2048] [node 1] ICP30 unregister session Session# [6:7577789250933276445:2048] VirtualId# [6:7577789250933276438:0]
2025-11-28T14:52:28.840036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789250933276445:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.840081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:28.840113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:28.848777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789249610918701:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:28.889056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:28.889215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789248228769181:2049] [node 2] ICS01 socket: 560 reason# EndOfStream
2025-11-28T14:52:28.889248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789248228769084:2049] [node 2] ICP30 unregister session Session# [3:7577789248228769181:2049] VirtualId# [3:7577789248228769172:0]
2025-11-28T14:52:28.889275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789248228769181:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.889330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:28.889373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:28.890747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:28.890876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789247953142539:2049] [node 2] ICS07 socket disconnect 562 reason# EndOfStream
2025-11-28T14:52:28.890901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789247953142539:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.890951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:28.890976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789247953142539:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:28.891431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:28.891644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789249213213464:2049] [node 2] ICS01 socket: 586 reason# EndOfStream
2025-11-28T14:52:28.891527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789247953142450:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7915) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.891568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789247953142539:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.891730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789249213213370:2049] [node 2] ICP30 unregister session Session# [5:7577789249213213464:2049] VirtualId# [5:7577789249213213452:0]
2025-11-28T14:52:28.891585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789247953142450:2049] [node 2] ICP30 unregister session Session# [4:7577789247953142539:2049] VirtualId# [4:7577789247953142532:0]
2025-11-28T14:52:28.891768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789249213213464:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.891673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:28.891848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:28.891696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789247953142450:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7915) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.891889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:28.891704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7915) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.892194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:28.892323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789250933276451:2049] [node 2] ICS01 socket: 587 reason# EndOfStream
2025-11-28T14:52:28.892350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789250933276343:2049] [node 2] ICP30 unregister session Session# [6:7577789250933276451:2049] VirtualId# [6:7577789250933276441:0]
2025-11-28T14:52:28.892407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789250933276451:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.892469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:28.892499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:28.901001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789247953142450:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:28.927584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:28.927743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789247953142536:2050] [node 3] ICS01 socket: 544 reason# EndOfStream
2025-11-28T14:52:28.927769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789247953142451:2050] [node 3] ICP30 unregister session Session# [4:7577789247953142536:2050] VirtualId# [4:7577789247953142530:0]
2025-11-28T14:52:28.927790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789247953142536:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.927846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:28.927879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:28.928400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:28.928602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789249213213460:2050] [node 3] ICS01 socket: 584 reason# EndOfStream
2025-11-28T14:52:28.928646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789249213213371:2050] [node 3] ICP30 unregister session Session# [5:7577789249213213460:2050] VirtualId# [5:7577789249213213451:0]
2025-11-28T14:52:28.928684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789249213213460:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.928765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:28.928809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:28.929062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:28.929250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789250933276449:2050] [node 3] ICS01 socket: 545 reason# EndOfStream
2025-11-28T14:52:28.929298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789250933276344:2050] [node 3] ICP30 unregister session Session# [6:7577789250933276449:2050] VirtualId# [6:7577789250933276440:0]
2025-11-28T14:52:28.929327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789250933276449:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.929393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:28.929454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:28.965204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:28.965449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789249213213470:2108] [node 4] ICS07 socket disconnect 565 reason# EndOfStream
2025-11-28T14:52:28.965487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789249213213470:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.965585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:28.965632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789249213213470:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:28.965889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:28.966061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789250933276457:2105] [node 4] ICS07 socket disconnect 558 reason# EndOfStream
2025-11-28T14:52:28.966098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789250933276457:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:28.966177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:28.966221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789250933276457:2105] [node 4] ICS15 start handshake
2025-11-28T14:52:28.966333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789249213213432:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5386) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.966441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789249213213470:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.966488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789249213213432:2108] [node 4] ICP30 unregister session Session# [5:7577789249213213470:2108] VirtualId# [5:7577789249213213466:0]
2025-11-28T14:52:28.966537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:28.966597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789249213213432:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.966634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.966748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789250933276426:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5386) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:28.966783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789250933276457:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:28.966845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789250933276426:2105] [node 4] ICP30 unregister session Session# [6:7577789250933276457:2105] VirtualId# [6:7577789250933276454:0]
2025-11-28T14:52:28.966878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:28.966896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789250933276426:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.966903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:28.976873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789249213213432:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:28.976874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789250933276426:2105] [node 4] ICP33 wake up from error state
2025-11-28T14:52:29.009776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:29.010009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789250933276460:2115] [node 5] ICS07 socket disconnect 593 reason# EndOfStream
2025-11-28T14:52:29.010059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789250933276460:2115] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.010154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:29.010212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789250933276460:2115] [node 5] ICS15 start handshake
2025-11-28T14:52:29.010836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789250933276427:2115] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27040) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:29.010874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789250933276460:2115] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:29.010930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789250933276427:2115] [node 5] ICP30 unregister session Session# [6:7577789250933276460:2115] VirtualId# [6:7577789250933276456:0]
2025-11-28T14:52:29.010969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:29.010994Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789250933276427:2115] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.011007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27040) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:29.107108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789253257391585:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:29.107108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789253257391584:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:29.124296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789254826630340:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.124338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789254826630341:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:29.146702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789254303692921:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.146752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789254303692922:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:29.168779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789252388511469:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.168825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789252388511471:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:29.168875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789252388511470:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:29.186714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789253269072908:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.186760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789253269072909:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:29.186826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789253269072910:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:29.210914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789255833329597:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.210946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789255833329598:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:29.210961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789255833329599:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:29.272741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789255833329659:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:29.287461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789253257391647:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:29.296514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789253257391648:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:29.293229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789252388511532:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.296210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789253269072972:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:29.296227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789254303692984:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:29.296607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789254303692985:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.296570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789254826630403:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:29.296679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789253257391649:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.297176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789253269072973:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.297166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789255833329660:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:29.296917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789254826630404:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.297803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789254303692986:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:29.297452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789254826630405:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:29.297738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789252388511533:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:29.305309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789253257391584:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51869788ns) {Configure:104(51864624ns) {ProcessConfigured:135(364062ns) {StartInitialHandshake:157(361793ns) {DropHandshakes:470(1123ns) {DropIncomingHandshake:442(386ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(447ns) PrepareNewSessionHandshake:477(1543ns) {SwitchToState:179(768ns)}}}}}
2025-11-28T14:52:29.305533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789254303692922:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31381774ns) {Configure:104(31374766ns) {ProcessConfigured:135(43117ns) {StartInitialHandshake:157(41635ns) {DropHandshakes:470(862ns) {DropIncomingHandshake:442(226ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(440ns) PrepareNewSessionHandshake:477(606ns) {SwitchToState:179(177ns)}}}}}
2025-11-28T14:52:29.305851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789253257391585:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52568004ns) {Configure:104(52565115ns) {ProcessConfigured:135(1167375ns) {StartInitialHandshake:157(1165908ns) {DropHandshakes:470(824ns) {DropIncomingHandshake:442(286ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(559ns) {SwitchToState:179(97ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.310401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789252388511541:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
2025-11-28T14:52:29.310628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329671:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.312660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789253257391658:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9559
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.312856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789255833329671:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.313442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789255833329598:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.313403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789253269072987:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14861
2025-11-28T14:52:29.312726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789252388511541:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.312959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789252388511543:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.313498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789253257391659:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9559
2025-11-28T14:52:29.313701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789252388511470:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.313971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789254826630412:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.313929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391658:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.313982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789254303692998:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6642
2025-11-28T14:52:29.314015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789253257391663:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9559
2025-11-28T14:52:29.316645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789254303693000:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6642
2025-11-28T14:52:29.314007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789254826630410:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.317867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789252388511549:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
2025-11-28T14:52:29.314018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789254826630412:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.314425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789253257391665:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9559
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.314040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789254826630410:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.318453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391659:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.318494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789254303693000:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.318525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789252388511543:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.314460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789254826630420:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5138
2025-11-28T14:52:29.318553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789254303692998:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.319452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391665:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.318335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789254826630419:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5138
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.318558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789253269072987:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.318359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789254826630421:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5138
2025-11-28T14:52:29.319635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789254303693002:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6642
2025-11-28T14:52:29.319167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789253269072989:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14861
2025-11-28T14:52:29.319441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789254826630410:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:29.320225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789253257391667:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9559
2025-11-28T14:52:29.319171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789253269072986:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14861
2025-11-28T14:52:29.319638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789254826630419:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.322007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391663:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.320212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789253269072986:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.323196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789252388511549:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.322717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789254303693002:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.319827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789254826630404:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.322715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789254303692993:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.321264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789253269072910:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.322630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789253269072989:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.324258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391655:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.320223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:29.322853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789253269072908:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.324161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789252388511469:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.324290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789253257391655:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.320237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789254826630412:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.325272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329678:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
2025-11-28T14:52:29.325092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789252388511552:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
2025-11-28T14:52:29.322747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789254303692993:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.325449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789254303692993:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.320251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789254826630404:2109] [node 6] ICP22 created new session: [2:7577789254826630424:2109]
2025-11-28T14:52:29.325450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391654:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.326209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789254303693005:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6642
2025-11-28T14:52:29.325473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789253257391654:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.321912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789254826630424:2109] [node 6] ICS09 handshake done sender: [2:7577789254826630410:2110] self: [2:7577789254826630407:0] peer: [6:7577789255833329677:0] socket: 534 qp: -1
2025-11-28T14:52:29.325881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391667:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.326283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789252388511541:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.322183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789254826630403:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.326597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391656:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.328032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789254303692984:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.327995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789252388511470:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.326619Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789253257391656:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.328133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789254303693005:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.322368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789254826630424:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.327111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789253257391655:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:29.330265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:29.322381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:29.330837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789255833329671:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.322616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:29.330999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789254303692922:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.322693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789254826630403:2107] [node 4] ICP22 created new session: [2:7577789254826630427:2107]
2025-11-28T14:52:29.331011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.331163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789253257391648:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.331015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789254303692984:2108] [node 5] ICP22 created new session: [3:7577789254303693008:2108]
2025-11-28T14:52:29.332773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789255833329598:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.331072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789252388511470:2049] [node 2] ICP22 created new session: [4:7577789252388511555:2049]
2025-11-28T14:52:29.333041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789253269072995:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14861
2025-11-28T14:52:29.331529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789253257391654:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.323845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789254826630427:2107] [node 4] ICS09 handshake done sender: [2:7577789254826630412:2112] self: [2:7577789254826630409:0] peer: [4:7577789252388511550:0] socket: 535 qp: -1
2025-11-28T14:52:29.331067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789254303693008:2108] [node 5] ICS09 handshake done sender: [3:7577789254303692993:2112] self: [3:7577789254303692991:0] peer: [5:7577789253269072991:0] socket: 546 qp: -1
2025-11-28T14:52:29.331125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789252388511555:2049] [node 2] ICS09 handshake done sender: [4:7577789252388511541:2113] self: [4:7577789252388511550:0] peer: [2:7577789254826630409:0] socket: 540 qp: -1
2025-11-28T14:52:29.334934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.333074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:29.324120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789254826630427:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:29.331546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789252388511555:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.331104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789254303693008:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:29.324155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:29.335109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789255833329598:2049] [node 2] ICP22 created new session: [6:7577789255833329680:2049]
2025-11-28T14:52:29.333103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789253257391656:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.333075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789253269072994:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14861
2025-11-28T14:52:29.336348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789255833329680:2049] [node 2] ICS09 handshake done sender: [6:7577789255833329671:2115] self: [6:7577789255833329677:0] peer: [2:7577789254826630407:0] socket: 539 qp: -1
2025-11-28T14:52:29.331521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.332841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:29.324287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789254826630420:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.337268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789255833329680:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:29.333637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789253257391647:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.334027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789253269072993:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14861
2025-11-28T14:52:29.333023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789252388511554:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.337442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.334086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789253257391648:2108] [node 5] ICP22 created new session: [1:7577789253257391676:2108]
2025-11-28T14:52:29.334711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:29.334723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789253269072910:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61977) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789253269072986:2115] held: no
2025-11-28T14:52:29.324377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789254826630421:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.335019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789252388511549:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.335087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789253257391676:2108] [node 5] ICS09 handshake done sender: [1:7577789253257391655:2111] self: [1:7577789253257391650:0] peer: [5:7577789253269072992:0] socket: 560 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:29.336480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789253269072994:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.325511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789254826630340:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.336884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789252388511469:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.339514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789254303693012:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6642
2025-11-28T14:52:29.335156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789253257391647:2107] [node 4] ICP22 created new session: [1:7577789253257391678:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.337890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789253269072908:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789253269072989:2118] held: no
2025-11-28T14:52:29.339822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329682:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
2025-11-28T14:52:29.337901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:29.331014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789254826630403:2107] [node 4] ICP18 handshake (actor [2:7577789254826630421:2117]) is too old
2025-11-28T14:52:29.337928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789253269072986:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:29.335449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789253257391676:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:29.339532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789254303693013:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6642
2025-11-28T14:52:29.338145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789252388511469:2048] [node 1] ICP22 created new session: [4:7577789252388511559:2048]
2025-11-28T14:52:29.333575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789254826630413:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.340662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329679:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
2025-11-28T14:52:29.335486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.342595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789255833329679:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.336350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789253257391678:2107] [node 4] ICS09 handshake done sender: [1:7577789253257391654:2110] self: [1:7577789253257391653:0] peer: [4:7577789252388511553:0] socket: 563 qp: -1
2025-11-28T14:52:29.341132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789254303692922:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789254303693005:2118] held: no
2025-11-28T14:52:29.333610Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789254826630413:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.343280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789255833329682:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.339511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789252388511559:2048] [node 1] ICS09 handshake done sender: [4:7577789252388511549:2117] self: [4:7577789252388511553:0] peer: [1:7577789253257391653:0] socket: 566 qp: -1
2025-11-28T14:52:29.338056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789253269072989:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.342516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789254303693005:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.338395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789253269072910:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.336441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789253257391584:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.336480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.335897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789254826630413:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:29.339532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789252388511559:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:29.344784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789255833329599:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.337258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789253257391584:2048] [node 2] ICP22 created new session: [1:7577789253257391681:2048]
2025-11-28T14:52:29.343481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789254303692922:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.337459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789254826630341:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.340972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:29.337542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.343514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.339307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789253269072908:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.337273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789253257391678:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:29.344887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329684:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
2025-11-28T14:52:29.337314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:29.342291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789252388511558:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.339761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.344864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789254303692922:2049] [node 2] ICP22 created new session: [3:7577789254303693014:2049]
2025-11-28T14:52:29.337893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789253257391681:2048] [node 2] ICS09 handshake done sender: [1:7577789253257391656:2112] self: [1:7577789253257391651:0] peer: [2:7577789254826630431:0] socket: 564 qp: -1
2025-11-28T14:52:29.343673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789252388511558:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.338022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789253257391681:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:29.338062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:29.345252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789252388511469:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:29.339803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789253269072910:2050] [node 3] ICP22 created new session: [5:7577789253269072998:2050]
2025-11-28T14:52:29.345236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789255833329660:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.344975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789254303693014:2049] [node 2] ICS09 handshake done sender: [3:7577789254303693005:2118] self: [3:7577789254303693009:0] peer: [2:7577789254826630408:0] socket: 581 qp: -1
2025-11-28T14:52:29.347063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329685:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
2025-11-28T14:52:29.345115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789254303693014:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:29.337876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789254826630432:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5138
::ffff:127.1.0.0
2025-11-28T14:52:29.340995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789253269072909:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.345137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:29.338012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789254826630341:2049] [node 3] ICP22 created new session: [2:7577789254826630434:2049]
2025-11-28T14:52:29.347224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789252388511542:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.339474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789254826630420:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.341019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:29.347244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789254303692992:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.347254Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789252388511542:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:52:29.339531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789254826630434:2049] [node 3] ICS09 handshake done sender: [2:7577789254826630413:2113] self: [2:7577789254826630408:0] peer: [3:7577789254303693009:0] socket: 579 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:29.342184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789253269072908:2048] [node 1] ICP22 created new session: [5:7577789253269073000:2048]
2025-11-28T14:52:29.347266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789254303692992:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.340590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789254826630434:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:29.340627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.347315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789254303693012:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.342335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789253269072998:2050] [node 3] ICS09 handshake done sender: [5:7577789253269072986:2115] self: [5:7577789253269072991:0] peer: [3:7577789254303692991:0] socket: 549 qp: -1
2025-11-28T14:52:29.340670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789254826630340:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.348220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789255833329685:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.348270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789254303692921:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.342200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:29.342475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789253269073000:2048] [node 1] ICS09 handshake done sender: [5:7577789253269072989:2118] self: [5:7577789253269072992:0] peer: [1:7577789253257391650:0] socket: 567 qp: -1
2025-11-28T14:52:29.342238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789254826630340:2048] [node 1] ICP22 created new session: [2:7577789254826630436:2048]
2025-11-28T14:52:29.349098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789255833329659:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.349099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789252388511542:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:29.342362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789254826630436:2048] [node 1] ICS09 handshake done sender: [2:7577789254826630420:2116] self: [2:7577789254826630431:0] peer: [1:7577789253257391651:0] socket: 565 qp: -1
2025-11-28T14:52:29.342499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789253269072998:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:29.350579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789255833329684:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.350565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391657:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.342504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789254826630436:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:29.342540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.351950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789252388511532:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.351994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789255833329597:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.350608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789253257391657:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.342593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789253269073000:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:29.342525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.352224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789252388511548:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.342722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:29.344796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789254826630418:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.352242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789252388511548:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.347307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789253269073003:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14861
2025-11-28T14:52:29.344831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789254826630418:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.352282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789253257391657:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:29.353727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789254303692992:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:29.350381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789253269073003:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.353409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:29.350534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789253269073004:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14861
2025-11-28T14:52:29.353813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789252388511532:2109] [node 6] ICP22 created new session: [4:7577789252388511565:2109]
2025-11-28T14:52:29.348216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789254826630438:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5138
2025-11-28T14:52:29.350728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789253269072972:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.348420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789254826630418:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:29.353941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789253257391666:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.353860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789252388511565:2109] [node 6] ICS09 handshake done sender: [4:7577789252388511542:2114] self: [4:7577789252388511538:0] peer: [6:7577789255833329687:0] socket: 592 qp: -1
2025-11-28T14:52:29.353898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329688:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
2025-11-28T14:52:29.353972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789253257391666:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.351963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789253269072988:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.350647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789254826630405:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.353887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789252388511565:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.355297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789254303692985:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.351990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789253269072988:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.353915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:29.350681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:29.355362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:29.353445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789253269072994:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.353996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789253257391585:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.351962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789254826630405:2108] [node 5] ICP22 created new session: [2:7577789254826630440:2108]
2025-11-28T14:52:29.356549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789254303692985:2109] [node 6] ICP22 created new session: [3:7577789254303693020:2109]
2025-11-28T14:52:29.353807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789253269072988:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:29.353928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789252388511548:2116] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:29.354146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.352258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789254826630438:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.358272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789255833329682:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.356614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789254303693020:2109] [node 6] ICS09 handshake done sender: [3:7577789254303692992:2111] self: [3:7577789254303692989:0] peer: [6:7577789255833329686:0] socket: 550 qp: -1
2025-11-28T14:52:29.354371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789252388511564:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
2025-11-28T14:52:29.354210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789253269072909:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.354447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.355359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789252388511533:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:29.353785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789254826630440:2108] [node 5] ICS09 handshake done sender: [2:7577789254826630418:2114] self: [2:7577789254826630417:0] peer: [5:7577789253269072999:0] socket: 583 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:29.354209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789253257391585:2049] [node 3] ICP22 created new session: [1:7577789253257391685:2049]
2025-11-28T14:52:29.358268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789254303693020:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.359309Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789255833329660:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789255833329682:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.355131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789253269072909:2049] [node 2] ICP22 created new session: [5:7577789253269073007:2049]
2025-11-28T14:52:29.353930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789254826630440:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:29.359425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789255833329660:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.359098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789254303692921:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789254303693012:2119] held: no
2025-11-28T14:52:29.354678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789253257391685:2049] [node 3] ICS09 handshake done sender: [1:7577789253257391657:2113] self: [1:7577789253257391652:0] peer: [3:7577789254303693017:0] socket: 569 qp: -1
2025-11-28T14:52:29.356115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789252388511564:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.356091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789253269073007:2049] [node 2] ICS09 handshake done sender: [5:7577789253269072994:2120] self: [5:7577789253269072999:0] peer: [2:7577789254826630417:0] socket: 586 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.355139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789253257391685:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:29.359130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:29.353981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.360647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:29.359152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789254826630405:2108] [node 5] ICP18 handshake (actor [2:7577789254826630438:2119]) is too old
2025-11-28T14:52:29.356132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789253269073007:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:29.357964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:29.355180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.360720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789255833329660:2108] [node 4] ICP22 created new session: [6:7577789255833329693:2108]
2025-11-28T14:52:29.359402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789254303693019:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6642
2025-11-28T14:52:29.356549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789253269072973:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.357961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789253257391666:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:29.358292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789252388511533:2108] [node 5] ICP22 created new session: [4:7577789252388511568:2108]
2025-11-28T14:52:29.361427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789255833329599:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61977) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789255833329679:2118] held: no
2025-11-28T14:52:29.360798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789254303693001:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.361633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329691:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
2025-11-28T14:52:29.357927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:29.359655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789253257391649:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.360819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789254303693001:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.359101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789252388511471:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.357993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:29.361757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789255833329693:2108] [node 4] ICS09 handshake done sender: [6:7577789255833329682:2119] self: [6:7577789255833329687:0] peer: [4:7577789252388511538:0] socket: 593 qp: -1
2025-11-28T14:52:29.361909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329692:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
2025-11-28T14:52:29.360620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:29.361142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789254303693012:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.358045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789253269072973:2109] [node 6] ICP22 created new session: [5:7577789253269073010:2109]
2025-11-28T14:52:29.361916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789255833329693:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:29.359168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789252388511568:2108] [node 5] ICS09 handshake done sender: [4:7577789252388511548:2116] self: [4:7577789252388511546:0] peer: [5:7577789253269073005:0] socket: 553 qp: -1
2025-11-28T14:52:29.361998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789254303692921:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.360649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789253257391649:2109] [node 6] ICP22 created new session: [1:7577789253257391688:2109]
2025-11-28T14:52:29.363055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:29.359619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789252388511568:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:29.359277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789253269073010:2109] [node 6] ICS09 handshake done sender: [5:7577789253269072988:2117] self: [5:7577789253269072985:0] peer: [6:7577789255833329689:0] socket: 555 qp: -1
2025-11-28T14:52:29.363020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:29.361549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789253257391688:2109] [node 6] ICS09 handshake done sender: [1:7577789253257391666:2118] self: [1:7577789253257391664:0] peer: [6:7577789255833329690:0] socket: 541 qp: -1
2025-11-28T14:52:29.363137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789254303692921:2048] [node 1] ICP22 created new session: [3:7577789254303693023:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.363826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789254303693023:2048] [node 1] ICS09 handshake done sender: [3:7577789254303693012:2119] self: [3:7577789254303693017:0] peer: [1:7577789253257391652:0] socket: 570 qp: -1
2025-11-28T14:52:29.359619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789253269073010:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.361798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789253257391688:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.360144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.363941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789254303693001:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.362059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:29.366775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789252388511571:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
2025-11-28T14:52:29.359654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:29.363981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789254303693023:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:29.366839Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789255833329659:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3386) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789255833329685:2121] held: no
2025-11-28T14:52:29.361687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789253269072972:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789253269073003:2122] held: no
2025-11-28T14:52:29.364023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:29.365714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789254303692986:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.362889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789253269073008:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14861
2025-11-28T14:52:29.367178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789255833329597:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789255833329684:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.365769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:29.363446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789253269073003:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.366646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789254303692986:2107] [node 4] ICP22 created new session: [3:7577789254303693025:2107]
2025-11-28T14:52:29.367248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789252388511471:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61977) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789252388511564:2121] held: no
2025-11-28T14:52:29.367406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789255833329679:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:29.365238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789253269072972:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.367903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789255833329694:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54633
2025-11-28T14:52:29.365622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:29.367124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789254303693025:2107] [node 4] ICS09 handshake done sender: [3:7577789254303693001:2116] self: [3:7577789254303692999:0] peer: [4:7577789252388511569:0] socket: 536 qp: -1
2025-11-28T14:52:29.368440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789255833329599:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.365730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789253269072972:2108] [node 4] ICP22 created new session: [5:7577789253269073012:2108]
2025-11-28T14:52:29.367157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789254303693025:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:29.367988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789252388511564:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:29.367202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:29.366826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789253269073012:2108] [node 4] ICS09 handshake done sender: [5:7577789253269073003:2122] self: [5:7577789253269073005:0] peer: [4:7577789252388511546:0] socket: 598 qp: -1
2025-11-28T14:52:29.368455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789255833329685:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:29.367179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789253269073012:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:29.368603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.368690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789252388511471:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.367241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:29.368651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789255833329599:2050] [node 3] ICP22 created new session: [6:7577789255833329696:2050]
2025-11-28T14:52:29.368872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.368672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789255833329659:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.369266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789252388511471:2050] [node 3] ICP22 created new session: [4:7577789252388511572:2050]
2025-11-28T14:52:29.369380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789252388511572:2050] [node 3] ICS09 handshake done sender: [4:7577789252388511564:2121] self: [4:7577789252388511569:0] peer: [3:7577789254303692999:0] socket: 601 qp: -1
2025-11-28T14:52:29.369343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789255833329684:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.369415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789252388511572:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:29.369344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:29.370763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.369381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789255833329696:2050] [node 3] ICS09 handshake done sender: [6:7577789255833329679:2118] self: [6:7577789255833329686:0] peer: [3:7577789254303692989:0] socket: 544 qp: -1
2025-11-28T14:52:29.369395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789255833329659:2106] [node 5] ICP22 created new session: [6:7577789255833329697:2106]
2025-11-28T14:52:29.369419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789255833329696:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:29.369449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789255833329697:2106] [node 5] ICS09 handshake done sender: [6:7577789255833329685:2121] self: [6:7577789255833329689:0] peer: [5:7577789253269072985:0] socket: 538 qp: -1
2025-11-28T14:52:29.369478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.369492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789255833329597:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.369537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789255833329697:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:29.369567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.370850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:29.370910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789255833329597:2048] [node 1] ICP22 created new session: [6:7577789255833329700:2048]
2025-11-28T14:52:29.370951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789255833329700:2048] [node 1] ICS09 handshake done sender: [6:7577789255833329684:2120] self: [6:7577789255833329690:0] peer: [1:7577789253257391664:0] socket: 596 qp: -1
2025-11-28T14:52:29.370981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789255833329700:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:29.371012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:29.428452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:29.428631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789254826630436:2048] [node 1] ICS07 socket disconnect 565 reason# EndOfStream
2025-11-28T14:52:29.428654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789254826630436:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.428742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:29.428780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789254826630436:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:29.429905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:29.430046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789254303693023:2048] [node 1] ICS01 socket: 570 reason# EndOfStream
2025-11-28T14:52:29.430075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789254303692921:2048] [node 1] ICP30 unregister session Session# [3:7577789254303693023:2048] VirtualId# [3:7577789254303693017:0]
2025-11-28T14:52:29.430098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789254303693023:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.430158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:29.430205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:29.430985Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789254826630340:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:29.431030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789254826630436:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:29.431771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789254826630340:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:29.431809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789254826630436:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:29.432542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789254826630340:2048] [node 1] ICP30 unregister session Session# [2:7577789254826630436:2048] VirtualId# [2:7577789254826630431:0]
2025-11-28T14:52:29.432599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:29.432621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789254826630340:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.432631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.432787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:29.433017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789252388511559:2048] [node 1] ICS07 socket disconnect 566 reason# EndOfStream
2025-11-28T14:52:29.433046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789252388511559:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.433111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:29.433155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789252388511559:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:29.433509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:29.433649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789253269073000:2048] [node 1] ICS01 socket: 567 reason# EndOfStream
2025-11-28T14:52:29.433676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789253269072908:2048] [node 1] ICP30 unregister session Session# [5:7577789253269073000:2048] VirtualId# [5:7577789253269072992:0]
2025-11-28T14:52:29.433699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789253269073000:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.433758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:29.433835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789252388511469:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:29.433817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:29.434002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789252388511559:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:29.434043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789252388511469:2048] [node 1] ICP30 unregister session Session# [4:7577789252388511559:2048] VirtualId# [4:7577789252388511553:0]
2025-11-28T14:52:29.434090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:29.434117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789252388511469:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.434125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22309) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.434263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:29.434463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789255833329700:2048] [node 1] ICS01 socket: 596 reason# EndOfStream
2025-11-28T14:52:29.434493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789255833329597:2048] [node 1] ICP30 unregister session Session# [6:7577789255833329700:2048] VirtualId# [6:7577789255833329690:0]
2025-11-28T14:52:29.434511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789255833329700:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.434578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:29.434623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:29.443724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789252388511469:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:29.487469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:29.487628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789254303693014:2049] [node 2] ICS01 socket: 581 reason# EndOfStream
2025-11-28T14:52:29.487674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789254303692922:2049] [node 2] ICP30 unregister session Session# [3:7577789254303693014:2049] VirtualId# [3:7577789254303693009:0]
2025-11-28T14:52:29.487700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789254303693014:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.487762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:29.487800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:29.489351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:29.489481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789252388511555:2049] [node 2] ICS07 socket disconnect 540 reason# EndOfStream
2025-11-28T14:52:29.489502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789252388511555:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.489553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:29.489579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789252388511555:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:29.490052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:29.490214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789253269073007:2049] [node 2] ICS07 socket disconnect 586 reason# EndOfStream
2025-11-28T14:52:29.490236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789253269073007:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.490325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:29.490275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789252388511470:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:29.490427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789253269073007:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:29.490359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789252388511555:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:29.490458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789252388511470:2049] [node 2] ICP30 unregister session Session# [4:7577789252388511555:2049] VirtualId# [4:7577789252388511550:0]
2025-11-28T14:52:29.490491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:29.490515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789252388511470:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.490524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.490845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:29.491108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789253269072909:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:29.491105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789255833329680:2049] [node 2] ICS07 socket disconnect 539 reason# EndOfStream
2025-11-28T14:52:29.491152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789255833329680:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.491237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789253269073007:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:29.491250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:29.491296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789255833329680:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:29.491291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789253269072909:2049] [node 2] ICP30 unregister session Session# [5:7577789253269073007:2049] VirtualId# [5:7577789253269072999:0]
2025-11-28T14:52:29.491350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:29.491378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789253269072909:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.491394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.491971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789255833329598:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:29.492022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789255833329680:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:29.492063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789255833329598:2049] [node 2] ICP30 unregister session Session# [6:7577789255833329680:2049] VirtualId# [6:7577789255833329677:0]
2025-11-28T14:52:29.492106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:29.492139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789255833329598:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.492152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:29.500078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789252388511470:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:29.501057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789253269072909:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:29.502162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789255833329598:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:29.531829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:29.532051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789252388511572:2050] [node 3] ICS01 socket: 601 reason# EndOfStream
2025-11-28T14:52:29.532104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789252388511471:2050] [node 3] ICP30 unregister session Session# [4:7577789252388511572:2050] VirtualId# [4:7577789252388511569:0]
2025-11-28T14:52:29.532145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789252388511572:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.532227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:29.532295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:29.532572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:29.532752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789253269072998:2050] [node 3] ICS01 socket: 549 reason# EndOfStream
2025-11-28T14:52:29.532781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789253269072910:2050] [node 3] ICP30 unregister session Session# [5:7577789253269072998:2050] VirtualId# [5:7577789253269072991:0]
2025-11-28T14:52:29.532802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789253269072998:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.532892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:29.532935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:29.533235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:29.533379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789255833329696:2050] [node 3] ICS01 socket: 544 reason# EndOfStream
2025-11-28T14:52:29.533407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789255833329599:2050] [node 3] ICP30 unregister session Session# [6:7577789255833329696:2050] VirtualId# [6:7577789255833329686:0]
2025-11-28T14:52:29.533431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789255833329696:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.533510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:29.533554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:29.596437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:29.596612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789253269073012:2108] [node 4] ICS01 socket: 598 reason# EndOfStream
2025-11-28T14:52:29.596645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789253269072972:2108] [node 4] ICP30 unregister session Session# [5:7577789253269073012:2108] VirtualId# [5:7577789253269073005:0]
2025-11-28T14:52:29.596677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789253269073012:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.596767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:29.596810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:29.597310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:29.597442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789255833329693:2108] [node 4] ICS01 socket: 593 reason# EndOfStream
2025-11-28T14:52:29.597470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789255833329660:2108] [node 4] ICP30 unregister session Session# [6:7577789255833329693:2108] VirtualId# [6:7577789255833329687:0]
2025-11-28T14:52:29.597495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789255833329693:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.597572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:29.597612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:29.650722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:29.650901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789255833329697:2106] [node 5] ICS01 socket: 538 reason# EndOfStream
2025-11-28T14:52:29.650934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789255833329659:2106] [node 5] ICP30 unregister session Session# [6:7577789255833329697:2106] VirtualId# [6:7577789255833329689:0]
2025-11-28T14:52:29.650957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789255833329697:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:29.651037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:29.651074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:29.731779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789255880019257:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:29.731797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789255880019258:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:29.748893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789252980215688:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.748942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789252980215689:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:29.766840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789251981468798:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.767329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789251981468799:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:29.784470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789254645145758:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.784554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789254645145759:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:29.784723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789254645145760:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:29.801695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789252465507027:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.801737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789252465507028:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:29.801839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789252465507029:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:29.820109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789253360222567:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:29.820210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789253360222565:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:29.820250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789253360222566:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:29.894615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789253360222627:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:29.900180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789254645145824:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:29.903896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789251981468862:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.905319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789252465507090:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:29.905766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789254645145825:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.905746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789255880019320:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:29.905765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789255880019321:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:29.905793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789251981468864:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:29.905779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789252980215751:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:29.906550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789253360222628:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:29.906593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789255880019322:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.906093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789251981468863:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:29.906060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789252980215752:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.906791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789252465507091:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:29.906095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789252980215753:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:29.914934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789252980215688:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18919564ns) {Configure:104(18915818ns) {ProcessConfigured:135(66109ns) {StartInitialHandshake:157(63736ns) {DropHandshakes:470(1769ns) {DropIncomingHandshake:442(421ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(593ns) PrepareNewSessionHandshake:477(1672ns) {SwitchToState:179(689ns)}}}}}
2025-11-28T14:52:29.915404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789254645145758:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46087214ns) {Configure:104(46082221ns) {ProcessConfigured:135(484642ns) {StartInitialHandshake:157(483325ns) {DropHandshakes:470(241023ns) {DropIncomingHandshake:442(976ns) DropOutgoingHandshake:457(524ns)} GenerateSessionVirtualId:988(775ns) PrepareNewSessionHandshake:477(558ns) {SwitchToState:179(103ns)}}}}}
2025-11-28T14:52:29.916083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789251981468799:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45816229ns) {Configure:104(45809879ns) {ProcessConfigured:135(303177ns) {StartInitialHandshake:157(301403ns) {DropHandshakes:470(1186ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(441ns) PrepareNewSessionHandshake:477(1040ns) {SwitchToState:179(442ns)}}}}}
2025-11-28T14:52:29.918117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789253360222565:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46078647ns) {Configure:104(46076183ns) {ProcessConfigured:135(1292900ns) {StartInitialHandshake:157(1291369ns) {DropHandshakes:470(443175ns) {DropIncomingHandshake:442(619ns) DropOutgoingHandshake:457(208ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:477(965ns) {SwitchToState:179(537ns)}}}}}
2025-11-28T14:52:29.917263Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789255880019258:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(54638301ns) {Configure:104(47201193ns) {ProcessConfigured:135(646208ns) {StartInitialHandshake:157(644243ns) {DropHandshakes:470(930ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(387ns) PrepareNewSessionHandshake:477(685ns) {SwitchToState:179(316ns)}}}}}
2025-11-28T14:52:29.918849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789253360222566:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31095326ns) {Configure:104(17584417ns) {ProcessConfigured:135(1359921ns) {StartInitialHandshake:157(1357763ns) {DropHandshakes:470(1059ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(170ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:477(1027ns) {SwitchToState:179(443ns)}}}}}
2025-11-28T14:52:29.917750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789255880019257:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(54813047ns) {Configure:104(54807715ns) {ProcessConfigured:135(628022ns) {StartInitialHandshake:157(626638ns) {DropHandshakes:470(133786ns) {DropIncomingHandshake:442(603ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(633ns) PrepareNewSessionHandshake:477(947ns) {SwitchToState:179(393ns)}}}}}
2025-11-28T14:52:29.918315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789252465507027:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22755577ns) {Configure:104(22747595ns) {ProcessConfigured:135(699177ns) {StartInitialHandshake:157(697221ns) {DropHandshakes:470(1208ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:477(2199ns) {SwitchToState:179(1170ns)}}}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: 2025-11-28T14:52:29.923791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789255880019337:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29985
::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-28T14:52:29.925467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789251981468878:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57145
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.925455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789252465507108:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32628
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.925696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789255880019336:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29985
2025-11-28T14:52:29.926840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789251981468879:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57145
2025-11-28T14:52:29.926827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789252980215768:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1572
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.925720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789252465507102:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32628
2025-11-28T14:52:29.927372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222644:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
2025-11-28T14:52:29.927842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789254645145839:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14688
2025-11-28T14:52:29.929245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222645:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
2025-11-28T14:52:29.926221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789255880019338:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29985
2025-11-28T14:52:29.927339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789252980215767:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1572
2025-11-28T14:52:29.928941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789251981468879:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.927783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789252465507106:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32628
2025-11-28T14:52:29.927801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789252980215769:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1572
2025-11-28T14:52:29.928940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789252465507102:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.929615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789251981468880:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57145
2025-11-28T14:52:29.929887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789253360222644:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.929389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789252980215770:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1572
2025-11-28T14:52:29.929184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789252465507108:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.930488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789255880019337:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.929638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789251981468878:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.929430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789252980215768:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.931490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789255880019329:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.929289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789252465507028:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.930498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789253360222567:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.930371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789251981468799:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.929695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789252980215688:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.933679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789253360222645:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.931515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789255880019329:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.934792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789254645145839:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.931506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789251981468880:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.929827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789252465507107:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32628
2025-11-28T14:52:29.931529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789255880019336:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.929849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789252980215769:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.932636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789251981468881:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57145
2025-11-28T14:52:29.930205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789252465507107:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.934961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789253360222627:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.930038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789252980215767:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.931530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789255880019338:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.932644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789251981468869:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.930283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789252465507027:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.933241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789255880019325:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.930287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789252980215761:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.930463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789252465507106:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.932665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789251981468869:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.930311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789252980215761:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.933115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789251981468798:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.932723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789252465507029:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.933259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789255880019325:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.933042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789252980215762:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.939912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222648:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
2025-11-28T14:52:29.934418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789251981468872:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.933570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789255880019329:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.936085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789252465507100:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.933061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789252980215762:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.934942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789255880019330:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.934436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789251981468872:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.933630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789252980215761:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:29.936103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789252465507100:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.934967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789255880019330:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.934988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789251981468881:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.936317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789252980215753:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.938666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789252465507100:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:29.936518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789255880019257:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.935872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789251981468869:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.936489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789252980215762:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.938931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789251981468872:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.936947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789255880019325:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:29.937604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:29.939716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789252465507113:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32628
2025-11-28T14:52:29.936991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.939577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789251981468862:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.939894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789252465507114:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32628
2025-11-28T14:52:29.938679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789252980215770:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.937650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789255880019257:2048] [node 2] ICP22 created new session: [1:7577789255880019344:2048]
2025-11-28T14:52:29.944191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222649:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
2025-11-28T14:52:29.939933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:29.942792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789252465507091:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.938890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789252980215753:2108] [node 5] ICP22 created new session: [2:7577789252980215777:2108]
2025-11-28T14:52:29.944774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222650:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.943240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789252465507107:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.945427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789254645145841:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14688
2025-11-28T14:52:29.940002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789251981468887:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57145
2025-11-28T14:52:29.943399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789252465507102:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.939123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789255880019344:2048] [node 2] ICS09 handshake done sender: [1:7577789255880019329:2111] self: [1:7577789255880019327:0] peer: [2:7577789252980215771:0] socket: 548 qp: -1
2025-11-28T14:52:29.939193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789252980215688:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789252980215768:2116] held: no
2025-11-28T14:52:29.943425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.945396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222651:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
2025-11-28T14:52:29.939158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789255880019344:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:29.939351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789252980215689:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.939211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789255880019320:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.939525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789252980215777:2108] [node 5] ICS09 handshake done sender: [2:7577789252980215761:2112] self: [2:7577789252980215754:0] peer: [5:7577789252465507109:0] socket: 540 qp: -1
2025-11-28T14:52:29.945459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789253360222567:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14815) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789253360222644:2116] held: no
2025-11-28T14:52:29.939362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.943739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789252465507091:2109] [node 6] ICP22 created new session: [5:7577789252465507117:2109]
2025-11-28T14:52:29.939557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789252980215777:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:29.940625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789251981468862:2109] [node 6] ICP22 created new session: [3:7577789251981468890:2109]
2025-11-28T14:52:29.947674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789254645145842:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14688
2025-11-28T14:52:29.941260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789251981468864:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.947807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789254645145841:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.939374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:29.945647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789253360222644:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:29.939563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789255880019320:2108] [node 5] ICP22 created new session: [1:7577789255880019347:2108]
2025-11-28T14:52:29.947113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789253360222627:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29823) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789253360222645:2117] held: no
2025-11-28T14:52:29.939913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789255880019347:2108] [node 5] ICS09 handshake done sender: [1:7577789255880019325:2110] self: [1:7577789255880019323:0] peer: [5:7577789252465507111:0] socket: 553 qp: -1
2025-11-28T14:52:29.939740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789252980215775:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1572
2025-11-28T14:52:29.944723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789252465507028:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.947177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789253360222567:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.941339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:29.939944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789255880019347:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:29.939920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.939960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789255880019330:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:29.945419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.947557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.942279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789251981468864:2108] [node 5] ICP22 created new session: [3:7577789251981468891:2108]
2025-11-28T14:52:29.948089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789254645145758:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.945425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789252465507116:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32628
2025-11-28T14:52:29.943658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789251981468891:2108] [node 5] ICS09 handshake done sender: [3:7577789251981468872:2113] self: [3:7577789251981468866:0] peer: [5:7577789252465507112:0] socket: 550 qp: -1
2025-11-28T14:52:29.940159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.941847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.947757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789253360222567:2050] [node 3] ICP22 created new session: [6:7577789253360222652:2050]
2025-11-28T14:52:29.940700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789252980215689:2049] [node 3] ICP22 created new session: [2:7577789252980215780:2049]
2025-11-28T14:52:29.949180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789253360222650:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.943214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789255880019258:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.941912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789252980215780:2049] [node 3] ICS09 handshake done sender: [2:7577789252980215762:2113] self: [2:7577789252980215758:0] peer: [3:7577789251981468883:0] socket: 542 qp: -1
2025-11-28T14:52:29.951791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789254645145843:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14688
2025-11-28T14:52:29.945458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789252465507027:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.949192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789253360222652:2050] [node 3] ICS09 handshake done sender: [6:7577789253360222644:2116] self: [6:7577789253360222646:0] peer: [3:7577789251981468865:0] socket: 568 qp: -1
2025-11-28T14:52:29.943762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.942285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789252980215780:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:29.943682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789251981468890:2109] [node 6] ICS09 handshake done sender: [3:7577789251981468869:2111] self: [3:7577789251981468865:0] peer: [6:7577789253360222646:0] socket: 544 qp: -1
2025-11-28T14:52:29.953453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789254645145843:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.951447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789253360222652:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.944123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789255880019258:2049] [node 3] ICP22 created new session: [1:7577789255880019351:2049]
2025-11-28T14:52:29.942687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.945465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789252465507028:2049] [node 2] ICP22 created new session: [5:7577789252465507119:2049]
2025-11-28T14:52:29.943687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789251981468891:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:29.944282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789255880019351:2049] [node 3] ICS09 handshake done sender: [1:7577789255880019330:2112] self: [1:7577789255880019326:0] peer: [3:7577789251981468884:0] socket: 560 qp: -1
2025-11-28T14:52:29.943762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789252980215768:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.951776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789253360222645:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:29.945475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789252465507117:2109] [node 6] ICS09 handshake done sender: [5:7577789252465507100:2113] self: [5:7577789252465507095:0] peer: [6:7577789253360222647:0] socket: 575 qp: -1
2025-11-28T14:52:29.943710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789251981468890:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.953913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789254645145759:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.944321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789255880019351:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:29.951832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.945500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789252465507117:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.945482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789252980215688:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.954512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789254645145842:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.943722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.945467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.945623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789252980215753:2108] [node 5] ICP18 handshake (actor [2:7577789252980215770:2118]) is too old
2025-11-28T14:52:29.943739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789251981468878:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.949649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789255880019352:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29985
2025-11-28T14:52:29.946067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:29.951844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789253360222565:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.943743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:29.947143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789252980215688:2048] [node 1] ICP22 created new session: [2:7577789252980215783:2048]
2025-11-28T14:52:29.951829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789255880019331:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.945534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:29.955348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789254645145760:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.951862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789255880019331:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.951924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789253360222627:2106] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:29.945615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
::ffff:127.1.0.02025-11-28T14:52:29.945897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789251981468889:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57145
2025-11-28T14:52:29.947729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789252980215783:2048] [node 1] ICS09 handshake done sender: [2:7577789252980215768:2116] self: [2:7577789252980215771:0] peer: [1:7577789255880019327:0] socket: 561 qp: -1
2025-11-28T14:52:29.953683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789255880019332:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.949634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789251981468799:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9222) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789251981468878:2116] held: no
2025-11-28T14:52:29.951958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: 2025-11-28T14:52:29.953700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789255880019332:2114] [node 6] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-28T14:52:29.945631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789252465507119:2049] [node 2] ICS09 handshake done sender: [5:7577789252465507102:2115] self: [5:7577789252465507109:0] peer: [2:7577789252980215754:0] socket: 547 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:29.949903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789251981468799:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.952691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789253360222627:2106] [node 5] ICP22 created new session: [6:7577789253360222657:2106]
::ffff:127.1.0.02025-11-28T14:52:29.947998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789252980215783:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.954021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789255880019331:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.953183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789253360222657:2106] [node 5] ICS09 handshake done sender: [6:7577789253360222645:2117] self: [6:7577789253360222647:0] peer: [5:7577789252465507095:0] socket: 574 qp: -1
2025-11-28T14:52:29.954216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789255880019354:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29985
2025-11-28T14:52:29.953462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789253360222651:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.945884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789252465507027:2048] [node 1] ICP22 created new session: [5:7577789252465507121:2048]
2025-11-28T14:52:29.955251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789255880019352:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.949944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.948035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:29.953474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789253360222657:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:29.955478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789255880019321:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.951548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789251981468799:2049] [node 2] ICP22 created new session: [3:7577789251981468896:2049]
2025-11-28T14:52:29.953890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.945908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789252465507119:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:29.954663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789253360222566:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.955148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789252980215763:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.956097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.958912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789254645145847:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14688
2025-11-28T14:52:29.945931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:29.951884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789251981468896:2049] [node 2] ICS09 handshake done sender: [3:7577789251981468878:2116] self: [3:7577789251981468883:0] peer: [2:7577789252980215758:0] socket: 558 qp: -1
2025-11-28T14:52:29.956449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789255880019321:2107] [node 4] ICP22 created new session: [1:7577789255880019356:2107]
2025-11-28T14:52:29.955165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789252980215763:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.959393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789254645145758:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789254645145841:2119] held: no
2025-11-28T14:52:29.956791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789252980215760:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.958065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789255880019356:2107] [node 4] ICS09 handshake done sender: [1:7577789255880019331:2113] self: [1:7577789255880019324:0] peer: [4:7577789254645145844:0] socket: 578 qp: -1
2025-11-28T14:52:29.946065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789252465507121:2048] [node 1] ICS09 handshake done sender: [5:7577789252465507107:2117] self: [5:7577789252465507111:0] peer: [1:7577789255880019323:0] socket: 573 qp: -1
2025-11-28T14:52:29.952483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789251981468896:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:29.960182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222660:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
2025-11-28T14:52:29.952672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:29.959405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789254645145846:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14688
2025-11-28T14:52:29.958081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789255880019258:2049] [node 3] ICP18 handshake (actor [1:7577789255880019352:2118]) is too old
2025-11-28T14:52:29.947512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789252465507121:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:29.956823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789252980215760:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.958118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789255880019356:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:29.958572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789252980215763:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.947696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:29.958187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789255880019332:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:29.961422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789253360222650:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.951508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789252465507106:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:29.953385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789251981468880:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.958279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:29.962013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789254645145846:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.962584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789253360222565:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.954555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789251981468798:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.958920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789252980215785:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1572
2025-11-28T14:52:29.962513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789254645145759:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9222) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789254645145843:2121] held: no
2025-11-28T14:52:29.960627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789255880019322:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.953425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789252465507029:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.959659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789252980215760:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:29.962636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:29.955109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.962481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:29.962532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789254645145841:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:29.953558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.962527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789255880019322:2109] [node 6] ICP22 created new session: [1:7577789255880019359:2109]
2025-11-28T14:52:29.959960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789252980215785:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.955381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789251981468798:2048] [node 1] ICP22 created new session: [3:7577789251981468899:2048]
2025-11-28T14:52:29.965952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789254645145758:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.964356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789253360222565:2048] [node 1] ICP22 created new session: [6:7577789253360222662:2048]
2025-11-28T14:52:29.964630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789255880019359:2109] [node 6] ICS09 handshake done sender: [1:7577789255880019332:2114] self: [1:7577789255880019328:0] peer: [6:7577789253360222656:0] socket: 545 qp: -1
2025-11-28T14:52:29.953758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789252465507029:2050] [node 3] ICP22 created new session: [5:7577789252465507125:2050]
2025-11-28T14:52:29.965579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789253360222662:2048] [node 1] ICS09 handshake done sender: [6:7577789253360222650:2120] self: [6:7577789253360222656:0] peer: [1:7577789255880019328:0] socket: 596 qp: -1
2025-11-28T14:52:29.960025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789252980215751:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.965527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789255880019359:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.966182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789254645145760:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14815) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789254645145842:2120] held: no
No RDMA context found for GID: 2025-11-28T14:52:29.965836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
2025-11-28T14:52:29.966212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789254645145850:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14688
2025-11-28T14:52:29.954073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789252465507125:2050] [node 3] ICS09 handshake done sender: [5:7577789252465507106:2116] self: [5:7577789252465507112:0] peer: [3:7577789251981468866:0] socket: 556 qp: -1
2025-11-28T14:52:29.956174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789251981468899:2048] [node 1] ICS09 handshake done sender: [3:7577789251981468880:2118] self: [3:7577789251981468884:0] peer: [1:7577789255880019326:0] socket: 571 qp: -1
2025-11-28T14:52:29.966219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:29.960087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:29.965749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789253360222566:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9222) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789253360222651:2121] held: no
2025-11-28T14:52:29.960266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789252980215751:2107] [node 4] ICP22 created new session: [2:7577789252980215788:2107]
2025-11-28T14:52:29.966271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789254645145849:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14688
2025-11-28T14:52:29.965906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789253360222662:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:29.956493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789251981468873:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.966326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789254645145758:2048] [node 1] ICP22 created new session: [4:7577789254645145853:2048]
2025-11-28T14:52:29.965933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:29.960314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789252980215788:2107] [node 4] ICS09 handshake done sender: [2:7577789252980215763:2114] self: [2:7577789252980215756:0] peer: [4:7577789254645145845:0] socket: 549 qp: -1
2025-11-28T14:52:29.954198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789252465507125:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:29.966855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789254645145853:2048] [node 1] ICS09 handshake done sender: [4:7577789254645145841:2119] self: [4:7577789254645145844:0] peer: [1:7577789255880019324:0] socket: 601 qp: -1
2025-11-28T14:52:29.956513Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789251981468873:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.954660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.956774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789251981468899:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:29.966269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222661:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
2025-11-28T14:52:29.966886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789254645145853:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:29.966861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222663:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
2025-11-28T14:52:29.960393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789252980215752:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.957288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:29.967386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789253360222651:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.967214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:29.960658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:29.959449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789251981468873:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.967423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789253360222565:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:29.967422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789254645145843:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:29.960686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789252980215752:2109] [node 6] ICP22 created new session: [2:7577789252980215790:2109]
2025-11-28T14:52:29.967681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789253360222566:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.959623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789251981468898:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57145
2025-11-28T14:52:29.960884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789252980215788:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:29.968021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789254645145842:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:29.960974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789251981468863:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.967717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.960911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:29.967813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789253360222566:2049] [node 2] ICP22 created new session: [6:7577789253360222665:2049]
2025-11-28T14:52:29.961715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:29.961656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789252980215790:2109] [node 6] ICS09 handshake done sender: [2:7577789252980215760:2111] self: [2:7577789252980215757:0] peer: [6:7577789253360222659:0] socket: 598 qp: -1
2025-11-28T14:52:29.967868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789253360222665:2049] [node 2] ICS09 handshake done sender: [6:7577789253360222651:2121] self: [6:7577789253360222659:0] peer: [2:7577789252980215757:0] socket: 599 qp: -1
2025-11-28T14:52:29.962238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789252980215790:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.968226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789254645145759:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.962497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:29.968044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789253360222665:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:29.969237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789254645145760:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.962034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789251981468863:2107] [node 4] ICP22 created new session: [3:7577789251981468902:2107]
2025-11-28T14:52:29.968084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:29.969353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:29.969402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789254645145759:2049] [node 2] ICP22 created new session: [4:7577789254645145856:2049]
2025-11-28T14:52:29.964297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789251981468902:2107] [node 4] ICS09 handshake done sender: [3:7577789251981468873:2114] self: [3:7577789251981468868:0] peer: [4:7577789254645145848:0] socket: 541 qp: -1
2025-11-28T14:52:29.964638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789251981468902:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:29.969462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789254645145856:2049] [node 2] ICS09 handshake done sender: [4:7577789254645145843:2121] self: [4:7577789254645145845:0] peer: [2:7577789252980215756:0] socket: 576 qp: -1
2025-11-28T14:52:29.964699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:29.969656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789254645145856:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:29.969691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.969720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:29.969761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789254645145760:2050] [node 3] ICP22 created new session: [4:7577789254645145858:2050]
2025-11-28T14:52:29.969803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789254645145858:2050] [node 3] ICS09 handshake done sender: [4:7577789254645145842:2120] self: [4:7577789254645145848:0] peer: [3:7577789251981468868:0] socket: 602 qp: -1
2025-11-28T14:52:29.969833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789254645145858:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:29.969863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:29.969892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789252465507127:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32628
2025-11-28T14:52:29.970217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789253360222663:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.970417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789253360222628:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.970635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789254645145832:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.970669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789254645145832:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.970853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789252465507127:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.971002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789252465507090:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:29.971185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789254645145835:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:29.971214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789254645145835:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:29.971457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789254645145832:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:29.971667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789254645145825:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.971763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:29.971800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789254645145825:2109] [node 6] ICP22 created new session: [4:7577789254645145862:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.971875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789254645145862:2109] [node 6] ICS09 handshake done sender: [4:7577789254645145832:2114] self: [4:7577789254645145829:0] peer: [6:7577789253360222667:0] socket: 552 qp: -1
2025-11-28T14:52:29.971932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789254645145862:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:29.971986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:29.972069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789254645145835:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:29.971946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789253360222668:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:29.972164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789253360222628:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24633) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789253360222663:2124] held: no
2025-11-28T14:52:29.972299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789252465507129:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32628
2025-11-28T14:52:29.972452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789254645145824:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:29.972482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789253360222663:2124] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.973470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789252465507127:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:29.973271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789253360222628:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.973498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:29.974091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789252465507090:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24633) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789252465507127:2122] held: no
2025-11-28T14:52:29.974130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789254645145824:2108] [node 5] ICP22 created new session: [4:7577789254645145864:2108]
2025-11-28T14:52:29.974459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789252465507090:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:29.974548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789254645145864:2108] [node 5] ICS09 handshake done sender: [4:7577789254645145835:2117] self: [4:7577789254645145828:0] peer: [5:7577789252465507128:0] socket: 543 qp: -1
2025-11-28T14:52:29.973310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:29.974497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:29.974611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789254645145864:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:29.973354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789253360222628:2107] [node 4] ICP22 created new session: [6:7577789253360222669:2107]
2025-11-28T14:52:29.974527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789252465507090:2108] [node 4] ICP22 created new session: [5:7577789252465507130:2108]
2025-11-28T14:52:29.974085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789253360222669:2107] [node 4] ICS09 handshake done sender: [6:7577789253360222663:2124] self: [6:7577789253360222667:0] peer: [4:7577789254645145829:0] socket: 554 qp: -1
2025-11-28T14:52:29.974557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789252465507130:2108] [node 4] ICS09 handshake done sender: [5:7577789252465507127:2122] self: [5:7577789252465507128:0] peer: [4:7577789254645145828:0] socket: 555 qp: -1
2025-11-28T14:52:29.974670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:29.974118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789253360222669:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:29.974609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789252465507130:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:29.974154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:29.974642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.018801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:30.018967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789252980215783:2048] [node 1] ICS01 socket: 561 reason# EndOfStream
2025-11-28T14:52:30.018999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789252980215688:2048] [node 1] ICP30 unregister session Session# [2:7577789252980215783:2048] VirtualId# [2:7577789252980215771:0]
2025-11-28T14:52:30.019023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789252980215783:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.019102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.019162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.020333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:30.020572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789251981468899:2048] [node 1] ICS07 socket disconnect 571 reason# EndOfStream
2025-11-28T14:52:30.020627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789251981468899:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.020722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.020775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789251981468899:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:30.022938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789251981468798:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:30.022981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789251981468899:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:30.022965Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:30.023388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789254645145853:2048] [node 1] ICS01 socket: 601 reason# EndOfStream
2025-11-28T14:52:30.023440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789254645145758:2048] [node 1] ICP30 unregister session Session# [4:7577789254645145853:2048] VirtualId# [4:7577789254645145844:0]
2025-11-28T14:52:30.023477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789254645145853:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.023560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.023617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.023923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789251981468798:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.023969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789251981468899:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.024028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789251981468798:2048] [node 1] ICP30 unregister session Session# [3:7577789251981468899:2048] VirtualId# [3:7577789251981468884:0]
2025-11-28T14:52:30.024072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.024095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789251981468798:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.024103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.024341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:30.024598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789252465507121:2048] [node 1] ICS07 socket disconnect 573 reason# EndOfStream
2025-11-28T14:52:30.024632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789252465507121:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.024707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.024745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789252465507121:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:30.025326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789252465507027:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.025292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:30.025581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789252465507121:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.025613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789252465507027:2048] [node 1] ICP30 unregister session Session# [5:7577789252465507121:2048] VirtualId# [5:7577789252465507111:0]
2025-11-28T14:52:30.025646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.025639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789253360222662:2048] [node 1] ICS07 socket disconnect 596 reason# EndOfStream
2025-11-28T14:52:30.025681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789252465507027:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.025695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.025663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789253360222662:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.025720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.025760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789253360222662:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:30.026443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789253360222565:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.026485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789253360222662:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.026546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789253360222565:2048] [node 1] ICP30 unregister session Session# [6:7577789253360222662:2048] VirtualId# [6:7577789253360222656:0]
2025-11-28T14:52:30.026588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.026608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789253360222565:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.026617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.033591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789251981468798:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:30.035600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789252465507027:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:30.036633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789253360222565:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:30.064394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:30.064663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789251981468896:2049] [node 2] ICS01 socket: 558 reason# EndOfStream
2025-11-28T14:52:30.064717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789251981468799:2049] [node 2] ICP30 unregister session Session# [3:7577789251981468896:2049] VirtualId# [3:7577789251981468883:0]
2025-11-28T14:52:30.064754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789251981468896:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.064865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:30.064947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:30.067407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:30.067627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789254645145856:2049] [node 2] ICS01 socket: 576 reason# EndOfStream
2025-11-28T14:52:30.067681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789254645145759:2049] [node 2] ICP30 unregister session Session# [4:7577789254645145856:2049] VirtualId# [4:7577789254645145845:0]
2025-11-28T14:52:30.067723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789254645145856:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.067813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:30.067870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:30.068104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:30.068253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789252465507119:2049] [node 2] ICS07 socket disconnect 547 reason# EndOfStream
2025-11-28T14:52:30.068290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789252465507119:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.068360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:30.068426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789252465507119:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:30.068867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:30.069002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789253360222665:2049] [node 2] ICS01 socket: 599 reason# EndOfStream
2025-11-28T14:52:30.069032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789253360222566:2049] [node 2] ICP30 unregister session Session# [6:7577789253360222665:2049] VirtualId# [6:7577789253360222659:0]
2025-11-28T14:52:30.069054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789253360222665:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.069102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:30.069088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789252465507028:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9222) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.069149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:30.069247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789252465507119:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.069276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789252465507028:2049] [node 2] ICP30 unregister session Session# [5:7577789252465507119:2049] VirtualId# [5:7577789252465507109:0]
2025-11-28T14:52:30.069319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:30.069342Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789252465507028:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9222) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.069352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9222) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.079639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789252465507028:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:30.125502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:30.125748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789254645145858:2050] [node 3] ICS01 socket: 602 reason# EndOfStream
2025-11-28T14:52:30.125808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789254645145760:2050] [node 3] ICP30 unregister session Session# [4:7577789254645145858:2050] VirtualId# [4:7577789254645145848:0]
2025-11-28T14:52:30.125858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789254645145858:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.125972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:30.126029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:30.126197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:30.126347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789252465507125:2050] [node 3] ICS07 socket disconnect 556 reason# EndOfStream
2025-11-28T14:52:30.126370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789252465507125:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.126439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:30.126476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789252465507125:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:30.126826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:30.126968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789252465507029:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14815) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.126962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789253360222652:2050] [node 3] ICS01 socket: 568 reason# EndOfStream
2025-11-28T14:52:30.127062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789252465507125:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.126985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789253360222567:2050] [node 3] ICP30 unregister session Session# [6:7577789253360222652:2050] VirtualId# [6:7577789253360222646:0]
2025-11-28T14:52:30.127110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789252465507029:2050] [node 3] ICP30 unregister session Session# [5:7577789252465507125:2050] VirtualId# [5:7577789252465507112:0]
2025-11-28T14:52:30.127011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789253360222652:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.127079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:30.127149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:30.127120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:30.127182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789252465507029:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.127193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.136982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789252465507029:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:30.171460Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:30.171635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789252465507130:2108] [node 4] ICS01 socket: 555 reason# EndOfStream
2025-11-28T14:52:30.171666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789252465507090:2108] [node 4] ICP30 unregister session Session# [5:7577789252465507130:2108] VirtualId# [5:7577789252465507128:0]
2025-11-28T14:52:30.171691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789252465507130:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.171771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:30.171823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:30.172244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:30.172424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789253360222669:2107] [node 4] ICS01 socket: 554 reason# EndOfStream
2025-11-28T14:52:30.172452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789253360222628:2107] [node 4] ICP30 unregister session Session# [6:7577789253360222669:2107] VirtualId# [6:7577789253360222667:0]
2025-11-28T14:52:30.172513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789253360222669:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.172604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:30.172646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:30.204979Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:30.205146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789253360222657:2106] [node 5] ICS01 socket: 574 reason# EndOfStream
2025-11-28T14:52:30.205195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789253360222627:2106] [node 5] ICP30 unregister session Session# [6:7577789253360222657:2106] VirtualId# [6:7577789253360222647:0]
2025-11-28T14:52:30.205230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789253360222657:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.205336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:30.205376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:30.314498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789260261854533:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:30.314538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789260261854534:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:30.332482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789258388677676:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.332529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789258388677677:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:30.349868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789260351214616:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.349904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789260351214617:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:30.368594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258453754243:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.368672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258453754244:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:30.368719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258453754245:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:30.385305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789256938802575:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.385348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789256938802576:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:30.385376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789256938802577:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:30.401957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789257030140063:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.401994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789257030140064:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:30.402017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789257030140065:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:30.454530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789257030140123:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:30.480845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789260261854597:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:30.481092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789260351214679:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:30.481206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789260261854598:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:30.481196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789256938802638:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:30.481301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258453754307:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:30.481167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789258388677739:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:30.481477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789260351214680:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:30.481378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789257030140127:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:30.481743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789256938802639:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:30.481949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258453754308:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:30.481935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789260261854599:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:30.482026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789260351214681:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:30.482030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789258388677740:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:30.482265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789258388677741:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.490085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789256938802653:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.490408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789260351214695:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1633
2025-11-28T14:52:30.490461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789260351214696:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1633
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.491865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789258388677752:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58869
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.492066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789258388677753:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58869
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:30.492341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789260261854613:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43872
2025-11-28T14:52:30.492331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789258388677754:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58869
2025-11-28T14:52:30.493088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789260261854614:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.493098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789258388677755:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58869
2025-11-28T14:52:30.493333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789260261854615:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.495253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258453754319:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28706
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.495564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258453754320:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28706
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.495953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258453754321:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28706
2025-11-28T14:52:30.496033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789256938802653:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.497040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854615:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.497105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854614:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.496973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789260351214696:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.497161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789260351214697:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1633
2025-11-28T14:52:30.497238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258453754320:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.497218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789260351214698:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1633
2025-11-28T14:52:30.497315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789260351214695:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.497562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789260351214616:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.497761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854613:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.497936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854603:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.497954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789260261854603:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.498727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789260261854616:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.498843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789260261854617:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43872
2025-11-28T14:52:30.498805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789258388677755:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.498878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789258388677754:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.499014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789258388677753:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.499101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789258388677752:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.499154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789260261854603:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:30.499325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789260351214697:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:30.499450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789260351214701:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1633
2025-11-28T14:52:30.499506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854616:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.499541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789260261854534:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.499684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:30.499831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789260261854534:2049] [node 3] ICP22 created new session: [1:7577789260261854622:2049]
2025-11-28T14:52:30.499920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789260261854622:2049] [node 3] ICS09 handshake done sender: [1:7577789260261854603:2112] self: [1:7577789260261854602:0] peer: [3:7577789260351214700:0] socket: 547 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.499915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789258388677676:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.499944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789260261854622:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:30.499967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.499969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789258388677759:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58869
2025-11-28T14:52:30.499990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789260351214617:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.500024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789260351214695:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:30.500272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789260351214616:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.500312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.500338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789260351214616:2048] [node 1] ICP22 created new session: [3:7577789260351214703:2048]
2025-11-28T14:52:30.500342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789258388677745:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.500359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789258388677745:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.500382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789260351214703:2048] [node 1] ICS09 handshake done sender: [3:7577789260351214695:2115] self: [3:7577789260351214700:0] peer: [1:7577789260261854602:0] socket: 560 qp: -1
2025-11-28T14:52:30.500407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789260351214703:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:30.500442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:30.500927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854601:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.500980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789258388677745:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:30.500947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789260261854601:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.502358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789258388677677:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.502397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:30.502378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258453754319:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.502420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789258388677677:2049] [node 3] ICP22 created new session: [2:7577789258388677765:2049]
2025-11-28T14:52:30.502446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789258388677765:2049] [node 3] ICS09 handshake done sender: [2:7577789258388677745:2111] self: [2:7577789258388677744:0] peer: [3:7577789260351214702:0] socket: 575 qp: -1
2025-11-28T14:52:30.502464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789258388677765:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:30.502489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.502838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789258453754245:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.502840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854617:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.502965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789258453754243:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.503102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854605:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.503128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789260261854605:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.503695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258453754321:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.503687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789256938802655:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-28T14:52:30.503761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789258453754244:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.503771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789258388677759:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.503781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789260261854601:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:30.503854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789260351214687:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.503877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789260351214687:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.504489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789258388677747:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.504510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789258388677747:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.504980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789260261854533:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.505353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.505398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789260261854533:2048] [node 2] ICP22 created new session: [1:7577789260261854629:2048]
2025-11-28T14:52:30.505448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789260261854629:2048] [node 2] ICS09 handshake done sender: [1:7577789260261854601:2111] self: [1:7577789260261854600:0] peer: [2:7577789258388677763:0] socket: 546 qp: -1
2025-11-28T14:52:30.505493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789260261854629:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:30.505541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.505700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789258388677767:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58869
2025-11-28T14:52:30.505735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789256938802656:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-28T14:52:30.505750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789260351214698:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.505849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789260351214617:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62949) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789260351214697:2117] held: no
2025-11-28T14:52:30.505871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258453754328:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28706
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.506137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789258388677747:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:30.506200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789260351214705:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1633
2025-11-28T14:52:30.506416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789258388677752:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.506830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258453754329:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28706
2025-11-28T14:52:30.506875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258453754321:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:30.506950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789260351214687:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.507158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258453754330:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28706
2025-11-28T14:52:30.507151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789256938802657:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-28T14:52:30.507269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789260351214697:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:30.507647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789258388677739:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.507884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789258453754244:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.507897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:30.507914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.508004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789258388677739:2107] [node 4] ICP22 created new session: [2:7577789258388677771:2107]
2025-11-28T14:52:30.507938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258453754244:2049] [node 2] ICP22 created new session: [4:7577789258453754331:2049]
2025-11-28T14:52:30.507995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789260351214679:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.508033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789258388677771:2107] [node 4] ICS09 handshake done sender: [2:7577789258388677747:2112] self: [2:7577789258388677746:0] peer: [4:7577789258453754327:0] socket: 576 qp: -1
2025-11-28T14:52:30.508030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.508061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789258388677771:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:30.508018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258453754331:2049] [node 2] ICS09 handshake done sender: [4:7577789258453754321:2118] self: [4:7577789258453754327:0] peer: [2:7577789258388677746:0] socket: 578 qp: -1
2025-11-28T14:52:30.508082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.508066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789260351214679:2107] [node 4] ICP22 created new session: [3:7577789260351214710:2107]
2025-11-28T14:52:30.508094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789260351214710:2107] [node 4] ICS09 handshake done sender: [3:7577789260351214687:2112] self: [3:7577789260351214686:0] peer: [4:7577789258453754325:0] socket: 555 qp: -1
2025-11-28T14:52:30.508036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258453754331:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:30.508119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789260351214710:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:30.508065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:30.508136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.508123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140141:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.508260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789260261854605:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:30.508649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789260351214617:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.508682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.508709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789260351214617:2049] [node 2] ICP22 created new session: [3:7577789260351214712:2049]
2025-11-28T14:52:30.508708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789260261854597:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.508755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:30.508745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789260351214712:2049] [node 2] ICS09 handshake done sender: [3:7577789260351214697:2117] self: [3:7577789260351214702:0] peer: [2:7577789258388677744:0] socket: 582 qp: -1
2025-11-28T14:52:30.508764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789260351214712:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:30.508791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789260261854597:2107] [node 4] ICP22 created new session: [1:7577789260261854632:2107]
2025-11-28T14:52:30.508831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789260261854632:2107] [node 4] ICS09 handshake done sender: [1:7577789260261854605:2113] self: [1:7577789260261854604:0] peer: [4:7577789258453754326:0] socket: 548 qp: -1
2025-11-28T14:52:30.508780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:30.509276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789260261854632:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:30.509300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.509271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789256938802655:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.509390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789256938802575:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.509640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789258388677676:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.509674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.509713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789258388677676:2048] [node 1] ICP22 created new session: [2:7577789258388677773:2048]
2025-11-28T14:52:30.509749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789258388677773:2048] [node 1] ICS09 handshake done sender: [2:7577789258388677752:2115] self: [2:7577789258388677763:0] peer: [1:7577789260261854600:0] socket: 566 qp: -1
2025-11-28T14:52:30.509803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789258388677773:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:30.509840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:30.510035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789258388677676:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:30.510111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258453754320:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.510406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140142:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.510484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854607:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.510510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789260261854607:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.511164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789256938802656:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.511441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789256938802576:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.511570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789258453754245:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.511602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:30.511625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258453754245:2050] [node 3] ICP22 created new session: [4:7577789258453754333:2050]
2025-11-28T14:52:30.511651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258453754333:2050] [node 3] ICS09 handshake done sender: [4:7577789258453754320:2117] self: [4:7577789258453754325:0] peer: [3:7577789260351214686:0] socket: 573 qp: -1
2025-11-28T14:52:30.511675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258453754333:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:30.511726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.511763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789258388677749:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.511790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789258388677749:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.511914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789256938802660:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-28T14:52:30.511960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789258453754245:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:30.512951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789258388677749:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.512992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789260261854607:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:30.513332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140143:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.513650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789260261854598:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.513702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.513716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789257030140142:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.513779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789260261854598:2109] [node 5] ICP22 created new session: [1:7577789260261854635:2109]
2025-11-28T14:52:30.513776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258453754319:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:30.513829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789260261854635:2109] [node 5] ICS09 handshake done sender: [1:7577789260261854607:2114] self: [1:7577789260261854606:0] peer: [5:7577789256938802658:0] socket: 591 qp: -1
2025-11-28T14:52:30.513836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789257030140063:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.513868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789260261854635:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:30.513943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:30.513959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789257030140141:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.514161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789257030140065:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.514220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789258453754243:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24747) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789258453754319:2116] held: no
2025-11-28T14:52:30.514237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789256938802657:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.514297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789258453754243:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.514397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.514404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789260351214691:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.514489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258453754243:2048] [node 1] ICP22 created new session: [4:7577789258453754335:2048]
2025-11-28T14:52:30.514429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789260351214691:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.514502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789256938802661:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-28T14:52:30.514857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258453754335:2048] [node 1] ICS09 handshake done sender: [4:7577789258453754319:2116] self: [4:7577789258453754326:0] peer: [1:7577789260261854604:0] socket: 572 qp: -1
2025-11-28T14:52:30.514895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258453754335:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:30.515096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789260351214691:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:30.514925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:30.514916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789258388677740:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.515007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.515037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789258388677740:2108] [node 5] ICP22 created new session: [2:7577789258388677777:2108]
2025-11-28T14:52:30.515184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789258388677777:2108] [node 5] ICS09 handshake done sender: [2:7577789258388677749:2113] self: [2:7577789258388677748:0] peer: [5:7577789256938802659:0] socket: 552 qp: -1
2025-11-28T14:52:30.515228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789256938802655:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:30.515207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789258388677777:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:30.515238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.515543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789260261854609:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.515529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789260351214681:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.515568Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789260261854609:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.515591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140146:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.515561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.515594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789260351214681:2109] [node 6] ICP22 created new session: [3:7577789260351214715:2109]
2025-11-28T14:52:30.515641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789260351214715:2109] [node 6] ICS09 handshake done sender: [3:7577789260351214691:2114] self: [3:7577789260351214690:0] peer: [6:7577789257030140145:0] socket: 601 qp: -1
2025-11-28T14:52:30.515693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789260351214715:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:30.515713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:30.516720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789257030140141:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:30.516766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789256938802575:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.516819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.516865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789256938802575:2048] [node 1] ICP22 created new session: [5:7577789256938802662:2048]
2025-11-28T14:52:30.517446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789256938802662:2048] [node 1] ICS09 handshake done sender: [5:7577789256938802655:2116] self: [5:7577789256938802658:0] peer: [1:7577789260261854606:0] socket: 592 qp: -1
2025-11-28T14:52:30.517492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789256938802662:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:30.517541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:30.517553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789260261854609:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:30.517558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789257030140065:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.517623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:30.517647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789257030140065:2050] [node 3] ICP22 created new session: [6:7577789257030140147:2050]
2025-11-28T14:52:30.517690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789257030140147:2050] [node 3] ICS09 handshake done sender: [6:7577789257030140141:2115] self: [6:7577789257030140145:0] peer: [3:7577789260351214690:0] socket: 602 qp: -1
2025-11-28T14:52:30.517805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789256938802575:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:30.517711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789257030140147:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:30.517730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.518157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789256938802577:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.518173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789260261854599:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.518205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789256938802656:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:30.518214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.518240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789260261854599:2110] [node 6] ICP22 created new session: [1:7577789260261854638:2110]
2025-11-28T14:52:30.518269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789260261854638:2110] [node 6] ICS09 handshake done sender: [1:7577789260261854609:2115] self: [1:7577789260261854608:0] peer: [6:7577789257030140144:0] socket: 574 qp: -1
2025-11-28T14:52:30.518290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789260261854638:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:30.518301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789257030140143:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.518306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:30.519564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789256938802576:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.519600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.519632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789256938802576:2049] [node 2] ICP22 created new session: [5:7577789256938802666:2049]
2025-11-28T14:52:30.519808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789256938802666:2049] [node 2] ICS09 handshake done sender: [5:7577789256938802656:2117] self: [5:7577789256938802659:0] peer: [2:7577789258388677748:0] socket: 593 qp: -1
2025-11-28T14:52:30.519841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789256938802666:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:30.519869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:30.519845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789260351214689:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.519877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789260351214689:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.520757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789256938802576:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: 2025-11-28T14:52:30.521276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789257030140064:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-28T14:52:30.521381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140149:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.521389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789260351214689:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.521765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140150:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.521881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789260351214680:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.521884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789257030140063:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24747) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789257030140142:2116] held: no
2025-11-28T14:52:30.521944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.522003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789260351214680:2108] [node 5] ICP22 created new session: [3:7577789260351214718:2108]
2025-11-28T14:52:30.522051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789260351214718:2108] [node 5] ICS09 handshake done sender: [3:7577789260351214689:2113] self: [3:7577789260351214688:0] peer: [5:7577789256938802664:0] socket: 585 qp: -1
2025-11-28T14:52:30.522086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789260351214718:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:30.522127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:30.522170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789257030140142:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:30.522181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789258388677751:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.522215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789258388677751:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.522840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789257030140150:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.522819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789256938802665:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-28T14:52:30.522825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258453754339:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28706
2025-11-28T14:52:30.522882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789257030140063:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.522910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.522938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789257030140063:2048] [node 1] ICP22 created new session: [6:7577789257030140153:2048]
2025-11-28T14:52:30.522968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789257030140153:2048] [node 1] ICS09 handshake done sender: [6:7577789257030140142:2116] self: [6:7577789257030140144:0] peer: [1:7577789260261854608:0] socket: 580 qp: -1
2025-11-28T14:52:30.522994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789257030140153:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:30.523017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:30.523099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789258388677751:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:30.523380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789256938802665:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.523693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789258388677741:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.523738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258453754340:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28706
2025-11-28T14:52:30.523755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.523808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789258388677741:2109] [node 6] ICP22 created new session: [2:7577789258388677780:2109]
2025-11-28T14:52:30.523853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789258388677780:2109] [node 6] ICS09 handshake done sender: [2:7577789258388677751:2114] self: [2:7577789258388677750:0] peer: [6:7577789257030140151:0] socket: 586 qp: -1
2025-11-28T14:52:30.523882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789257030140127:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.523906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789258388677780:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:30.523944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.525179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789256938802668:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-28T14:52:30.525253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258453754318:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.525280Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789258453754318:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.526049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140155:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.526050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789256938802577:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24838) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789256938802657:2118] held: no
2025-11-28T14:52:30.526113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258453754318:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.526348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789258453754308:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.526336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140156:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.526378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.526367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789256938802657:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:30.526425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258453754308:2109] [node 6] ICP22 created new session: [4:7577789258453754342:2109]
2025-11-28T14:52:30.526469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258453754342:2109] [node 6] ICS09 handshake done sender: [4:7577789258453754318:2115] self: [4:7577789258453754317:0] peer: [6:7577789257030140157:0] socket: 605 qp: -1
2025-11-28T14:52:30.526530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258453754342:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:30.526468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789257030140064:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62949) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789257030140143:2117] held: no
2025-11-28T14:52:30.526560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:30.526669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789257030140143:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:30.526668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789256938802638:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.526871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789257030140064:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.526897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.526906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258453754339:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.526922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789257030140064:2049] [node 2] ICP22 created new session: [6:7577789257030140159:2049]
2025-11-28T14:52:30.526952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789257030140159:2049] [node 2] ICS09 handshake done sender: [6:7577789257030140143:2117] self: [6:7577789257030140151:0] peer: [2:7577789258388677750:0] socket: 587 qp: -1
2025-11-28T14:52:30.526972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789257030140159:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:30.526990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:30.527044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789258453754308:2109] [node 6] ICP18 handshake (actor [4:7577789258453754339:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.527344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140158:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.528530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789257030140127:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789257030140150:2120] held: no
2025-11-28T14:52:30.528653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789257030140150:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:30.528882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789257030140127:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.528917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:30.528953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789257030140127:2107] [node 4] ICP22 created new session: [6:7577789257030140161:2107]
2025-11-28T14:52:30.528992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789257030140161:2107] [node 4] ICS09 handshake done sender: [6:7577789257030140150:2120] self: [6:7577789257030140157:0] peer: [4:7577789258453754317:0] socket: 607 qp: -1
2025-11-28T14:52:30.529020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789257030140161:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:30.529050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.529779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789256938802577:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.529834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:30.529887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789256938802577:2050] [node 3] ICP22 created new session: [5:7577789256938802672:2050]
2025-11-28T14:52:30.529940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789256938802672:2050] [node 3] ICS09 handshake done sender: [5:7577789256938802657:2118] self: [5:7577789256938802664:0] peer: [3:7577789260351214688:0] socket: 600 qp: -1
2025-11-28T14:52:30.529986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789256938802672:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:30.530016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.530577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258453754316:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.530608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789258453754316:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.530812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789257030140156:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.530941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789257030140123:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.531148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789256938802649:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.531181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789256938802649:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.531296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258453754340:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.531634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258453754316:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:30.531839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789258453754307:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.531879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.531908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258453754307:2108] [node 5] ICP22 created new session: [4:7577789258453754346:2108]
2025-11-28T14:52:30.531933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258453754346:2108] [node 5] ICS09 handshake done sender: [4:7577789258453754316:2114] self: [4:7577789258453754315:0] peer: [5:7577789256938802671:0] socket: 550 qp: -1
2025-11-28T14:52:30.531951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258453754346:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:30.531973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.532141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789256938802674:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.532332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789257030140164:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30302
2025-11-28T14:52:30.532744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789256938802649:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:30.533089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789257030140156:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:30.533237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789257030140123:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.533267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.533319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789257030140123:2106] [node 5] ICP22 created new session: [6:7577789257030140165:2106]
2025-11-28T14:52:30.533364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789257030140165:2106] [node 5] ICS09 handshake done sender: [6:7577789257030140156:2122] self: [6:7577789257030140163:0] peer: [5:7577789256938802648:0] socket: 614 qp: -1
2025-11-28T14:52:30.533396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789257030140165:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:30.533424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:30.534095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789256938802639:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.534159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.534229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789256938802639:2109] [node 6] ICP22 created new session: [5:7577789256938802676:2109]
2025-11-28T14:52:30.534287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789256938802676:2109] [node 6] ICS09 handshake done sender: [5:7577789256938802649:2114] self: [5:7577789256938802648:0] peer: [6:7577789257030140163:0] socket: 612 qp: -1
2025-11-28T14:52:30.534321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789256938802676:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:30.534354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:30.535178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789256938802665:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:30.535381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789256938802638:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789256938802665:2121] held: no
2025-11-28T14:52:30.535483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789256938802638:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.535514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:30.535555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789256938802638:2108] [node 4] ICP22 created new session: [5:7577789256938802678:2108]
2025-11-28T14:52:30.535593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789256938802678:2108] [node 4] ICS09 handshake done sender: [5:7577789256938802665:2121] self: [5:7577789256938802671:0] peer: [4:7577789258453754315:0] socket: 554 qp: -1
2025-11-28T14:52:30.535624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789256938802678:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:30.535657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.548548Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:30.548740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789258388677773:2048] [node 1] ICS01 socket: 566 reason# EndOfStream
2025-11-28T14:52:30.548779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789258388677676:2048] [node 1] ICP30 unregister session Session# [2:7577789258388677773:2048] VirtualId# [2:7577789258388677763:0]
2025-11-28T14:52:30.548816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789258388677773:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.548944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.548999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.549461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:30.549620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789260351214703:2048] [node 1] ICS01 socket: 560 reason# EndOfStream
2025-11-28T14:52:30.549648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789260351214616:2048] [node 1] ICP30 unregister session Session# [3:7577789260351214703:2048] VirtualId# [3:7577789260351214700:0]
2025-11-28T14:52:30.549667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789260351214703:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.549749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.549792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.551161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:30.551327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789258453754335:2048] [node 1] ICS01 socket: 572 reason# EndOfStream
2025-11-28T14:52:30.551362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789258453754243:2048] [node 1] ICP30 unregister session Session# [4:7577789258453754335:2048] VirtualId# [4:7577789258453754326:0]
2025-11-28T14:52:30.551387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789258453754335:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.551468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.551526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.551851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:30.552092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789256938802662:2048] [node 1] ICS01 socket: 592 reason# EndOfStream
2025-11-28T14:52:30.552135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789256938802575:2048] [node 1] ICP30 unregister session Session# [5:7577789256938802662:2048] VirtualId# [5:7577789256938802658:0]
2025-11-28T14:52:30.552174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789256938802662:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.552271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.552341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.552427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:30.552561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789257030140153:2048] [node 1] ICS01 socket: 580 reason# EndOfStream
2025-11-28T14:52:30.552607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789257030140063:2048] [node 1] ICP30 unregister session Session# [6:7577789257030140153:2048] VirtualId# [6:7577789257030140144:0]
2025-11-28T14:52:30.552631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789257030140153:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.552703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.552743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.568997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:30.569163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789260351214712:2049] [node 2] ICS01 socket: 582 reason# EndOfStream
2025-11-28T14:52:30.569193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789260351214617:2049] [node 2] ICP30 unregister session Session# [3:7577789260351214712:2049] VirtualId# [3:7577789260351214702:0]
2025-11-28T14:52:30.569217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789260351214712:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.569280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:30.569333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:30.571125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:30.571298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789258453754331:2049] [node 2] ICS01 socket: 578 reason# EndOfStream
2025-11-28T14:52:30.571336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789258453754244:2049] [node 2] ICP30 unregister session Session# [4:7577789258453754331:2049] VirtualId# [4:7577789258453754327:0]
2025-11-28T14:52:30.571368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789258453754331:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.571458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:30.571515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:30.571898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:30.572128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789256938802666:2049] [node 2] ICS07 socket disconnect 593 reason# EndOfStream
2025-11-28T14:52:30.572162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789256938802666:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.572251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:30.572308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789256938802666:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:30.572425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:30.572553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789257030140159:2049] [node 2] ICS01 socket: 587 reason# EndOfStream
2025-11-28T14:52:30.572579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789257030140064:2049] [node 2] ICP30 unregister session Session# [6:7577789257030140159:2049] VirtualId# [6:7577789257030140151:0]
2025-11-28T14:52:30.572603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789257030140159:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.572648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:30.572688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:30.573080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789256938802576:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62949) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.573130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789256938802666:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.573160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789256938802576:2049] [node 2] ICP30 unregister session Session# [5:7577789256938802666:2049] VirtualId# [5:7577789256938802659:0]
2025-11-28T14:52:30.573205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:30.573239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789256938802576:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.573256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.583489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789256938802576:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:30.604261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:30.604471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789258453754333:2050] [node 3] ICS01 socket: 573 reason# EndOfStream
2025-11-28T14:52:30.604511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789258453754245:2050] [node 3] ICP30 unregister session Session# [4:7577789258453754333:2050] VirtualId# [4:7577789258453754325:0]
2025-11-28T14:52:30.604542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789258453754333:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.604639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:30.604717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:30.605144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:30.605382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789256938802672:2050] [node 3] ICS01 socket: 600 reason# EndOfStream
2025-11-28T14:52:30.605433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789256938802577:2050] [node 3] ICP30 unregister session Session# [5:7577789256938802672:2050] VirtualId# [5:7577789256938802664:0]
2025-11-28T14:52:30.605468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789256938802672:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.605571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:30.605632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:30.605691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:30.605811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789257030140147:2050] [node 3] ICS07 socket disconnect 602 reason# EndOfStream
2025-11-28T14:52:30.605832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789257030140147:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.605880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:30.605922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789257030140147:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:30.606458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789257030140065:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24838) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.606492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789257030140147:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.606507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789257030140065:2050] [node 3] ICP30 unregister session Session# [6:7577789257030140147:2050] VirtualId# [6:7577789257030140145:0]
2025-11-28T14:52:30.606539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:30.606559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789257030140065:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24838) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.606566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24838) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.616221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789257030140065:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:30.636351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:30.636628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789256938802678:2108] [node 4] ICS01 socket: 554 reason# EndOfStream
2025-11-28T14:52:30.636708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789256938802638:2108] [node 4] ICP30 unregister session Session# [5:7577789256938802678:2108] VirtualId# [5:7577789256938802671:0]
2025-11-28T14:52:30.636749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789256938802678:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.636851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:30.636898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:30.637211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:30.637376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789257030140161:2107] [node 4] ICS01 socket: 607 reason# EndOfStream
2025-11-28T14:52:30.637407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789257030140127:2107] [node 4] ICP30 unregister session Session# [6:7577789257030140161:2107] VirtualId# [6:7577789257030140157:0]
2025-11-28T14:52:30.637446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789257030140161:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.637534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:30.637568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:30.659746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:30.659905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789257030140165:2106] [node 5] ICS01 socket: 614 reason# EndOfStream
2025-11-28T14:52:30.659938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789257030140123:2106] [node 5] ICP30 unregister session Session# [6:7577789257030140165:2106] VirtualId# [6:7577789257030140163:0]
2025-11-28T14:52:30.659960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789257030140165:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.660035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:30.660085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:30.715839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789256411307674:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:30.715867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789256411307675:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:30.732498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789256736124780:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:30.732500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789256736124779:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.749219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789258904218273:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.749360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789258904218274:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:30.766504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258727503610:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.766543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258727503611:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:30.766587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258727503612:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:30.784559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789260162621039:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.784576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789260162621040:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:30.784614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789260162621041:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:30.802151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260296716259:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:30.802187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260296716261:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:30.802309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260296716260:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:30.881237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260296716320:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:30.881445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258727503673:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:30.881982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260296716323:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:30.882252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789258727503674:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:30.883457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789258727503610:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(37402719ns) {Configure:104(37399707ns) {ProcessConfigured:135(66969ns) {StartInitialHandshake:157(65265ns) {DropHandshakes:470(2069ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:477(1027ns) {SwitchToState:179(331ns)}}}}}
2025-11-28T14:52:30.884504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789260162621040:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27705864ns) {Configure:104(27700885ns) {ProcessConfigured:135(68826ns) {StartInitialHandshake:157(67381ns) {DropHandshakes:470(973ns) {DropIncomingHandshake:442(277ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:477(2776ns) {SwitchToState:179(280ns)}}}}}
2025-11-28T14:52:30.884538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789260162621039:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27697758ns) {Configure:104(27689218ns) {ProcessConfigured:135(59261ns) {StartInitialHandshake:157(57301ns) {DropHandshakes:470(1195ns) {DropIncomingHandshake:442(280ns) DropOutgoingHandshake:457(364ns)} GenerateSessionVirtualId:988(698ns) PrepareNewSessionHandshake:477(608ns) {SwitchToState:179(249ns)}}}}}
2025-11-28T14:52:30.884983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789260162621107:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:30.884999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789260162621108:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:30.885048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789256411307675:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45039649ns) {Configure:104(45036215ns) {ProcessConfigured:135(192774ns) {StartInitialHandshake:157(190609ns) {DropHandshakes:470(66993ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:477(1243ns) {SwitchToState:179(515ns)}}}}}
2025-11-28T14:52:30.885183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789256411307674:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44177665ns) {Configure:104(44172320ns) {ProcessConfigured:135(131341ns) {StartInitialHandshake:157(129645ns) {DropHandshakes:470(3354ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(2295ns)} GenerateSessionVirtualId:988(857ns) PrepareNewSessionHandshake:477(851ns) {SwitchToState:179(373ns)}}}}}
2025-11-28T14:52:30.885118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789256736124780:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43865092ns) {Configure:104(43862973ns) {ProcessConfigured:135(153879ns) {StartInitialHandshake:157(152753ns) {DropHandshakes:470(1037ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(518ns) PrepareNewSessionHandshake:477(604ns) {SwitchToState:179(223ns)}}}}}
2025-11-28T14:52:30.885192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789256736124779:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44018274ns) {Configure:104(44014896ns) {ProcessConfigured:135(58238ns) {StartInitialHandshake:157(52602ns) {DropHandshakes:470(850ns) {DropIncomingHandshake:442(207ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(439ns) PrepareNewSessionHandshake:477(724ns) {SwitchToState:179(286ns)}}}}}
2025-11-28T14:52:30.885383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789256411307742:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:30.885399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789256736124847:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:30.885445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789256736124848:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:30.885606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789256411307743:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:30.885666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789256736124849:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:30.885717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789256411307744:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:30.886128Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789258904218273:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35444895ns) {Configure:104(35438409ns) {ProcessConfigured:135(195700ns) {StartInitialHandshake:157(194058ns) {DropHandshakes:470(1130ns) {DropIncomingHandshake:442(175ns) DropOutgoingHandshake:457(519ns)} GenerateSessionVirtualId:988(408ns) PrepareNewSessionHandshake:477(868ns) {SwitchToState:179(466ns)}}}}}
2025-11-28T14:52:30.886335Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789258904218274:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(42647154ns) {Configure:104(35458760ns) {ProcessConfigured:135(148093ns) {StartInitialHandshake:157(147137ns) {DropHandshakes:470(789ns) {DropIncomingHandshake:442(231ns) DropOutgoingHandshake:457(138ns)} GenerateSessionVirtualId:988(353ns) PrepareNewSessionHandshake:477(672ns) {SwitchToState:179(302ns)}}}}}
2025-11-28T14:52:30.886651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789258904218340:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:30.886841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789258904218341:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:30.886891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789258904218342:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.889426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789260162621115:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4654
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.890024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789260162621116:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4654
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.893866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258727503688:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59715
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.895250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789256736124854:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39761
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.895574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789260162621116:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.895445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789256736124855:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39761
2025-11-28T14:52:30.895880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789256736124856:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39761
2025-11-28T14:52:30.896290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789256736124855:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.896316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258727503688:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.896403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789256736124854:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.896635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789260162621115:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.896633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789256736124856:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.896747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789260162621107:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.896756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789256411307749:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6437
2025-11-28T14:52:30.896803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789256411307750:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6437
2025-11-28T14:52:30.896889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789258904218349:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42578
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:30.896927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789256411307751:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6437
2025-11-28T14:52:30.896944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258727503689:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59715
2025-11-28T14:52:30.896982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258727503682:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.897557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789258904218352:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42578
2025-11-28T14:52:30.897010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789258727503682:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.898568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307750:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.898968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307751:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.898560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789256736124862:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39761
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.899010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307749:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.899180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789256411307756:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6437
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.899232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258727503691:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59715
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.899385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258727503682:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:30.899446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789256411307757:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6437
2025-11-28T14:52:30.899537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789260162621122:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4654
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.899893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789258727503673:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.900687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789260162621123:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4654
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.901336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307756:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.900704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.901077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258727503673:2108] [node 5] ICP22 created new session: [4:7577789258727503693:2108]
2025-11-28T14:52:30.901899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789256736124863:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39761
2025-11-28T14:52:30.901282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258727503693:2108] [node 5] ICS09 handshake done sender: [4:7577789258727503682:2113] self: [4:7577789258727503681:0] peer: [5:7577789260162621121:0] socket: 553 qp: -1
2025-11-28T14:52:30.902677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789256736124862:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.902268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789258904218349:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.901378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258727503693:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:30.903326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789260162621107:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789260162621115:2115] held: no
2025-11-28T14:52:30.904292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789258904218274:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.901582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:30.904604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789256736124845:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.904673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789260162621115:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:30.904640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789256736124845:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.904759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307757:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.905557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789260162621107:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.905578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789256736124863:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.905602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:30.905630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789260162621107:2108] [node 4] ICP22 created new session: [5:7577789260162621124:2108]
2025-11-28T14:52:30.905620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258727503689:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.905659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789260162621124:2108] [node 4] ICS09 handshake done sender: [5:7577789260162621115:2115] self: [5:7577789260162621121:0] peer: [4:7577789258727503681:0] socket: 558 qp: -1
2025-11-28T14:52:30.905682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789256736124779:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.905692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789260162621124:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:30.905743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.905729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258727503691:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.905813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789256736124845:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:30.905796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789258904218352:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.905820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789258727503611:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.905804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307741:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.905877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789258727503612:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.905829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789256411307741:2112] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.907086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789256736124780:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.907186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789256736124851:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.907199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:30.907329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789258904218355:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42578
2025-11-28T14:52:30.907202Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789256736124851:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.907358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789256411307741:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:30.907328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789256736124780:2049] [node 3] ICP22 created new session: [2:7577789256736124870:2049]
2025-11-28T14:52:30.907369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789258904218356:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42578
2025-11-28T14:52:30.907383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789256736124870:2049] [node 3] ICS09 handshake done sender: [2:7577789256736124845:2111] self: [2:7577789256736124843:0] peer: [3:7577789258904218354:0] socket: 573 qp: -1
2025-11-28T14:52:30.907403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789256736124870:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:30.907427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.907546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789258904218344:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.907563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789258904218344:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.907583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789260162621122:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.907620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789256411307674:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.907889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.908184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789256411307674:2048] [node 2] ICP22 created new session: [1:7577789256411307765:2048]
2025-11-28T14:52:30.908273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789258904218273:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.908301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789260162621039:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.908222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789256411307765:2048] [node 2] ICS09 handshake done sender: [1:7577789256411307741:2112] self: [1:7577789256411307740:0] peer: [2:7577789256736124869:0] socket: 586 qp: -1
2025-11-28T14:52:30.908263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789256411307765:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:30.908356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789258904218356:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.908294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:30.908538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789258904218349:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:30.908548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789258904218357:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42578
2025-11-28T14:52:30.909114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307746:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.909138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789256411307746:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.909913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258727503698:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59715
2025-11-28T14:52:30.910029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789256736124851:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:30.909180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789258904218274:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:30.910249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307739:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:30.909889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.910281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789258904218274:2049] [node 2] ICP22 created new session: [3:7577789258904218360:2049]
2025-11-28T14:52:30.910281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789256411307739:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.910724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789258904218360:2049] [node 2] ICS09 handshake done sender: [3:7577789258904218349:2114] self: [3:7577789258904218354:0] peer: [2:7577789256736124843:0] socket: 575 qp: -1
2025-11-28T14:52:30.910831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789258904218360:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:30.911127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789256411307746:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.910864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:30.911106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789256736124872:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39761
2025-11-28T14:52:30.911152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789258904218357:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.911144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789256736124779:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9497) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789256736124863:2120] held: no
2025-11-28T14:52:30.911447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258727503689:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.911302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789256736124847:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.911471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789260162621128:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4654
2025-11-28T14:52:30.911496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789258904218274:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:30.911767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789260162621129:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4654
2025-11-28T14:52:30.911458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.911798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789258727503611:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.911506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789256736124847:2107] [node 4] ICP22 created new session: [2:7577789256736124875:2107]
2025-11-28T14:52:30.911800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258727503699:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59715
2025-11-28T14:52:30.911800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789256736124875:2107] [node 4] ICS09 handshake done sender: [2:7577789256736124851:2113] self: [2:7577789256736124850:0] peer: [4:7577789258727503696:0] socket: 578 qp: -1
2025-11-28T14:52:30.911877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.911829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789256736124875:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:30.912028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789258904218344:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:30.912202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789260162621122:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:30.912195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789256411307743:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.911998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258727503700:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59715
2025-11-28T14:52:30.912027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.912245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.912164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258727503611:2049] [node 2] ICP22 created new session: [4:7577789258727503701:2049]
2025-11-28T14:52:30.913131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789260162621039:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.912265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789256411307739:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:30.912236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258727503701:2049] [node 2] ICS09 handshake done sender: [4:7577789258727503689:2116] self: [4:7577789258727503696:0] peer: [2:7577789256736124850:0] socket: 579 qp: -1
2025-11-28T14:52:30.912432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789256411307743:2108] [node 5] ICP22 created new session: [1:7577789256411307770:2108]
2025-11-28T14:52:30.913361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.913040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789256411307770:2108] [node 5] ICS09 handshake done sender: [1:7577789256411307746:2113] self: [1:7577789256411307745:0] peer: [5:7577789260162621127:0] socket: 591 qp: -1
2025-11-28T14:52:30.913403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789260162621039:2048] [node 1] ICP22 created new session: [5:7577789260162621130:2048]
2025-11-28T14:52:30.912414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258727503701:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:30.913891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789256736124863:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:30.913341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789256411307770:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:30.912953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:30.913778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789260162621130:2048] [node 1] ICS09 handshake done sender: [5:7577789260162621122:2118] self: [5:7577789260162621127:0] peer: [1:7577789256411307745:0] socket: 592 qp: -1
2025-11-28T14:52:30.913375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:30.913880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789260162621130:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:30.914308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789256411307675:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.914019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:30.914837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:30.914851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789258904218340:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.914868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789256411307675:2049] [node 3] ICP22 created new session: [1:7577789256411307772:2049]
2025-11-28T14:52:30.914943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789256411307772:2049] [node 3] ICS09 handshake done sender: [1:7577789256411307739:2111] self: [1:7577789256411307738:0] peer: [3:7577789258904218358:0] socket: 581 qp: -1
2025-11-28T14:52:30.914971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789256411307772:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:30.915208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:30.915050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.915301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789258904218340:2107] [node 4] ICP22 created new session: [3:7577789258904218366:2107]
2025-11-28T14:52:30.915348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789258904218366:2107] [node 4] ICS09 handshake done sender: [3:7577789258904218344:2112] self: [3:7577789258904218343:0] peer: [4:7577789258727503697:0] socket: 588 qp: -1
2025-11-28T14:52:30.915381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789258904218366:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:30.915407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.915745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258727503691:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:30.915780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789256736124779:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.916793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789258727503612:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21158) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789258727503691:2117] held: no
2025-11-28T14:52:30.915826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.916951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789258904218363:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42578
2025-11-28T14:52:30.916127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789256736124779:2048] [node 1] ICP22 created new session: [2:7577789256736124878:2048]
2025-11-28T14:52:30.917114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789258727503612:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.916627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789256736124878:2048] [node 1] ICS09 handshake done sender: [2:7577789256736124863:2120] self: [2:7577789256736124869:0] peer: [1:7577789256411307740:0] socket: 587 qp: -1
2025-11-28T14:52:30.916761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789256736124878:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.916796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.917270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:30.917671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716337:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
2025-11-28T14:52:30.917368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258727503612:2050] [node 3] ICP22 created new session: [4:7577789258727503704:2050]
2025-11-28T14:52:30.917803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789260162621133:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4654
2025-11-28T14:52:30.917437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258727503704:2050] [node 3] ICS09 handshake done sender: [4:7577789258727503691:2117] self: [4:7577789258727503697:0] peer: [3:7577789258904218343:0] socket: 589 qp: -1
2025-11-28T14:52:30.917468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258727503704:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:30.917499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.917701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258727503700:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.918762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789258904218273:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9497) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789258904218352:2116] held: no
2025-11-28T14:52:30.918975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789258904218352:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:30.918870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716338:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.919006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789260162621128:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.919054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260296716337:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.919063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789258727503610:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.919138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789260162621040:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.919075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716339:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
2025-11-28T14:52:30.919412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716340:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.919706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260296716320:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.919686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789256736124853:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.919729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307748:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.920691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789260162621119:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.919715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789256736124853:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.919768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789256411307748:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.919891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789258904218273:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.920723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789260162621119:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.919851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716341:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
2025-11-28T14:52:30.921531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.922012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789260162621119:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:30.921571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789258904218273:2048] [node 1] ICP22 created new session: [3:7577789258904218369:2048]
2025-11-28T14:52:30.922099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789256736124853:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:30.920676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260296716338:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.922034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260296716323:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.922165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789258904218369:2048] [node 1] ICS09 handshake done sender: [3:7577789258904218352:2116] self: [3:7577789258904218358:0] peer: [1:7577789256411307738:0] socket: 584 qp: -1
2025-11-28T14:52:30.922212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789258904218369:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:30.922632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789258727503684:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.922233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260296716339:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.922352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:30.922682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789260162621108:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.922363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789256411307748:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:30.922663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789258727503684:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.923109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789256736124848:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.923169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.922432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260296716259:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.923430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789256736124848:2108] [node 5] ICP22 created new session: [2:7577789256736124881:2108]
2025-11-28T14:52:30.923422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260296716341:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.923490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789256411307742:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.923475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789256736124881:2108] [node 5] ICS09 handshake done sender: [2:7577789256736124853:2114] self: [2:7577789256736124852:0] peer: [5:7577789260162621134:0] socket: 603 qp: -1
2025-11-28T14:52:30.923453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.923570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789256736124881:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:30.923548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-28T14:52:30.923603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.02025-11-28T14:52:30.923573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789256411307742:2107] [node 4] ICP22 created new session: [1:7577789256411307776:2107]
2025-11-28T14:52:30.923474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258727503684:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:30.923518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789260162621108:2109] [node 6] ICP22 created new session: [5:7577789260162621137:2109]
2025-11-28T14:52:30.923683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789256411307776:2107] [node 4] ICS09 handshake done sender: [1:7577789256411307748:2114] self: [1:7577789256411307747:0] peer: [4:7577789258727503706:0] socket: 574 qp: -1
2025-11-28T14:52:30.923565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789260162621137:2109] [node 6] ICS09 handshake done sender: [5:7577789260162621119:2117] self: [5:7577789260162621118:0] peer: [6:7577789260296716342:0] socket: 554 qp: -1
2025-11-28T14:52:30.923712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789256411307776:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.923593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789260162621137:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:30.923753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.923992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789258727503674:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.923699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:30.923900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260296716340:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.923991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789258727503707:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59715
2025-11-28T14:52:30.923995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789256411307753:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.924232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789260162621133:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.924049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716344:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
2025-11-28T14:52:30.924087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.924021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789256411307753:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.924376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789260162621136:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4654
2025-11-28T14:52:30.924229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258727503674:2109] [node 6] ICP22 created new session: [4:7577789258727503709:2109]
2025-11-28T14:52:30.924323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258727503709:2109] [node 6] ICS09 handshake done sender: [4:7577789258727503684:2114] self: [4:7577789258727503683:0] peer: [6:7577789260296716343:0] socket: 583 qp: -1
2025-11-28T14:52:30.924411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258727503709:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:30.924443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:30.925798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260296716260:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.925201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789258727503700:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:30.926206Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260296716320:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11794) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789260296716337:2115] held: no
2025-11-28T14:52:30.926395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789258727503610:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.926360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789260162621040:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20891) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789260162621128:2120] held: no
2025-11-28T14:52:30.926522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.926566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789258727503610:2048] [node 1] ICP22 created new session: [4:7577789258727503711:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.926639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789258727503711:2048] [node 1] ICS09 handshake done sender: [4:7577789258727503700:2120] self: [4:7577789258727503706:0] peer: [1:7577789256411307747:0] socket: 606 qp: -1
2025-11-28T14:52:30.926675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789258727503711:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:30.926639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789260162621128:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:30.926707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:30.926656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716346:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
2025-11-28T14:52:30.926715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789260162621041:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.926708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789256411307753:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:30.926842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789260162621040:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.926887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.926949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260296716337:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:30.927118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789260162621040:2049] [node 2] ICP22 created new session: [5:7577789260162621140:2049]
2025-11-28T14:52:30.926936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789258904218346:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.927095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789256736124859:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.927552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789256411307744:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.926969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789258904218346:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.928283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260296716261:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:30.927153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789256736124859:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.928286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789260162621140:2049] [node 2] ICS09 handshake done sender: [5:7577789260162621128:2120] self: [5:7577789260162621134:0] peer: [2:7577789256736124852:0] socket: 605 qp: -1
2025-11-28T14:52:30.928644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.928625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789260162621140:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:30.928667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260296716320:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.928672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:30.928641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789258727503610:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:30.928699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789256411307744:2109] [node 6] ICP22 created new session: [1:7577789256411307779:2109]
2025-11-28T14:52:30.928714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.929042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789258904218346:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:30.929311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789256411307779:2109] [node 6] ICS09 handshake done sender: [1:7577789256411307753:2118] self: [1:7577789256411307752:0] peer: [6:7577789260296716345:0] socket: 598 qp: -1
2025-11-28T14:52:30.929339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789256411307779:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:30.929383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789258904218351:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.929313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260296716320:2106] [node 5] ICP22 created new session: [6:7577789260296716350:2106]
2025-11-28T14:52:30.929417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:30.929405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789258904218351:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:30.929584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260296716350:2106] [node 5] ICS09 handshake done sender: [6:7577789260296716337:2115] self: [6:7577789260296716342:0] peer: [5:7577789260162621118:0] socket: 562 qp: -1
2025-11-28T14:52:30.929842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260296716350:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:30.929886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:30.929993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260296716338:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:30.930256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789256736124859:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:30.930367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789258904218341:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.930421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:30.930490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789256736124849:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.930457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789258904218341:2108] [node 5] ICP22 created new session: [3:7577789258904218372:2108]
2025-11-28T14:52:30.930859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.930856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789260162621142:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4654
2025-11-28T14:52:30.930919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789256736124849:2109] [node 6] ICP22 created new session: [2:7577789256736124884:2109]
2025-11-28T14:52:30.930870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789258904218372:2108] [node 5] ICS09 handshake done sender: [3:7577789258904218346:2113] self: [3:7577789258904218345:0] peer: [5:7577789260162621139:0] socket: 556 qp: -1
2025-11-28T14:52:30.930996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789256736124884:2109] [node 6] ICS09 handshake done sender: [2:7577789256736124859:2118] self: [2:7577789256736124858:0] peer: [6:7577789260296716347:0] socket: 612 qp: -1
2025-11-28T14:52:30.930975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789258904218372:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:30.931082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789256736124884:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:30.931082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:30.931423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:30.931878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789260162621041:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21158) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789260162621133:2122] held: no
2025-11-28T14:52:30.932549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789260162621133:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:30.932514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260296716323:2107] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.932569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:30.932595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260296716259:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9497) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789260296716339:2117] held: no
2025-11-28T14:52:30.932903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789260162621041:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.932643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260296716323:2107] [node 4] ICP22 created new session: [6:7577789260296716354:2107]
2025-11-28T14:52:30.932938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.932972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789260162621041:2050] [node 3] ICP22 created new session: [5:7577789260162621143:2050]
2025-11-28T14:52:30.932695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260296716354:2107] [node 4] ICS09 handshake done sender: [6:7577789260296716338:2116] self: [6:7577789260296716343:0] peer: [4:7577789258727503683:0] socket: 596 qp: -1
2025-11-28T14:52:30.932723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260296716354:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:30.932804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:30.933010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789258904218351:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:30.932896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716349:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
2025-11-28T14:52:30.933023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789258904218375:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42578
2025-11-28T14:52:30.933020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789260162621143:2050] [node 3] ICS09 handshake done sender: [5:7577789260162621133:2122] self: [5:7577789260162621139:0] peer: [3:7577789258904218345:0] socket: 570 qp: -1
2025-11-28T14:52:30.933052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789260162621143:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:30.933160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789258904218342:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:30.933198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:30.933265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789258904218342:2109] [node 6] ICP22 created new session: [3:7577789258904218376:2109]
2025-11-28T14:52:30.933086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.933494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789258904218376:2109] [node 6] ICS09 handshake done sender: [3:7577789258904218351:2115] self: [3:7577789258904218350:0] peer: [6:7577789260296716348:0] socket: 576 qp: -1
2025-11-28T14:52:30.933538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789258904218376:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:30.933576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:30.934548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260296716260:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20891) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789260296716341:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.934670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260296716339:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:30.934713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716353:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
2025-11-28T14:52:30.935036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260296716259:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.935094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:30.935130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260296716259:2048] [node 1] ICP22 created new session: [6:7577789260296716357:2048]
2025-11-28T14:52:30.935171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260296716357:2048] [node 1] ICS09 handshake done sender: [6:7577789260296716339:2117] self: [6:7577789260296716345:0] peer: [1:7577789256411307752:0] socket: 610 qp: -1
2025-11-28T14:52:30.935200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260296716357:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:30.935229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:30.935581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260296716356:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11886
2025-11-28T14:52:30.935963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260296716341:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:30.936461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260296716260:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.936517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:30.936549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260296716340:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:30.936558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260296716260:2049] [node 2] ICP22 created new session: [6:7577789260296716359:2049]
2025-11-28T14:52:30.936601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260296716359:2049] [node 2] ICS09 handshake done sender: [6:7577789260296716341:2119] self: [6:7577789260296716347:0] peer: [2:7577789256736124858:0] socket: 613 qp: -1
2025-11-28T14:52:30.936628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260296716359:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:30.936659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:30.936977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260296716261:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:30.937029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:30.937076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260296716261:2050] [node 3] ICP22 created new session: [6:7577789260296716361:2050]
2025-11-28T14:52:30.937072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789258904218375:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:30.937122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260296716361:2050] [node 3] ICS09 handshake done sender: [6:7577789260296716340:2118] self: [6:7577789260296716348:0] peer: [3:7577789258904218350:0] socket: 611 qp: -1
2025-11-28T14:52:30.937160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260296716361:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:30.937195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:30.937219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789258904218342:2109] [node 6] ICP18 handshake (actor [3:7577789258904218375:2121]) is too old
2025-11-28T14:52:30.968117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:30.968353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789256736124878:2048] [node 1] ICS01 socket: 587 reason# EndOfStream
2025-11-28T14:52:30.968420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789256736124779:2048] [node 1] ICP30 unregister session Session# [2:7577789256736124878:2048] VirtualId# [2:7577789256736124869:0]
2025-11-28T14:52:30.968459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789256736124878:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.968556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.968657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.969183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:30.969383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789258904218369:2048] [node 1] ICS01 socket: 584 reason# EndOfStream
2025-11-28T14:52:30.969423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789258904218273:2048] [node 1] ICP30 unregister session Session# [3:7577789258904218369:2048] VirtualId# [3:7577789258904218358:0]
2025-11-28T14:52:30.969449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789258904218369:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.969543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.969605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.971708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:30.971840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789258727503711:2048] [node 1] ICS07 socket disconnect 606 reason# EndOfStream
2025-11-28T14:52:30.971877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789258727503711:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.971953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.971997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789258727503711:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:30.972282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:30.972517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789260162621130:2048] [node 1] ICS07 socket disconnect 592 reason# EndOfStream
2025-11-28T14:52:30.972561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789260162621130:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.972650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.972572Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789258727503610:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9497) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.972706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789260162621130:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:30.972665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789258727503711:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.972684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789258727503610:2048] [node 1] ICP30 unregister session Session# [4:7577789258727503711:2048] VirtualId# [4:7577789258727503706:0]
2025-11-28T14:52:30.972730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.972755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789258727503610:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9497) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.972765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9497) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.972930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:30.973217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260296716357:2048] [node 1] ICS01 socket: 610 reason# EndOfStream
2025-11-28T14:52:30.973271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260296716259:2048] [node 1] ICP30 unregister session Session# [6:7577789260296716357:2048] VirtualId# [6:7577789260296716345:0]
2025-11-28T14:52:30.973308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260296716357:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.973390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:30.973361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789260162621039:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9497) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.973468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.973486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789260162621130:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.973533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789260162621039:2048] [node 1] ICP30 unregister session Session# [5:7577789260162621130:2048] VirtualId# [5:7577789260162621127:0]
2025-11-28T14:52:30.973579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:30.973609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789260162621039:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9497) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.973635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9497) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.982870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789258727503610:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:30.983820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789260162621039:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:30.998047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:30.998211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789258904218360:2049] [node 2] ICS07 socket disconnect 575 reason# EndOfStream
2025-11-28T14:52:30.998235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789258904218360:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:30.998332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:30.998379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789258904218360:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:30.999045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789258904218274:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20891) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:30.999144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789258904218360:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:30.999180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789258904218274:2049] [node 2] ICP30 unregister session Session# [3:7577789258904218360:2049] VirtualId# [3:7577789258904218354:0]
2025-11-28T14:52:30.999240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:30.999263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789258904218274:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20891) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:30.999273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20891) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.000227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:31.000348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789258727503701:2049] [node 2] ICS01 socket: 579 reason# EndOfStream
2025-11-28T14:52:31.000416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789258727503611:2049] [node 2] ICP30 unregister session Session# [4:7577789258727503701:2049] VirtualId# [4:7577789258727503696:0]
2025-11-28T14:52:31.000442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789258727503701:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.000499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:31.000538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:31.001060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:31.001171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789260162621140:2049] [node 2] ICS01 socket: 605 reason# EndOfStream
2025-11-28T14:52:31.001196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789260162621040:2049] [node 2] ICP30 unregister session Session# [5:7577789260162621140:2049] VirtualId# [5:7577789260162621134:0]
2025-11-28T14:52:31.001214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789260162621140:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.001257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:31.001286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:31.001947Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:31.002192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260296716359:2049] [node 2] ICS01 socket: 613 reason# EndOfStream
2025-11-28T14:52:31.002244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260296716260:2049] [node 2] ICP30 unregister session Session# [6:7577789260296716359:2049] VirtualId# [6:7577789260296716347:0]
2025-11-28T14:52:31.002284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260296716359:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.002394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:31.002456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:31.009404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789258904218274:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:31.048342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:31.048507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789258727503704:2050] [node 3] ICS01 socket: 589 reason# EndOfStream
2025-11-28T14:52:31.048539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789258727503612:2050] [node 3] ICP30 unregister session Session# [4:7577789258727503704:2050] VirtualId# [4:7577789258727503697:0]
2025-11-28T14:52:31.048563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789258727503704:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.048646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:31.048691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:31.049270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:31.049405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789260162621143:2050] [node 3] ICS01 socket: 570 reason# EndOfStream
2025-11-28T14:52:31.049433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789260162621041:2050] [node 3] ICP30 unregister session Session# [5:7577789260162621143:2050] VirtualId# [5:7577789260162621139:0]
2025-11-28T14:52:31.049450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789260162621143:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.049509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:31.049561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:31.049938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:31.050200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789260296716361:2050] [node 3] ICS07 socket disconnect 611 reason# EndOfStream
2025-11-28T14:52:31.050241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260296716361:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.050342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:31.050404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789260296716361:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:31.051106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260296716261:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21158) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:31.051150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260296716361:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:31.051169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260296716261:2050] [node 3] ICP30 unregister session Session# [6:7577789260296716361:2050] VirtualId# [6:7577789260296716348:0]
2025-11-28T14:52:31.051230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:31.051262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789260296716261:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21158) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.051272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21158) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.060678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789260296716261:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:31.084145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:31.084397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789260162621124:2108] [node 4] ICS01 socket: 558 reason# EndOfStream
2025-11-28T14:52:31.084449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789260162621107:2108] [node 4] ICP30 unregister session Session# [5:7577789260162621124:2108] VirtualId# [5:7577789260162621121:0]
2025-11-28T14:52:31.084482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789260162621124:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.084562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:31.084630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:31.085163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:31.085425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789260296716354:2107] [node 4] ICS07 socket disconnect 596 reason# EndOfStream
2025-11-28T14:52:31.085474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260296716354:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.085593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:31.085659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789260296716354:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:31.086362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260296716323:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17385) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:31.086410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260296716354:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:31.086470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260296716323:2107] [node 4] ICP30 unregister session Session# [6:7577789260296716354:2107] VirtualId# [6:7577789260296716343:0]
2025-11-28T14:52:31.086518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:31.086541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789260296716323:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.086551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.096500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789260296716323:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:31.134069Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:31.134294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260296716350:2106] [node 5] ICS01 socket: 562 reason# EndOfStream
2025-11-28T14:52:31.134349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260296716320:2106] [node 5] ICP30 unregister session Session# [6:7577789260296716350:2106] VirtualId# [6:7577789260296716342:0]
2025-11-28T14:52:31.134407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260296716350:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.134504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:31.134554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:31.192421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789264327534425:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:31.192457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789264327534426:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:31.208395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789264122310547:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.208402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789264122310548:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:31.224780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789262658767466:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.224814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789262658767467:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:31.242067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789261219293646:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.242072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789261219293648:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:31.242080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789261219293647:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:31.258380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789262001164788:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:31.258389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789262001164787:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.258483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789262001164789:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:31.279032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260941140623:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.279075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260941140625:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:31.279190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260941140624:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:31.366989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260941140685:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:31.370094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789264327534488:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:31.369956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789264122310610:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:31.371240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789264122310611:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:31.371426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789262001164852:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:31.371249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260941140687:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:31.371386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789264327534489:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:31.371603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789262658767531:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:31.371687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789264122310612:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:31.371868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789262658767530:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:31.372430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789262001164853:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:31.372732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789264327534490:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:31.372884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789262658767532:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:31.375949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789264327534426:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43241516ns) {Configure:104(43230832ns) {ProcessConfigured:135(293115ns) {StartInitialHandshake:157(290851ns) {DropHandshakes:470(1427ns) {DropIncomingHandshake:442(412ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(499ns) PrepareNewSessionHandshake:477(1889ns) {SwitchToState:179(785ns)}}}}}
2025-11-28T14:52:31.375961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789264327534425:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49264676ns) {Configure:104(49259573ns) {ProcessConfigured:135(333672ns) {StartInitialHandshake:157(331712ns) {DropHandshakes:470(1891ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:477(1064ns) {SwitchToState:179(434ns)}}}}}
2025-11-28T14:52:31.376101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789261219293646:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(41718584ns) {Configure:104(41712636ns) {ProcessConfigured:135(242383ns) {StartInitialHandshake:157(240721ns) {DropHandshakes:470(1098ns) {DropIncomingHandshake:442(329ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(492ns) PrepareNewSessionHandshake:477(615ns) {SwitchToState:179(237ns)}}}}}
2025-11-28T14:52:31.376117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789261219293648:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41070629ns) {Configure:104(41067814ns) {ProcessConfigured:135(254992ns) {StartInitialHandshake:157(250032ns) {DropHandshakes:470(1005ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(316ns) PrepareNewSessionHandshake:477(978ns) {SwitchToState:179(433ns)}}}}}
2025-11-28T14:52:31.376120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789261219293647:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36809511ns) {Configure:104(36806272ns) {ProcessConfigured:135(254120ns) {StartInitialHandshake:157(252670ns) {DropHandshakes:470(1130ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(888ns) {SwitchToState:179(366ns)}}}}}
2025-11-28T14:52:31.376616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789261219293715:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:31.377006Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789262658767466:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44015860ns) {Configure:104(44010273ns) {ProcessConfigured:135(480421ns) {StartInitialHandshake:157(479480ns) {DropHandshakes:470(1016ns) {DropIncomingHandshake:442(268ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(630ns) PrepareNewSessionHandshake:477(987ns) {SwitchToState:179(287ns)}}}}}
2025-11-28T14:52:31.376625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789261219293716:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:31.376919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789262001164789:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39040319ns) {Configure:104(30354588ns) {ProcessConfigured:135(78133ns) {StartInitialHandshake:157(61674ns) {DropHandshakes:470(1317ns) {DropIncomingHandshake:442(458ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(456ns) PrepareNewSessionHandshake:477(667ns) {SwitchToState:179(226ns)}}}}}
2025-11-28T14:52:31.376968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789262001164787:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40995605ns) {Configure:104(40990407ns) {ProcessConfigured:135(103399ns) {StartInitialHandshake:157(101235ns) {DropHandshakes:470(982ns) {DropIncomingHandshake:442(269ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(494ns) PrepareNewSessionHandshake:477(928ns) {SwitchToState:179(443ns)}}}}}
2025-11-28T14:52:31.378230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789264122310547:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49878312ns) {Configure:104(49871800ns) {ProcessConfigured:135(438261ns) {StartInitialHandshake:157(436718ns) {DropHandshakes:470(1194ns) {DropIncomingHandshake:442(256ns) DropOutgoingHandshake:457(462ns)} GenerateSessionVirtualId:988(474ns) PrepareNewSessionHandshake:477(820ns) {SwitchToState:179(378ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.380983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789264327534503:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32605
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.382835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789264327534505:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32605
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.383773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789262658767543:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23106
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.385753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789262658767545:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23106
2025-11-28T14:52:31.387133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789264327534506:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32605
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.387236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789264327534507:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32605
2025-11-28T14:52:31.387325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789262658767546:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23106
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.387394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789261219293720:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41485
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.388009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789262658767549:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23106
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.388230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534503:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.388138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789264122310622:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33906
2025-11-28T14:52:31.388268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789261219293720:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.389843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789264122310620:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33906
2025-11-28T14:52:31.389310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789262658767543:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.389285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789261219293725:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41485
2025-11-28T14:52:31.389857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789264122310623:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33906
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.389509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789261219293646:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.390051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789262658767545:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.390486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789264122310622:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.390312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534496:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.390339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789264327534496:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.390834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789264122310624:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33906
2025-11-28T14:52:31.391183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789264122310620:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.390678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534507:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.392178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789262658767546:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.391164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789264327534508:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32605
2025-11-28T14:52:31.392415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789262658767549:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:31.393224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789262001164867:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12917
::ffff:127.1.0.0
2025-11-28T14:52:31.391675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789264122310630:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33906
2025-11-28T14:52:31.391714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789264122310547:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.391494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534506:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.393158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789262658767466:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.392160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789264122310624:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.394060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789262658767550:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23106
2025-11-28T14:52:31.392410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789264327534496:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:31.393130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534494:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.392424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789264122310630:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.393155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789264327534494:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.394287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789261219293725:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.394243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789264327534488:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.394633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789261219293727:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41485
2025-11-28T14:52:31.394315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534505:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.394637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789261219293648:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.394353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:31.395067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789261219293646:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789261219293720:2113] held: no
2025-11-28T14:52:31.394575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789264327534488:2107] [node 4] ICP22 created new session: [1:7577789264327534514:2107]
2025-11-28T14:52:31.396174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789264122310623:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.396129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789262658767538:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.396171Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789262658767538:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.394653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789264327534514:2107] [node 4] ICS09 handshake done sender: [1:7577789264327534496:2112] self: [1:7577789264327534495:0] peer: [4:7577789261219293726:0] socket: 563 qp: -1
2025-11-28T14:52:31.394678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789264327534514:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:31.397522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789262658767550:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.397920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789262658767467:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.394683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534493:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.397909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789261219293720:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:31.398494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789264122310616:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.394699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789264327534493:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.397977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789262658767538:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:31.398525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789264122310616:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.398524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789262001164867:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.398564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789261219293646:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.394733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:31.398650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789264122310635:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33906
2025-11-28T14:52:31.399295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789262658767531:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.400275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:31.398976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789262001164788:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.398612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:31.396491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789264327534494:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:31.398914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789261219293646:2048] [node 1] ICP22 created new session: [4:7577789261219293731:2048]
2025-11-28T14:52:31.398124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789264327534425:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.400832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789262658767531:2107] [node 4] ICP22 created new session: [3:7577789262658767558:2107]
2025-11-28T14:52:31.398167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:31.400030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789261219293731:2048] [node 1] ICS09 handshake done sender: [4:7577789261219293720:2113] self: [4:7577789261219293726:0] peer: [1:7577789264327534495:0] socket: 565 qp: -1
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.400066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789261219293731:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:31.398341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789264327534425:2048] [node 2] ICP22 created new session: [1:7577789264327534518:2048]
2025-11-28T14:52:31.401098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789262658767558:2107] [node 4] ICS09 handshake done sender: [3:7577789262658767538:2113] self: [3:7577789262658767537:0] peer: [4:7577789261219293728:0] socket: 578 qp: -1
2025-11-28T14:52:31.400871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789264122310618:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.402249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789262001164868:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12917
2025-11-28T14:52:31.400893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789264122310622:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:31.400638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:31.398507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789264327534518:2048] [node 2] ICS09 handshake done sender: [1:7577789264327534494:2111] self: [1:7577789264327534491:0] peer: [2:7577789264122310631:0] socket: 562 qp: -1
2025-11-28T14:52:31.401155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789262658767558:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.400901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789264122310618:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.402480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789261219293730:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41485
2025-11-28T14:52:31.402873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789264122310547:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.398537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789264327534518:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:31.401191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:31.403684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140702:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
2025-11-28T14:52:31.398904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:31.403508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:31.404120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789261219293648:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16986) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789261219293725:2116] held: no
2025-11-28T14:52:31.403167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789262658767557:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23106
2025-11-28T14:52:31.403740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789264122310547:2048] [node 1] ICP22 created new session: [2:7577789264122310639:2048]
2025-11-28T14:52:31.398911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534508:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.403757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789264122310616:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:31.401148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789264327534493:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:31.404769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789262658767543:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.404307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789264122310639:2048] [node 1] ICS09 handshake done sender: [2:7577789264122310622:2115] self: [2:7577789264122310631:0] peer: [1:7577789264327534491:0] socket: 575 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.405667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789262001164868:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.405504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789262658767466:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.405990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789261219293725:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:31.403440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789264327534426:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.406067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789262001164871:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12917
2025-11-28T14:52:31.404341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789264122310639:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:31.406000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:31.404304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789264327534425:2048] [node 2] ICP18 handshake (actor [1:7577789264327534508:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.406139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789262001164852:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.406941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789261219293648:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.405303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:31.404759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:31.406556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789262001164870:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12917
2025-11-28T14:52:31.405876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789264122310548:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.406110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789262658767466:2048] [node 1] ICP22 created new session: [3:7577789262658767562:2048]
2025-11-28T14:52:31.406144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:31.407302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789261219293722:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.404808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789264327534426:2049] [node 3] ICP22 created new session: [1:7577789264327534522:2049]
2025-11-28T14:52:31.406271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789264122310548:2049] [node 3] ICP22 created new session: [2:7577789264122310641:2049]
2025-11-28T14:52:31.405289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789264327534522:2049] [node 3] ICS09 handshake done sender: [1:7577789264327534493:2110] self: [1:7577789264327534492:0] peer: [3:7577789262658767551:0] socket: 560 qp: -1
2025-11-28T14:52:31.407108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789264122310641:2049] [node 3] ICS09 handshake done sender: [2:7577789264122310616:2110] self: [2:7577789264122310615:0] peer: [3:7577789262658767556:0] socket: 593 qp: -1
2025-11-28T14:52:31.406259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789262658767562:2048] [node 1] ICS09 handshake done sender: [3:7577789262658767543:2114] self: [3:7577789262658767551:0] peer: [1:7577789264327534492:0] socket: 566 qp: -1
2025-11-28T14:52:31.405517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789264327534522:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:31.407328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789261219293722:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.407259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789264122310641:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.407278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:31.406253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:31.409742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789262001164871:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.406288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789262658767562:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:31.408435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.410649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260941140702:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.406696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:31.408462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789261219293648:2050] [node 3] ICP22 created new session: [4:7577789261219293734:2050]
2025-11-28T14:52:31.411288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789262001164789:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.409298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789264122310618:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:31.411724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260941140623:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.407592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789262658767561:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23106
2025-11-28T14:52:31.408490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789261219293734:2050] [node 3] ICS09 handshake done sender: [4:7577789261219293725:2116] self: [4:7577789261219293728:0] peer: [3:7577789262658767537:0] socket: 584 qp: -1
2025-11-28T14:52:31.411424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789262001164873:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12917
2025-11-28T14:52:31.411308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789264122310610:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.411637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:31.411858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789264122310610:2108] [node 5] ICP22 created new session: [2:7577789264122310645:2108]
2025-11-28T14:52:31.411846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140703:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
2025-11-28T14:52:31.412525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534502:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.412557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789264327534502:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.412028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789262658767467:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789262658767550:2120] held: no
2025-11-28T14:52:31.412515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789264122310645:2108] [node 5] ICS09 handshake done sender: [2:7577789264122310618:2112] self: [2:7577789264122310614:0] peer: [5:7577789262001164869:0] socket: 596 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.412360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260941140703:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.413974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789264327534502:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:31.413177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789262658767544:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.408727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789261219293734:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:31.413354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789262001164867:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:31.412543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789264122310645:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:31.408748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:31.412793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260941140687:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.414559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789262001164874:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12917
2025-11-28T14:52:31.413198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789262658767544:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.412723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:31.409381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789261219293722:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:31.410998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789261219293715:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.413325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789262658767550:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:31.415926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789264327534490:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.411476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:31.415040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789262001164788:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.415079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:31.416090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:31.415539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789262001164788:2049] [node 2] ICP22 created new session: [5:7577789262001164876:2049]
2025-11-28T14:52:31.415915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789262658767467:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.411767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789261219293715:2108] [node 5] ICP22 created new session: [4:7577789261219293737:2108]
2025-11-28T14:52:31.416311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789264327534490:2109] [node 6] ICP22 created new session: [1:7577789264327534525:2109]
2025-11-28T14:52:31.416049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789262001164876:2049] [node 2] ICS09 handshake done sender: [5:7577789262001164867:2117] self: [5:7577789262001164869:0] peer: [2:7577789264122310614:0] socket: 598 qp: -1
2025-11-28T14:52:31.416460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.416924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789264327534525:2109] [node 6] ICS09 handshake done sender: [1:7577789264327534502:2113] self: [1:7577789264327534500:0] peer: [6:7577789260941140704:0] socket: 608 qp: -1
2025-11-28T14:52:31.412152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789261219293737:2108] [node 5] ICS09 handshake done sender: [4:7577789261219293722:2114] self: [4:7577789261219293721:0] peer: [5:7577789262001164872:0] socket: 558 qp: -1
2025-11-28T14:52:31.416278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789262001164876:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:31.416501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789262658767467:2049] [node 2] ICP22 created new session: [3:7577789262658767566:2049]
2025-11-28T14:52:31.417451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140707:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.412195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789261219293737:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:31.418123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789264327534525:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:31.416467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:31.416687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789262658767566:2049] [node 2] ICS09 handshake done sender: [3:7577789262658767550:2120] self: [3:7577789262658767556:0] peer: [2:7577789264122310615:0] socket: 597 qp: -1
2025-11-28T14:52:31.418177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:31.418104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140706:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
2025-11-28T14:52:31.412240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:31.416760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789262658767566:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:31.416860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789262001164870:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.413380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789261219293724:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.416897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789262658767544:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:31.417002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789262001164868:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:31.413400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789261219293724:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.415434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789261219293724:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:31.417009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:31.416401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789261219293716:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.419014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789262001164852:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.418762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789262658767530:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.416776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:31.419148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:31.419235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260941140623:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789260941140702:2116] held: no
2025-11-28T14:52:31.419176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:31.416977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789261219293716:2109] [node 6] ICP22 created new session: [4:7577789261219293741:2109]
2025-11-28T14:52:31.419189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789262658767530:2108] [node 5] ICP22 created new session: [3:7577789262658767568:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.418146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789261219293741:2109] [node 6] ICS09 handshake done sender: [4:7577789261219293724:2115] self: [4:7577789261219293723:0] peer: [6:7577789260941140705:0] socket: 572 qp: -1
2025-11-28T14:52:31.419346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789262001164852:2108] [node 4] ICP22 created new session: [5:7577789262001164879:2108]
No RDMA context found for GID: 2025-11-28T14:52:31.419778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789262001164879:2108] [node 4] ICS09 handshake done sender: [5:7577789262001164868:2118] self: [5:7577789262001164872:0] peer: [4:7577789261219293721:0] socket: 606 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:31.418646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789261219293741:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:31.419980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789262001164879:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:31.419805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260941140707:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.419492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789262658767568:2108] [node 5] ICS09 handshake done sender: [3:7577789262658767544:2115] self: [3:7577789262658767542:0] peer: [5:7577789262001164875:0] socket: 612 qp: -1
2025-11-28T14:52:31.418703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:31.420015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:31.419970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260941140702:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:31.419017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789261219293740:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41485
2025-11-28T14:52:31.419442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789261219293740:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.419565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789262658767568:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:31.419973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140708:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
2025-11-28T14:52:31.420249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789264122310621:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.420298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789262001164787:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.419704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789261219293647:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.420006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140709:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
2025-11-28T14:52:31.419711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:31.420273Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789264122310621:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.421178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260941140623:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.421189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260941140624:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.421246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:31.421247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260941140703:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:31.421474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789264122310621:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:31.421333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260941140623:2048] [node 1] ICP22 created new session: [6:7577789260941140712:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.421365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260941140712:2048] [node 1] ICS09 handshake done sender: [6:7577789260941140702:2116] self: [6:7577789260941140704:0] peer: [1:7577789264327534500:0] socket: 609 qp: -1
2025-11-28T14:52:31.422562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789262001164789:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16986) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789262001164871:2120] held: no
2025-11-28T14:52:31.421504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789264122310626:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.422065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789264327534504:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.421524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789264122310626:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.422877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789264122310611:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.423391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789262001164878:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12917
2025-11-28T14:52:31.422098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789264327534504:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.421386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260941140712:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:31.423397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:31.421440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.423823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789264122310611:2107] [node 4] ICP22 created new session: [2:7577789264122310648:2107]
2025-11-28T14:52:31.421945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260941140687:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.423683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:31.424062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789264327534504:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:31.423968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789264122310648:2107] [node 4] ICS09 handshake done sender: [2:7577789264122310621:2114] self: [2:7577789264122310619:0] peer: [4:7577789261219293743:0] socket: 573 qp: -1
2025-11-28T14:52:31.423940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260941140687:2108] [node 4] ICP22 created new session: [6:7577789260941140715:2108]
2025-11-28T14:52:31.424198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789261219293744:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41485
2025-11-28T14:52:31.423994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260941140715:2108] [node 4] ICS09 handshake done sender: [6:7577789260941140703:2117] self: [6:7577789260941140705:0] peer: [4:7577789261219293723:0] socket: 595 qp: -1
2025-11-28T14:52:31.423989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789264122310648:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.424013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:31.424441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789264327534489:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.424134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260941140715:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:31.424700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:31.424090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789264122310626:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:31.424165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:31.425498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789264327534489:2108] [node 5] ICP22 created new session: [1:7577789264327534528:2108]
2025-11-28T14:52:31.425905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140714:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
2025-11-28T14:52:31.425980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789262001164871:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.426060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260941140709:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.426110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789264122310612:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.426028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789264327534528:2108] [node 5] ICS09 handshake done sender: [1:7577789264327534504:2115] self: [1:7577789264327534501:0] peer: [5:7577789262001164881:0] socket: 568 qp: -1
2025-11-28T14:52:31.426034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789261219293647:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789261219293740:2119] held: no
2025-11-28T14:52:31.426066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789264327534528:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:31.426722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789261219293740:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:31.426133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:31.426661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789262001164883:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12917
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.426815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789262001164789:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.426172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:31.426860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789261219293647:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.426220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789264122310612:2109] [node 6] ICP22 created new session: [2:7577789264122310651:2109]
2025-11-28T14:52:31.427261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:31.426622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789264122310651:2109] [node 6] ICS09 handshake done sender: [2:7577789264122310626:2118] self: [2:7577789264122310625:0] peer: [6:7577789260941140710:0] socket: 617 qp: -1
2025-11-28T14:52:31.427830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789262001164789:2050] [node 3] ICP22 created new session: [5:7577789262001164885:2050]
2025-11-28T14:52:31.426653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789264122310651:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:31.427284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:31.426906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:31.427921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789262001164884:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12917
2025-11-28T14:52:31.427624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789261219293647:2049] [node 2] ICP22 created new session: [4:7577789261219293745:2049]
2025-11-28T14:52:31.428010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789262001164885:2050] [node 3] ICS09 handshake done sender: [5:7577789262001164871:2120] self: [5:7577789262001164875:0] peer: [3:7577789262658767542:0] socket: 613 qp: -1
2025-11-28T14:52:31.428015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789261219293745:2049] [node 2] ICS09 handshake done sender: [4:7577789261219293740:2119] self: [4:7577789261219293743:0] peer: [2:7577789264122310619:0] socket: 619 qp: -1
2025-11-28T14:52:31.428048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789261219293745:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:31.428029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789262001164885:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:31.428125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:31.428082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:31.429047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260941140624:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789260941140707:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.429003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789262001164787:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789262001164870:2119] held: no
2025-11-28T14:52:31.429488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789262001164870:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:31.429686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260941140625:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.430009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140717:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
2025-11-28T14:52:31.430014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789262001164787:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.430048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:31.430076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789262001164787:2048] [node 1] ICP22 created new session: [5:7577789262001164887:2048]
2025-11-28T14:52:31.430245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789260941140714:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.430150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789262001164887:2048] [node 1] ICS09 handshake done sender: [5:7577789262001164870:2119] self: [5:7577789262001164881:0] peer: [1:7577789264327534501:0] socket: 610 qp: -1
2025-11-28T14:52:31.430171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789262001164887:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:31.430424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260941140707:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:31.430191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:31.430482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789262658767548:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.430535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260941140685:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:31.430524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789262658767548:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.430570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260941140624:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.430728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789262001164883:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.431211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789262001164865:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:31.430671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:31.431224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789262001164865:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:31.431177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260941140624:2049] [node 2] ICP22 created new session: [6:7577789260941140720:2049]
2025-11-28T14:52:31.431807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260941140720:2049] [node 2] ICS09 handshake done sender: [6:7577789260941140707:2119] self: [6:7577789260941140710:0] peer: [2:7577789264122310625:0] socket: 618 qp: -1
2025-11-28T14:52:31.431826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260941140720:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:31.431900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:31.432142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789262001164865:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.432164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789262658767548:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:31.432254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140722:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
2025-11-28T14:52:31.432303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789262001164853:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.432286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789260941140723:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61566
2025-11-28T14:52:31.432359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789262658767532:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:31.432518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:31.432610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789262001164853:2110] [node 6] ICP22 created new session: [5:7577789262001164891:2110]
2025-11-28T14:52:31.432536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260941140714:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:31.432646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:31.432678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789262001164891:2110] [node 6] ICS09 handshake done sender: [5:7577789262001164865:2115] self: [5:7577789262001164864:0] peer: [6:7577789260941140719:0] socket: 624 qp: -1
2025-11-28T14:52:31.432846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789262001164891:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:31.433185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260941140685:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.432690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789262658767532:2109] [node 6] ICP22 created new session: [3:7577789262658767571:2109]
2025-11-28T14:52:31.432877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:31.433231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:31.433223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789262658767571:2109] [node 6] ICS09 handshake done sender: [3:7577789262658767548:2118] self: [3:7577789262658767547:0] peer: [6:7577789260941140718:0] socket: 622 qp: -1
2025-11-28T14:52:31.433265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260941140685:2106] [node 5] ICP22 created new session: [6:7577789260941140724:2106]
2025-11-28T14:52:31.433260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789262658767571:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:31.433350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789260941140709:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:31.433384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260941140724:2106] [node 5] ICS09 handshake done sender: [6:7577789260941140714:2122] self: [6:7577789260941140719:0] peer: [5:7577789262001164864:0] socket: 625 qp: -1
2025-11-28T14:52:31.433372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:31.433501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260941140724:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:31.433523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:31.434352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260941140625:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16986) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789260941140709:2121] held: no
2025-11-28T14:52:31.434422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260941140625:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:31.434452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:31.434478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260941140625:2050] [node 3] ICP22 created new session: [6:7577789260941140726:2050]
2025-11-28T14:52:31.434519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789260941140726:2050] [node 3] ICS09 handshake done sender: [6:7577789260941140709:2121] self: [6:7577789260941140718:0] peer: [3:7577789262658767547:0] socket: 623 qp: -1
2025-11-28T14:52:31.434565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789260941140726:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:31.434587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:31.474669Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:31.474891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789264122310639:2048] [node 1] ICS07 socket disconnect 575 reason# EndOfStream
2025-11-28T14:52:31.474928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789264122310639:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.475037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:31.475093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789264122310639:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:31.475569Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:31.475793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789262658767562:2048] [node 1] ICS07 socket disconnect 566 reason# EndOfStream
2025-11-28T14:52:31.475832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789262658767562:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.475924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:31.475978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789262658767562:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:31.476812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789264122310547:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:31.476890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789262658767466:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:31.476856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789264122310639:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:31.477153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789262658767562:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:31.477407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789262658767466:2048] [node 1] ICP30 unregister session Session# [3:7577789262658767562:2048] VirtualId# [3:7577789262658767551:0]
2025-11-28T14:52:31.477451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:31.477485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789262658767466:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.477505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.478503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789264122310547:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:31.478555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789264122310639:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:31.478367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:31.478576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789264122310547:2048] [node 1] ICP30 unregister session Session# [2:7577789264122310639:2048] VirtualId# [2:7577789264122310631:0]
2025-11-28T14:52:31.478587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789261219293731:2048] [node 1] ICS01 socket: 565 reason# EndOfStream
2025-11-28T14:52:31.478630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789261219293646:2048] [node 1] ICP30 unregister session Session# [4:7577789261219293731:2048] VirtualId# [4:7577789261219293726:0]
2025-11-28T14:52:31.478607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:31.478662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789261219293731:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.478637Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789264122310547:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.478735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:31.478651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.478801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:31.479156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:31.479299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789262001164887:2048] [node 1] ICS01 socket: 610 reason# EndOfStream
2025-11-28T14:52:31.479327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789262001164787:2048] [node 1] ICP30 unregister session Session# [5:7577789262001164887:2048] VirtualId# [5:7577789262001164881:0]
2025-11-28T14:52:31.479356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789262001164887:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.479413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:31.479461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:31.479752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:31.479880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260941140712:2048] [node 1] ICS01 socket: 609 reason# EndOfStream
2025-11-28T14:52:31.479910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260941140623:2048] [node 1] ICP30 unregister session Session# [6:7577789260941140712:2048] VirtualId# [6:7577789260941140704:0]
2025-11-28T14:52:31.479927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260941140712:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.479982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:31.480053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:31.487642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789262658767466:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:31.531750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:31.531995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789262658767566:2049] [node 2] ICS01 socket: 597 reason# EndOfStream
2025-11-28T14:52:31.532050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789262658767467:2049] [node 2] ICP30 unregister session Session# [3:7577789262658767566:2049] VirtualId# [3:7577789262658767556:0]
2025-11-28T14:52:31.532082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789262658767566:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.532199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:31.532275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:31.534520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:31.534738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789261219293745:2049] [node 2] ICS01 socket: 619 reason# EndOfStream
2025-11-28T14:52:31.534801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789261219293647:2049] [node 2] ICP30 unregister session Session# [4:7577789261219293745:2049] VirtualId# [4:7577789261219293743:0]
2025-11-28T14:52:31.534841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789261219293745:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.534931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:31.534983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:31.535527Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:31.535745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789262001164876:2049] [node 2] ICS07 socket disconnect 598 reason# EndOfStream
2025-11-28T14:52:31.535780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789262001164876:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.535857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:31.535905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789262001164876:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:31.536391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:31.536513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260941140720:2049] [node 2] ICS01 socket: 618 reason# EndOfStream
2025-11-28T14:52:31.536545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260941140624:2049] [node 2] ICP30 unregister session Session# [6:7577789260941140720:2049] VirtualId# [6:7577789260941140710:0]
2025-11-28T14:52:31.536567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260941140720:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.536644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:31.536689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:31.536653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789262001164788:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29316) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:31.536724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789262001164876:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:31.536777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789262001164788:2049] [node 2] ICP30 unregister session Session# [5:7577789262001164876:2049] VirtualId# [5:7577789262001164869:0]
2025-11-28T14:52:31.536822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:31.536880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789262001164788:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29316) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.536909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29316) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.547042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789262001164788:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:31.593660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:31.593897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789261219293734:2050] [node 3] ICS01 socket: 584 reason# EndOfStream
2025-11-28T14:52:31.593945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789261219293648:2050] [node 3] ICP30 unregister session Session# [4:7577789261219293734:2050] VirtualId# [4:7577789261219293728:0]
2025-11-28T14:52:31.593987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789261219293734:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.594093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:31.594149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:31.594656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:31.594802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789262001164885:2050] [node 3] ICS01 socket: 613 reason# EndOfStream
2025-11-28T14:52:31.594835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789262001164789:2050] [node 3] ICP30 unregister session Session# [5:7577789262001164885:2050] VirtualId# [5:7577789262001164875:0]
2025-11-28T14:52:31.594858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789262001164885:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.594924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:31.594962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:31.595533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:31.595689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260941140726:2050] [node 3] ICS01 socket: 623 reason# EndOfStream
2025-11-28T14:52:31.595722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260941140625:2050] [node 3] ICP30 unregister session Session# [6:7577789260941140726:2050] VirtualId# [6:7577789260941140718:0]
2025-11-28T14:52:31.595740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260941140726:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.595791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:31.595820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:31.652715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:31.652872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789262001164879:2108] [node 4] ICS07 socket disconnect 606 reason# EndOfStream
2025-11-28T14:52:31.652896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789262001164879:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.652963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:31.652997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789262001164879:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:31.653654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789262001164852:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3490) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:31.653753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789262001164879:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:31.653783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789262001164852:2108] [node 4] ICP30 unregister session Session# [5:7577789262001164879:2108] VirtualId# [5:7577789262001164872:0]
2025-11-28T14:52:31.653776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:31.653830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:31.653861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789262001164852:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.653871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.653927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789260941140715:2108] [node 4] ICS07 socket disconnect 595 reason# EndOfStream
2025-11-28T14:52:31.653961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260941140715:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.654014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:31.654042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789260941140715:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:31.654598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260941140687:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3490) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:31.654653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260941140715:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:31.654692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260941140687:2108] [node 4] ICP30 unregister session Session# [6:7577789260941140715:2108] VirtualId# [6:7577789260941140705:0]
2025-11-28T14:52:31.654731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:31.654754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789260941140687:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.654766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.663769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789262001164852:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:31.664770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789260941140687:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:31.714962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:31.715113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789260941140724:2106] [node 5] ICS07 socket disconnect 625 reason# EndOfStream
2025-11-28T14:52:31.715138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789260941140724:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:31.715201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:31.715238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789260941140724:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:31.715804Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260941140685:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30002) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:31.715887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789260941140724:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:31.715923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260941140685:2106] [node 5] ICP30 unregister session Session# [6:7577789260941140724:2106] VirtualId# [6:7577789260941140719:0]
2025-11-28T14:52:31.715964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:31.715989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789260941140685:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:31.716002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:31.805106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789264005972234:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:31.805136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789264005972235:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:31.829998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789262055521283:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:31.830005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789262055521282:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.852470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789264561262335:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.852885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789264561262336:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:31.876197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789263639535335:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.876240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789263639535336:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:31.876304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789263639535337:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:31.900620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789261205399424:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.900651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789261205399425:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:31.900735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789261205399426:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:31.925383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260718275244:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:31.925424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260718275246:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:31.925451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260718275245:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:32.010154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789264561262413:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:32.011990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789260718275305:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:32.010227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789268300939620:2122] [node 5] ICP01 ready to work
2025-11-28T14:52:32.010299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789268856229710:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:32.012113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789265500366801:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:32.013070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789265500366800:2115] [node 6] ICP01 ready to work
2025-11-28T14:52:32.013416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789266350488667:2128] [node 6] ICP01 ready to work
2025-11-28T14:52:32.013676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789268300939621:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:32.013456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789268856229711:2111] [node 6] ICP01 ready to work
2025-11-28T14:52:32.014092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789267934502694:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:32.013581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789266350488668:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:32.014882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789267934502695:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:32.014336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789268300939622:2130] [node 6] ICP01 ready to work
2025-11-28T14:52:32.014421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789265013242605:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:32.015101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789266350488669:2130] [node 5] ICP01 ready to work
2025-11-28T14:52:32.018524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789263639535336:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44628826ns) {Configure:104(44623709ns) {ProcessConfigured:135(146337ns) {StartInitialHandshake:157(144722ns) {DropHandshakes:470(2463ns) {DropIncomingHandshake:442(808ns) DropOutgoingHandshake:457(394ns)} GenerateSessionVirtualId:988(507ns) PrepareNewSessionHandshake:477(1464ns) {SwitchToState:179(752ns)}}}}}
2025-11-28T14:52:32.018580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789263639535337:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44339420ns) {Configure:104(38085284ns) {ProcessConfigured:135(58326ns) {StartInitialHandshake:157(53646ns) {DropHandshakes:470(1578ns) {DropIncomingHandshake:442(474ns) DropOutgoingHandshake:457(272ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:477(945ns) {SwitchToState:179(306ns)}}}}}
2025-11-28T14:52:32.019070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789263639535335:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38370377ns) {Configure:104(38363243ns) {ProcessConfigured:135(307967ns) {StartInitialHandshake:157(306649ns) {DropHandshakes:470(544ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(120ns)} GenerateSessionVirtualId:988(591ns) PrepareNewSessionHandshake:477(520ns) {SwitchToState:179(75ns)}}}}}
2025-11-28T14:52:32.020273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789262055521282:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46885978ns) {Configure:104(46881938ns) {ProcessConfigured:135(425535ns) {StartInitialHandshake:157(423418ns) {DropHandshakes:470(1146ns) {DropIncomingHandshake:442(310ns) DropOutgoingHandshake:457(305ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:477(694ns) {SwitchToState:179(285ns)}}}}}
2025-11-28T14:52:32.020176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789260718275246:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42139039ns) {Configure:104(42135139ns) {ProcessConfigured:135(522017ns) {StartInitialHandshake:157(520047ns) {DropHandshakes:470(1129ns) {DropIncomingHandshake:442(317ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(417ns) PrepareNewSessionHandshake:477(1080ns) {SwitchToState:179(431ns)}}}}}
2025-11-28T14:52:32.020653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789264005972235:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24922939ns) {Configure:104(24910169ns) {ProcessConfigured:135(394876ns) {StartInitialHandshake:157(392778ns) {DropHandshakes:470(1196ns) {DropIncomingHandshake:442(300ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(671ns) PrepareNewSessionHandshake:477(778ns) {SwitchToState:179(286ns)}}}}}
2025-11-28T14:52:32.020740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789261205399424:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(45355340ns) {Configure:104(45349548ns) {ProcessConfigured:135(180572ns) {StartInitialHandshake:157(178897ns) {DropHandshakes:470(1032ns) {DropIncomingHandshake:442(282ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:477(906ns) {SwitchToState:179(447ns)}}}}}
2025-11-28T14:52:32.020261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789260718275245:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(42383776ns) {Configure:104(42381145ns) {ProcessConfigured:135(711832ns) {StartInitialHandshake:157(710351ns) {DropHandshakes:470(2007ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(342ns)} GenerateSessionVirtualId:988(494ns) PrepareNewSessionHandshake:477(289517ns) {SwitchToState:179(891ns)}}}}}
2025-11-28T14:52:32.021096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789264561262335:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47088442ns) {Configure:104(47086004ns) {ProcessConfigured:135(573194ns) {StartInitialHandshake:157(571706ns) {DropHandshakes:470(1091ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(248ns)} GenerateSessionVirtualId:988(754ns) PrepareNewSessionHandshake:477(859ns) {SwitchToState:179(313ns)}}}}}
2025-11-28T14:52:32.020985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789264005972234:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47229881ns) {Configure:104(47224753ns) {ProcessConfigured:135(840040ns) {StartInitialHandshake:157(838565ns) {DropHandshakes:470(956ns) {DropIncomingHandshake:442(295ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(303ns) PrepareNewSessionHandshake:477(719ns) {SwitchToState:179(317ns)}}}}}
2025-11-28T14:52:32.022181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789262055521283:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49108506ns) {Configure:104(49101132ns) {ProcessConfigured:135(1012837ns) {StartInitialHandshake:157(1010809ns) {DropHandshakes:470(1295ns) {DropIncomingHandshake:442(452ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(570ns) PrepareNewSessionHandshake:477(1713ns) {SwitchToState:179(626ns)}}}}}
2025-11-28T14:52:32.022596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789261205399426:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26856983ns) {Configure:104(26851153ns) {ProcessConfigured:135(1333005ns) {StartInitialHandshake:157(1331745ns) {DropHandshakes:470(1001ns) {DropIncomingHandshake:442(176ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(1265ns) {SwitchToState:179(127ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.032759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789267934502709:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28278
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.033297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789267934502710:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28278
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.036011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266350488681:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43340
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.033949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242618:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.034153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789268856229723:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14642
2025-11-28T14:52:32.036710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789267934502711:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28278
2025-11-28T14:52:32.034119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242617:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.037221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266350488683:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43340
2025-11-28T14:52:32.037153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789268856229727:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14642
2025-11-28T14:52:32.037148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789268300939634:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
2025-11-28T14:52:32.038040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789267934502709:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.037913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266350488682:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43340
2025-11-28T14:52:32.038242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789267934502710:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.037163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789268300939636:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
2025-11-28T14:52:32.038326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789267934502711:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.038293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266350488684:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43340
2025-11-28T14:52:32.037479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789268856229723:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.038333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789263639535337:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.038532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789265013242617:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.038571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266350488683:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.038606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789265013242618:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.037738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789268856229728:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14642
2025-11-28T14:52:32.037529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789268300939637:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.038273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939634:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.039710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266350488681:2140] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.038303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789268300939639:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
2025-11-28T14:52:32.038657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939636:2142] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.037940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789268856229726:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14642
2025-11-28T14:52:32.039750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789263639535336:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.039782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260718275245:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.039778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789268300939640:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
2025-11-28T14:52:32.038538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789268856229715:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.039781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260718275244:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.039914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266350488684:2143] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.039793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939637:2143] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.038563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789268856229715:2125] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.039916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266350488682:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.039863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939639:2145] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.038702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789268856229727:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.040266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266350488673:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.040007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939628:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.039711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789268856229726:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.040022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789268300939628:2139] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.039908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789268856229728:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.040274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266350488676:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.040455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939625:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.040276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789262055521282:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.040006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789264561262335:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.040050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789268856229715:2125] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:32.040295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789266350488673:2137] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.040467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789268300939625:2137] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.043555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789264561262413:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.040300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789266350488676:2138] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.043601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.041836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939640:2146] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.044127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266350488673:2137] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:32.043464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939627:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.043638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789264561262413:2107] [node 4] ICP22 created new session: [3:7577789268856229734:2107]
2025-11-28T14:52:32.045064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789268856229734:2107] [node 4] ICS09 handshake done sender: [3:7577789268856229715:2125] self: [3:7577789268856229712:0] peer: [4:7577789267934502712:0] socket: 565 qp: -1
2025-11-28T14:52:32.045222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789263639535337:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12857) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789267934502709:2115] held: no
2025-11-28T14:52:32.045540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266350488676:2138] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:32.046108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242623:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
2025-11-28T14:52:32.043488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789268300939627:2138] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.046330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789266350488667:2128] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:32.043558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789268300939628:2139] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:32.045269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789267934502715:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28278
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.049196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789265500366816:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54072
2025-11-28T14:52:32.045104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789268856229734:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:32.045318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789268300939627:2138] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.046648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.047597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789267934502709:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.045186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.046712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789266350488668:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.048133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789263639535337:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.045333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789268300939622:2130] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.047431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.045347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789268300939625:2137] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.050003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260718275244:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789265013242617:2115] held: no
2025-11-28T14:52:32.048171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.050091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789268856229736:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14642
2025-11-28T14:52:32.045567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.047632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789266350488667:2128] [node 6] ICP22 created new session: [2:7577789266350488693:2128]
2025-11-28T14:52:32.047720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789266350488668:2107] [node 4] ICP22 created new session: [2:7577789266350488695:2107]
2025-11-28T14:52:32.051685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789265500366815:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54072
2025-11-28T14:52:32.050228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789265013242617:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.049381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266350488695:2107] [node 4] ICS09 handshake done sender: [2:7577789266350488676:2138] self: [2:7577789266350488674:0] peer: [4:7577789267934502714:0] socket: 578 qp: -1
2025-11-28T14:52:32.048224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789263639535337:2050] [node 3] ICP22 created new session: [4:7577789267934502717:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.045796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789268300939622:2130] [node 6] ICP22 created new session: [1:7577789268300939649:2130]
2025-11-28T14:52:32.049626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266350488693:2128] [node 6] ICS09 handshake done sender: [2:7577789266350488673:2137] self: [2:7577789266350488671:0] peer: [6:7577789265013242622:0] socket: 582 qp: -1
2025-11-28T14:52:32.051192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242624:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
2025-11-28T14:52:32.049639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266350488695:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:32.051898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260718275244:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.048251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789267934502717:2050] [node 3] ICS09 handshake done sender: [4:7577789267934502709:2115] self: [4:7577789267934502712:0] peer: [3:7577789268856229712:0] socket: 586 qp: -1
2025-11-28T14:52:32.049747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.047569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789268300939649:2130] [node 6] ICS09 handshake done sender: [1:7577789268300939628:2139] self: [1:7577789268300939626:0] peer: [6:7577789265013242621:0] socket: 569 qp: -1
2025-11-28T14:52:32.053622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789268856229728:2133] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.049819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266350488693:2128] [node 6] ICS10 traffic start
2025-11-28T14:52:32.048270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789267934502717:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:32.051937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.047689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789264005972234:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.050043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.053024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260718275245:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19625) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789265013242618:2116] held: no
2025-11-28T14:52:32.054847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789265500366816:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.048296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:32.047743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789264005972235:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.054472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266350488694:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43340
2025-11-28T14:52:32.049622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789267934502716:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28278
2025-11-28T14:52:32.053641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260718275244:2048] [node 1] ICP22 created new session: [6:7577789265013242625:2048]
2025-11-28T14:52:32.054997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789264561262335:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.056285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789265500366815:2132] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.055532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.051189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789267934502711:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:32.047830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789268300939649:2130] [node 6] ICS10 traffic start
2025-11-28T14:52:32.056508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789265500366801:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.054514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789265013242618:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:32.047860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.055561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789264561262335:2048] [node 1] ICP22 created new session: [3:7577789268856229738:2048]
2025-11-28T14:52:32.051998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789263639535336:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.057110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789262055521282:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789266350488684:2143] held: no
2025-11-28T14:52:32.057151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789261205399426:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.054577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789265013242625:2048] [node 1] ICS09 handshake done sender: [6:7577789265013242617:2115] self: [6:7577789265013242621:0] peer: [1:7577789268300939626:0] socket: 581 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.052966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:32.057103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789268856229738:2048] [node 1] ICS09 handshake done sender: [3:7577789268856229728:2133] self: [3:7577789268856229733:0] peer: [1:7577789268300939624:0] socket: 602 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.054777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789265013242625:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.049397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.057735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266350488684:2143] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.054818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: 2025-11-28T14:52:32.059773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789262055521282:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:32.049637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:32.053036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789263639535336:2049] [node 2] ICP22 created new session: [4:7577789267934502720:2049]
2025-11-28T14:52:32.060094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.059921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789265500366819:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54072
2025-11-28T14:52:32.055061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260718275245:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.057496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789268856229738:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.053640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789267934502720:2049] [node 2] ICS09 handshake done sender: [4:7577789267934502711:2117] self: [4:7577789267934502714:0] peer: [2:7577789266350488674:0] socket: 600 qp: -1
2025-11-28T14:52:32.059923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789265500366817:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54072
2025-11-28T14:52:32.060468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789262055521282:2048] [node 1] ICP22 created new session: [2:7577789266350488700:2048]
2025-11-28T14:52:32.061499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789265500366819:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.056341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:32.049729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789264005972235:2049] [node 3] ICP22 created new session: [1:7577789268300939651:2049]
2025-11-28T14:52:32.057642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.053890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789267934502720:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:32.056862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260718275245:2049] [node 2] ICP22 created new session: [6:7577789265013242629:2049]
2025-11-28T14:52:32.062301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266350488700:2048] [node 1] ICS09 handshake done sender: [2:7577789266350488684:2143] self: [2:7577789266350488690:0] peer: [1:7577789268300939623:0] socket: 603 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.062344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266350488700:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.056922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789265013242629:2049] [node 2] ICS09 handshake done sender: [6:7577789265013242618:2116] self: [6:7577789265013242622:0] peer: [2:7577789266350488671:0] socket: 588 qp: -1
2025-11-28T14:52:32.049762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789268300939651:2049] [node 3] ICS09 handshake done sender: [1:7577789268300939625:2137] self: [1:7577789268300939624:0] peer: [3:7577789268856229733:0] socket: 584 qp: -1
2025-11-28T14:52:32.059728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789268856229718:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.049816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789264005972234:2048] [node 2] ICP22 created new session: [1:7577789268300939652:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.055112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.062874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:32.062940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789261205399424:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.049978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789268300939651:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:32.056952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789265013242629:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:32.058179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789267934502703:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.050088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:32.057610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.058209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789267934502703:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.063318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
2025-11-28T14:52:32.063037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789265500366821:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54072
2025-11-28T14:52:32.059768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789268856229718:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.050098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789268300939652:2048] [node 2] ICS09 handshake done sender: [1:7577789268300939627:2138] self: [1:7577789268300939623:0] peer: [2:7577789266350488690:0] socket: 576 qp: -1
2025-11-28T14:52:32.060342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789267934502723:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28278
2025-11-28T14:52:32.063401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789265500366817:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.062409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789264561262335:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.051128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789268300939652:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:32.063037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789268856229739:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14642
2025-11-28T14:52:32.060811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789267934502703:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:32.051473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.063895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789265500366821:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.062911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789267934502694:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.063695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789268856229718:2127] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:32.062984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:32.064886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242632:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
2025-11-28T14:52:32.064188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939630:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.063081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789267934502694:2108] [node 5] ICP22 created new session: [4:7577789267934502726:2108]
2025-11-28T14:52:32.064218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789268300939630:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.064913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789268856229710:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.064913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789265500366822:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54072
2025-11-28T14:52:32.063398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789267934502726:2108] [node 5] ICS09 handshake done sender: [4:7577789267934502703:2113] self: [4:7577789267934502699:0] peer: [5:7577789265500366818:0] socket: 564 qp: -1
2025-11-28T14:52:32.065083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789261205399425:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.065706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789265013242632:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.065733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789268300939630:2140] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:32.064958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:32.063652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789267934502724:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28278
2025-11-28T14:52:32.065160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789268856229739:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.067032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266350488677:2139] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.065327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789265500366801:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30318) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789265500366816:2133] held: no
2025-11-28T14:52:32.066508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789265013242631:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.063852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789267934502726:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:32.066740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260718275246:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.067652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789268300939620:2122] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.065567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789265500366824:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54072
2025-11-28T14:52:32.067079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789266350488677:2139] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.065488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789268856229710:2108] [node 5] ICP22 created new session: [3:7577789268856229743:2108]
2025-11-28T14:52:32.063890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.068111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:32.067100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266350488686:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.065762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789268856229743:2108] [node 5] ICS09 handshake done sender: [3:7577789268856229718:2127] self: [3:7577789268856229717:0] peer: [5:7577789265500366820:0] socket: 617 qp: -1
2025-11-28T14:52:32.068096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789265500366816:2133] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:32.065478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789267934502724:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.067114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789265013242605:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.067117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789266350488686:2144] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.070287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789265500366815:2132] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.066534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789264561262336:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.068215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789268300939638:2144] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.066632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789267934502723:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.070577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789261205399426:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12857) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789265500366815:2132] held: no
2025-11-28T14:52:32.070521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266350488686:2144] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:32.066690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789268856229743:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:32.070640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266350488677:2139] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.066654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789263639535335:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.068065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.070713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789266350488669:2130] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.068226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789268300939620:2122] [node 5] ICP22 created new session: [1:7577789268300939658:2122]
2025-11-28T14:52:32.071243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789265500366801:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.068234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789268300939638:2144] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.071244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:32.068491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789267934502705:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.068203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789268856229725:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.071323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.071294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789266350488669:2130] [node 5] ICP22 created new session: [2:7577789266350488704:2130]
2025-11-28T14:52:32.068228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789268856229725:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.071670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789265500366801:2108] [node 4] ICP22 created new session: [5:7577789265500366828:2108]
2025-11-28T14:52:32.068400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789268300939658:2122] [node 5] ICS09 handshake done sender: [1:7577789268300939630:2140] self: [1:7577789268300939629:0] peer: [5:7577789265500366823:0] socket: 596 qp: -1
2025-11-28T14:52:32.071659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266350488704:2130] [node 5] ICS09 handshake done sender: [2:7577789266350488686:2144] self: [2:7577789266350488685:0] peer: [5:7577789265500366826:0] socket: 620 qp: -1
2025-11-28T14:52:32.068512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789267934502705:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.070704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789268856229725:2130] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:32.071692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266350488704:2130] [node 5] ICS10 traffic start
2025-11-28T14:52:32.072182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789265500366828:2108] [node 4] ICS09 handshake done sender: [5:7577789265500366816:2133] self: [5:7577789265500366818:0] peer: [4:7577789267934502699:0] socket: 595 qp: -1
2025-11-28T14:52:32.070259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789268300939658:2122] [node 5] ICS10 traffic start
2025-11-28T14:52:32.071951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789268856229711:2111] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.071892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789262055521283:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.072292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789261205399426:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.071648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789267934502705:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.073300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242635:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
2025-11-28T14:52:32.071990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.072754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.071929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.072806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789261205399426:2050] [node 3] ICP22 created new session: [5:7577789265500366829:2050]
2025-11-28T14:52:32.072028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789267934502695:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.072813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789268856229711:2111] [node 6] ICP22 created new session: [3:7577789268856229748:2111]
2025-11-28T14:52:32.071963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.074047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789265013242605:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30318) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789265013242631:2120] held: no
2025-11-28T14:52:32.072844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789265500366828:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:32.070604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.073493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789268856229748:2111] [node 6] ICS09 handshake done sender: [3:7577789268856229725:2130] self: [3:7577789268856229724:0] peer: [6:7577789265013242633:0] socket: 572 qp: -1
2025-11-28T14:52:32.073034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789262055521283:2049] [node 3] ICP22 created new session: [2:7577789266350488706:2049]
2025-11-28T14:52:32.073303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789265500366829:2050] [node 3] ICS09 handshake done sender: [5:7577789265500366815:2132] self: [5:7577789265500366820:0] peer: [3:7577789268856229717:0] socket: 616 qp: -1
2025-11-28T14:52:32.072250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.074427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242636:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
2025-11-28T14:52:32.073493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266350488706:2049] [node 3] ICS09 handshake done sender: [2:7577789266350488677:2139] self: [2:7577789266350488675:0] peer: [3:7577789268856229744:0] socket: 568 qp: -1
2025-11-28T14:52:32.073973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789268856229747:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14642
2025-11-28T14:52:32.072791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789267934502695:2109] [node 6] ICP22 created new session: [4:7577789267934502731:2109]
2025-11-28T14:52:32.073333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789268300939638:2144] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:32.073339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789265500366829:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:32.074022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789268856229748:2111] [node 6] ICS10 traffic start
2025-11-28T14:52:32.073462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789267934502731:2109] [node 6] ICS09 handshake done sender: [4:7577789267934502705:2114] self: [4:7577789267934502704:0] peer: [6:7577789265013242634:0] socket: 589 qp: -1
2025-11-28T14:52:32.075315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789265013242632:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.073457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.074432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.073820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789268300939621:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.073495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789267934502731:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:32.074040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266350488706:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:32.076398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789264561262336:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19625) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789268856229739:2135] held: no
2025-11-28T14:52:32.078512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260718275246:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12857) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789265013242632:2121] held: no
2025-11-28T14:52:32.073858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.074389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789265500366827:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54072
2025-11-28T14:52:32.074070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.076271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789263639535335:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789267934502724:2121] held: no
No RDMA context found for GID: 2025-11-28T14:52:32.078698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789268856229739:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:32.078738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260718275246:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.074469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0
2025-11-28T14:52:32.074037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789268300939621:2107] [node 4] ICP22 created new session: [1:7577789268300939661:2107]
2025-11-28T14:52:32.075335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:32.078775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.074065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789268300939661:2107] [node 4] ICS09 handshake done sender: [1:7577789268300939638:2144] self: [1:7577789268300939635:0] peer: [4:7577789267934502728:0] socket: 622 qp: -1
2025-11-28T14:52:32.079252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789264561262336:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.078416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789261205399424:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789265500366819:2135] held: no
2025-11-28T14:52:32.080004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789267934502732:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28278
2025-11-28T14:52:32.074458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789268300939661:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:32.078808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789265013242631:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:32.078813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260718275246:2050] [node 3] ICP22 created new session: [6:7577789265013242637:2050]
::ffff:127.1.0.0
2025-11-28T14:52:32.080322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:32.080444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789267934502724:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.078855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789265013242637:2050] [node 3] ICS09 handshake done sender: [6:7577789265013242632:2121] self: [6:7577789265013242633:0] peer: [3:7577789268856229724:0] socket: 627 qp: -1
2025-11-28T14:52:32.075312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.080452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789265500366819:2135] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.080735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789263639535335:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.080982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789264561262336:2049] [node 2] ICP22 created new session: [3:7577789268856229750:2049]
2025-11-28T14:52:32.078973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789265013242605:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.079221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789265013242637:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:32.081136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.080153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.081212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789263639535335:2048] [node 1] ICP22 created new session: [4:7577789267934502734:2048]
2025-11-28T14:52:32.081207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789268856229750:2049] [node 2] ICS09 handshake done sender: [3:7577789268856229739:2135] self: [3:7577789268856229744:0] peer: [2:7577789266350488675:0] socket: 621 qp: -1
2025-11-28T14:52:32.080828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789265500366830:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54072
2025-11-28T14:52:32.080164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.080348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789265013242605:2106] [node 4] ICP22 created new session: [6:7577789265013242638:2106]
2025-11-28T14:52:32.081375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789268856229750:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:32.081212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789261205399424:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.080998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789265013242638:2106] [node 4] ICS09 handshake done sender: [6:7577789265013242631:2120] self: [6:7577789265013242634:0] peer: [4:7577789267934502704:0] socket: 570 qp: -1
2025-11-28T14:52:32.081513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.081386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789267934502734:2048] [node 1] ICS09 handshake done sender: [4:7577789267934502724:2121] self: [4:7577789267934502728:0] peer: [1:7577789268300939635:0] socket: 624 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.081368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.081038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789265013242638:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:32.081546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789267934502734:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.081566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789261205399424:2048] [node 1] ICP22 created new session: [5:7577789265500366835:2048]
2025-11-28T14:52:32.081321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.081610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789265500366835:2048] [node 1] ICS09 handshake done sender: [5:7577789265500366819:2135] self: [5:7577789265500366823:0] peer: [1:7577789268300939629:0] socket: 601 qp: -1
2025-11-28T14:52:32.081639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:32.081715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242640:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
2025-11-28T14:52:32.081593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266350488708:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43340
2025-11-28T14:52:32.081678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789265500366835:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.081824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:32.081877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789265500366821:2136] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:32.082262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789261205399425:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.082355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:32.082676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789261205399425:2049] [node 2] ICP22 created new session: [5:7577789265500366837:2049]
2025-11-28T14:52:32.082709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789265500366837:2049] [node 2] ICS09 handshake done sender: [5:7577789265500366821:2136] self: [5:7577789265500366826:0] peer: [2:7577789266350488685:0] socket: 623 qp: -1
2025-11-28T14:52:32.082733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789265500366837:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:32.082790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.083264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789265013242640:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.083432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789260718275305:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.083490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789266350488669:2130] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:32.083582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789265500366814:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.083604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789265500366814:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.084311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789265500366814:2131] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.084482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789265500366800:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.084572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.084567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789265013242643:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35663
2025-11-28T14:52:32.084636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789265500366800:2115] [node 6] ICP22 created new session: [5:7577789265500366840:2115]
2025-11-28T14:52:32.084663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789265500366840:2115] [node 6] ICS09 handshake done sender: [5:7577789265500366814:2131] self: [5:7577789265500366813:0] peer: [6:7577789265013242642:0] socket: 577 qp: -1
2025-11-28T14:52:32.084686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789265500366840:2115] [node 6] ICS10 traffic start
2025-11-28T14:52:32.084742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.084934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789265013242640:2124] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:32.085256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789260718275305:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.085869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:32.085934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789260718275305:2105] [node 5] ICP22 created new session: [6:7577789265013242644:2105]
2025-11-28T14:52:32.086019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789265013242644:2105] [node 5] ICS09 handshake done sender: [6:7577789265013242640:2124] self: [6:7577789265013242642:0] peer: [5:7577789265500366813:0] socket: 579 qp: -1
2025-11-28T14:52:32.086068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789265013242644:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:32.086124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.086362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789260718275305:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:32.130462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:32.130742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789266350488700:2048] [node 1] ICS01 socket: 603 reason# EndOfStream
2025-11-28T14:52:32.130831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789262055521282:2048] [node 1] ICP30 unregister session Session# [2:7577789266350488700:2048] VirtualId# [2:7577789266350488690:0]
2025-11-28T14:52:32.130878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789266350488700:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.130978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.131044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.131388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:32.131534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789268856229738:2048] [node 1] ICS07 socket disconnect 602 reason# EndOfStream
2025-11-28T14:52:32.131559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789268856229738:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.131624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.131676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789268856229738:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:32.132293Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789264561262335:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:32.132396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789268856229738:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:32.132428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789264561262335:2048] [node 1] ICP30 unregister session Session# [3:7577789268856229738:2048] VirtualId# [3:7577789268856229733:0]
2025-11-28T14:52:32.132490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.132589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789264561262335:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.132625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.133395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:32.133657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789267934502734:2048] [node 1] ICS01 socket: 624 reason# EndOfStream
2025-11-28T14:52:32.133721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789263639535335:2048] [node 1] ICP30 unregister session Session# [4:7577789267934502734:2048] VirtualId# [4:7577789267934502728:0]
2025-11-28T14:52:32.133756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789267934502734:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.133831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.133885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.133988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:32.134146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789265500366835:2048] [node 1] ICS01 socket: 601 reason# EndOfStream
2025-11-28T14:52:32.134190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789261205399424:2048] [node 1] ICP30 unregister session Session# [5:7577789265500366835:2048] VirtualId# [5:7577789265500366823:0]
2025-11-28T14:52:32.134226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789265500366835:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.134323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.134378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.134650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:32.134788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789265013242625:2048] [node 1] ICS01 socket: 581 reason# EndOfStream
2025-11-28T14:52:32.134816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260718275244:2048] [node 1] ICP30 unregister session Session# [6:7577789265013242625:2048] VirtualId# [6:7577789265013242621:0]
2025-11-28T14:52:32.134835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789265013242625:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.134885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.134929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.141962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789264561262335:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:32.191497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:32.191679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789268856229750:2049] [node 2] ICS01 socket: 621 reason# EndOfStream
2025-11-28T14:52:32.191709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789264561262336:2049] [node 2] ICP30 unregister session Session# [3:7577789268856229750:2049] VirtualId# [3:7577789268856229744:0]
2025-11-28T14:52:32.191733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789268856229750:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.191806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:32.191853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:32.194450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:32.194674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789267934502720:2049] [node 2] ICS07 socket disconnect 600 reason# EndOfStream
2025-11-28T14:52:32.194710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789267934502720:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.194825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:32.194880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789267934502720:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:32.195272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:32.195402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789265013242629:2049] [node 2] ICS01 socket: 588 reason# EndOfStream
2025-11-28T14:52:32.195434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260718275245:2049] [node 2] ICP30 unregister session Session# [6:7577789265013242629:2049] VirtualId# [6:7577789265013242622:0]
2025-11-28T14:52:32.195462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789265013242629:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.195516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:32.195554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:32.195737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789263639535336:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19625) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:32.195857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789267934502720:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:32.195906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789263639535336:2049] [node 2] ICP30 unregister session Session# [4:7577789267934502720:2049] VirtualId# [4:7577789267934502714:0]
2025-11-28T14:52:32.195948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:32.195986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789263639535336:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.195975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:32.196007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.196172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789265500366837:2049] [node 2] ICS07 socket disconnect 623 reason# EndOfStream
2025-11-28T14:52:32.196201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789265500366837:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.196293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:32.196351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789265500366837:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:32.197171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789261205399425:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19625) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:32.197235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789265500366837:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:32.197261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789261205399425:2049] [node 2] ICP30 unregister session Session# [5:7577789265500366837:2049] VirtualId# [5:7577789265500366826:0]
2025-11-28T14:52:32.197329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:32.197379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789261205399425:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.197419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.205493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789263639535336:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:32.207485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789261205399425:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:32.245424Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:32.245690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789267934502717:2050] [node 3] ICS01 socket: 586 reason# EndOfStream
2025-11-28T14:52:32.245755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789263639535337:2050] [node 3] ICP30 unregister session Session# [4:7577789267934502717:2050] VirtualId# [4:7577789267934502712:0]
2025-11-28T14:52:32.245787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789267934502717:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.245900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:32.245970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:32.246072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:32.246252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789265500366829:2050] [node 3] ICS01 socket: 616 reason# EndOfStream
2025-11-28T14:52:32.246308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789261205399426:2050] [node 3] ICP30 unregister session Session# [5:7577789265500366829:2050] VirtualId# [5:7577789265500366820:0]
2025-11-28T14:52:32.246367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789265500366829:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.246461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:32.246510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:32.246826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:32.247093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789265013242637:2050] [node 3] ICS01 socket: 627 reason# EndOfStream
2025-11-28T14:52:32.247148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260718275246:2050] [node 3] ICP30 unregister session Session# [6:7577789265013242637:2050] VirtualId# [6:7577789265013242633:0]
2025-11-28T14:52:32.247190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789265013242637:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.247299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:32.247357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:32.292109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:32.292351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789265500366828:2108] [node 4] ICS01 socket: 595 reason# EndOfStream
2025-11-28T14:52:32.292425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789265500366801:2108] [node 4] ICP30 unregister session Session# [5:7577789265500366828:2108] VirtualId# [5:7577789265500366818:0]
2025-11-28T14:52:32.292462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789265500366828:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.292594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:32.292657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:32.292804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:32.292960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789265013242638:2106] [node 4] ICS01 socket: 570 reason# EndOfStream
2025-11-28T14:52:32.293038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789265013242605:2106] [node 4] ICP30 unregister session Session# [6:7577789265013242638:2106] VirtualId# [6:7577789265013242634:0]
2025-11-28T14:52:32.293091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789265013242638:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.293174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:32.293209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:32.325626Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:32.325794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789265013242644:2105] [node 5] ICS07 socket disconnect 579 reason# EndOfStream
2025-11-28T14:52:32.325818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789265013242644:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.325892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:32.325925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789265013242644:2105] [node 5] ICS15 start handshake
2025-11-28T14:52:32.326534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789260718275305:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14547) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:32.326580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789265013242644:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:32.326624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789260718275305:2105] [node 5] ICP30 unregister session Session# [6:7577789265013242644:2105] VirtualId# [6:7577789265013242642:0]
2025-11-28T14:52:32.326673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:32.326692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789260718275305:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.326716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.336637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789260718275305:2105] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:32.449924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789267206974718:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:32.450150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789267206974719:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:32.477385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789266230191410:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:32.477388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789266230191411:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:32.507913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789266239866867:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:32.508164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789266239866868:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:32.534663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789265444994760:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:32.534688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789265444994761:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:32.534907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789265444994762:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:32.556459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789268277261497:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:32.556512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789268277261498:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:32.556963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789268277261499:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:32.583583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789266608898120:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:32.583617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789266608898121:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:32.583734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789266608898122:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:32.652037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789267206974800:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:32.668538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789265444994841:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:32.669322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789268277261560:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:32.669397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789266608898182:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:32.668561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789266239866931:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:32.669581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789266230191475:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:32.669930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789267206974802:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:32.670329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789265444994842:2126] [node 6] ICP01 ready to work
2025-11-28T14:52:32.670353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789266230191477:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:32.670013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789266239866932:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:32.670027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789267206974801:2124] [node 6] ICP01 ready to work
2025-11-28T14:52:32.669627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789266608898183:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:32.670051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789266230191476:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:32.670373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789266239866933:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:32.670384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789268277261561:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:32.676067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789267206974718:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45718516ns) {Configure:104(45712198ns) {ProcessConfigured:135(135998ns) {StartInitialHandshake:157(132909ns) {DropHandshakes:470(1071ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(181ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:477(1089ns) {SwitchToState:179(374ns)}}}}}
2025-11-28T14:52:32.677702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789265444994762:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34661804ns) {Configure:104(34652221ns) {ProcessConfigured:135(427453ns) {StartInitialHandshake:157(425198ns) {DropHandshakes:470(1080ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(6905ns) {SwitchToState:179(403ns)}}}}}
2025-11-28T14:52:32.678246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789265444994761:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41878341ns) {Configure:104(41874610ns) {ProcessConfigured:135(456610ns) {StartInitialHandshake:157(454563ns) {DropHandshakes:470(1087ns) {DropIncomingHandshake:442(347ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(462ns) PrepareNewSessionHandshake:477(1016ns) {SwitchToState:179(500ns)}}}}}
2025-11-28T14:52:32.679824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789266239866868:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21049599ns) {Configure:104(21043249ns) {ProcessConfigured:135(787995ns) {StartInitialHandshake:157(786565ns) {DropHandshakes:470(1005ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:477(771ns) {SwitchToState:179(376ns)}}}}}
2025-11-28T14:52:32.679945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789266230191411:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35818391ns) {Configure:104(35809534ns) {ProcessConfigured:135(734320ns) {StartInitialHandshake:157(732039ns) {DropHandshakes:470(1802ns) {DropIncomingHandshake:442(960ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(153481ns) {SwitchToState:179(850ns)}}}}}
2025-11-28T14:52:32.680134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789268277261499:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41456887ns) {Configure:104(41449740ns) {ProcessConfigured:135(636248ns) {StartInitialHandshake:157(634572ns) {DropHandshakes:470(1181ns) {DropIncomingHandshake:442(449ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(612ns) PrepareNewSessionHandshake:477(1142ns) {SwitchToState:179(576ns)}}}}}
2025-11-28T14:52:32.680132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789266230191410:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48365146ns) {Configure:104(48361639ns) {ProcessConfigured:135(632633ns) {StartInitialHandshake:157(630793ns) {DropHandshakes:470(1210ns) {DropIncomingHandshake:442(385ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(731ns) {SwitchToState:179(326ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.681349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789266239866867:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47668338ns) {Configure:104(47661357ns) {ProcessConfigured:135(510167ns) {StartInitialHandshake:157(508372ns) {DropHandshakes:470(779ns) {DropIncomingHandshake:442(224ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(478ns) PrepareNewSessionHandshake:477(674ns) {SwitchToState:179(311ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.680978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789268277261498:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(42079011ns) {Configure:104(42074857ns) {ProcessConfigured:135(784161ns) {StartInitialHandshake:157(783096ns) {DropHandshakes:470(1043ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(487ns) PrepareNewSessionHandshake:477(730ns) {SwitchToState:179(355ns)}}}}}
2025-11-28T14:52:32.682063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789268277261497:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21743432ns) {Configure:104(15462196ns) {ProcessConfigured:135(57048ns) {StartInitialHandshake:157(55551ns) {DropHandshakes:470(887ns) {DropIncomingHandshake:442(247ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(110ns) PrepareNewSessionHandshake:477(699ns) {SwitchToState:179(256ns)}}}}}
2025-11-28T14:52:32.683914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789266239866943:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31269
2025-11-28T14:52:32.684061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789265444994853:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40952
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.685501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266230191488:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55128
2025-11-28T14:52:32.685670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789265444994853:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.685750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789266239866943:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.687144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266230191488:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.686151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789266239866867:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.688082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789265444994760:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.687967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789267206974816:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32606
2025-11-28T14:52:32.688210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266230191491:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55128
2025-11-28T14:52:32.688614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789266239866950:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31269
2025-11-28T14:52:32.689718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789267206974809:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.690837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898199:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.690027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266230191494:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55128
2025-11-28T14:52:32.689747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789267206974809:2131] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.695986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789268277261574:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60454
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.697278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789268277261577:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60454
2025-11-28T14:52:32.690144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789267206974816:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.697565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789266239866949:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31269
2025-11-28T14:52:32.690149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789266230191410:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.698144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789265444994857:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40952
2025-11-28T14:52:32.698295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789268277261574:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.698144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898198:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.690710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789267206974808:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.698576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789266608898199:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.690676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266230191489:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55128
2025-11-28T14:52:32.698837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789268277261576:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60454
2025-11-28T14:52:32.694020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266230191494:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.699638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789265444994859:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40952
2025-11-28T14:52:32.698855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789268277261578:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60454
2025-11-28T14:52:32.695280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266230191489:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.690729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789267206974808:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.699424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266230191487:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.699291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789268277261560:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: No RDMA context found for GID: 2025-11-28T14:52:32.699392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789266608898121:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.699451Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789266230191487:2116] [node 6] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-28T14:52:32.700904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789268277261577:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.700982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266230191491:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.690845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789267206974817:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32606
2025-11-28T14:52:32.700997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789268277261576:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.699915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789266608898198:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.701036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266230191498:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55128
2025-11-28T14:52:32.700999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789265444994852:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.693986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789267206974807:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.701128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789266239866949:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.700975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789266608898182:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.701039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789268277261499:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.694017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789267206974807:2129] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.701027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789265444994852:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.701341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789266239866951:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31269
2025-11-28T14:52:32.698284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789267206974808:2130] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:32.701350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789265444994860:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40952
2025-11-28T14:52:32.701122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789268277261578:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.701427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789265444994859:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.701404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789266239866950:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.704291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266230191487:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:32.698577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789267206974809:2131] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.702876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789266239866938:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.701493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789268277261498:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.702079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789265444994857:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.705102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266230191488:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.702786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789268277261567:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.702889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789266239866938:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.698580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789267206974807:2129] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:32.702889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789265444994761:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.710833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898202:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.705935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266230191483:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.699416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789267206974817:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.705830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789266239866943:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.702809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789268277261567:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.705108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789265444994852:2132] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:32.701405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789267206974719:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.705265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789268277261567:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:32.705431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789265444994853:2133] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.705958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789266230191483:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.709702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789266239866867:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789266239866943:2115] held: no
2025-11-28T14:52:32.701461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.706299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266230191486:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.711856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789268277261561:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.706318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789265444994760:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.710272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789266239866867:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.715621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898203:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.701574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789267206974718:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.712398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.706323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789266230191486:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.706358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.716645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789266608898199:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:32.710624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.717595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898204:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.718733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789266608898121:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22743) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789266608898199:2116] held: no
2025-11-28T14:52:32.707505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789266230191410:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.719143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789266608898121:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.707625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789265444994760:2048] [node 1] ICP22 created new session: [4:7577789265444994864:2048]
2025-11-28T14:52:32.707531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789266230191477:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.719694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:32.701606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:32.707707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789265444994841:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.707926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.719948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789266608898182:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9964) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789266608898198:2115] held: no
2025-11-28T14:52:32.710866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789266239866938:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.713624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789268277261561:2109] [node 6] ICP22 created new session: [5:7577789268277261585:2109]
2025-11-28T14:52:32.709686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.701651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789267206974718:2048] [node 2] ICP22 created new session: [1:7577789267206974824:2048]
2025-11-28T14:52:32.710172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789265444994841:2108] [node 5] ICP22 created new session: [4:7577789265444994865:2108]
2025-11-28T14:52:32.720253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789266608898198:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:32.714411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789268277261585:2109] [node 6] ICS09 handshake done sender: [5:7577789268277261567:2112] self: [5:7577789268277261566:0] peer: [6:7577789266608898201:0] socket: 591 qp: -1
2025-11-28T14:52:32.709697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789266230191410:2048] [node 1] ICP22 created new session: [2:7577789266230191501:2048]
2025-11-28T14:52:32.702204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789267206974800:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.711522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789266239866867:2048] [node 1] ICP22 created new session: [3:7577789266239866955:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.710811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.710138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789265444994864:2048] [node 1] ICS09 handshake done sender: [4:7577789265444994853:2133] self: [4:7577789265444994858:0] peer: [1:7577789267206974804:0] socket: 573 qp: -1
2025-11-28T14:52:32.720342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789266608898121:2049] [node 2] ICP22 created new session: [6:7577789266608898206:2049]
2025-11-28T14:52:32.712690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789266239866955:2048] [node 1] ICS09 handshake done sender: [3:7577789266239866943:2115] self: [3:7577789266239866948:0] peer: [1:7577789267206974803:0] socket: 575 qp: -1
2025-11-28T14:52:32.710850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266230191501:2048] [node 1] ICS09 handshake done sender: [2:7577789266230191488:2117] self: [2:7577789266230191495:0] peer: [1:7577789267206974805:0] socket: 576 qp: -1
2025-11-28T14:52:32.711440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789265444994865:2108] [node 5] ICS09 handshake done sender: [4:7577789265444994852:2132] self: [4:7577789265444994846:0] peer: [5:7577789268277261579:0] socket: 580 qp: -1
2025-11-28T14:52:32.720403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789266608898206:2049] [node 2] ICS09 handshake done sender: [6:7577789266608898199:2116] self: [6:7577789266608898200:0] peer: [2:7577789266230191484:0] socket: 611 qp: -1
2025-11-28T14:52:32.702236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789267206974824:2048] [node 2] ICS09 handshake done sender: [1:7577789267206974807:2129] self: [1:7577789267206974805:0] peer: [2:7577789266230191495:0] socket: 572 qp: -1
2025-11-28T14:52:32.710887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266230191501:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.713231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789266239866955:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.711442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789265444994864:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.720921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789266608898206:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:32.714593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789268277261584:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60454
2025-11-28T14:52:32.711776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789266230191477:2109] [node 6] ICP22 created new session: [2:7577789266230191503:2109]
2025-11-28T14:52:32.711461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789265444994865:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:32.702633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.711843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:32.711475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.715645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789268277261585:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:32.702749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789267206974719:2049] [node 3] ICP22 created new session: [1:7577789267206974823:2049]
2025-11-28T14:52:32.712941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266230191486:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.712739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.713183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266230191503:2109] [node 6] ICS09 handshake done sender: [2:7577789266230191487:2116] self: [2:7577789266230191484:0] peer: [6:7577789266608898200:0] socket: 601 qp: -1
2025-11-28T14:52:32.721162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789266608898182:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.713836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:32.715685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.719623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789265444994760:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:32.702772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789267206974824:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:32.713238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266230191483:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:32.721206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
::ffff:127.1.0.0
2025-11-28T14:52:32.714293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266230191503:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:32.721242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789266608898182:2106] [node 5] ICP22 created new session: [6:7577789266608898209:2106]
2025-11-28T14:52:32.702797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.714021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789266239866931:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.718199Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789268277261560:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63647) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789268277261574:2115] held: no
2025-11-28T14:52:32.714599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.720892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789265444994869:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40952
2025-11-28T14:52:32.714068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:32.722481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.715654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789266230191475:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.720321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789268277261574:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:32.702937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789267206974800:2107] [node 4] ICP22 created new session: [1:7577789267206974825:2107]
2025-11-28T14:52:32.723559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789266608898204:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.723802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789265444994761:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22743) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789265444994859:2135] held: no
2025-11-28T14:52:32.717549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789266230191476:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.723776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789266608898209:2106] [node 5] ICS09 handshake done sender: [6:7577789266608898198:2115] self: [6:7577789266608898201:0] peer: [5:7577789268277261566:0] socket: 606 qp: -1
2025-11-28T14:52:32.717647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: 2025-11-28T14:52:32.714367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789266239866931:2108] [node 5] ICP22 created new session: [3:7577789266239866958:2108]
::ffff:127.1.0.0
2025-11-28T14:52:32.718171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.724728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789265444994859:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:32.724656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789266608898209:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:32.718701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789266230191476:2107] [node 4] ICP22 created new session: [2:7577789266230191510:2107]
2025-11-28T14:52:32.724945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.703712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789267206974823:2049] [node 3] ICS09 handshake done sender: [1:7577789267206974809:2131] self: [1:7577789267206974803:0] peer: [3:7577789266239866948:0] socket: 571 qp: -1
2025-11-28T14:52:32.718194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789266239866958:2108] [node 5] ICS09 handshake done sender: [3:7577789266239866938:2111] self: [3:7577789266239866934:0] peer: [5:7577789268277261580:0] socket: 592 qp: -1
2025-11-28T14:52:32.718878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789266230191475:2108] [node 5] ICP22 created new session: [2:7577789266230191509:2108]
2025-11-28T14:52:32.725411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789266608898120:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.722435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789268277261587:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60454
2025-11-28T14:52:32.725030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789265444994871:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40952
2025-11-28T14:52:32.719097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266230191510:2107] [node 4] ICS09 handshake done sender: [2:7577789266230191486:2115] self: [2:7577789266230191485:0] peer: [4:7577789265444994863:0] socket: 604 qp: -1
2025-11-28T14:52:32.718231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789266239866958:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:32.703905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789267206974825:2107] [node 4] ICS09 handshake done sender: [1:7577789267206974808:2130] self: [1:7577789267206974804:0] peer: [4:7577789265444994858:0] socket: 570 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.725128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789265444994761:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.727904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898208:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.723380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789268277261560:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.718731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.719650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266230191509:2108] [node 5] ICS09 handshake done sender: [2:7577789266230191483:2114] self: [2:7577789266230191480:0] peer: [5:7577789268277261581:0] socket: 600 qp: -1
2025-11-28T14:52:32.723844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.704054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789267206974823:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:32.704063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789267206974825:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:32.725176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:32.719676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266230191510:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:32.722407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789266239866959:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31269
2025-11-28T14:52:32.728950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898207:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.719885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266230191509:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:32.724117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789268277261499:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789268277261577:2117] held: no
2025-11-28T14:52:32.730373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789266608898208:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.732085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789266608898183:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.704098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.724291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789268277261588:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60454
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.733184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789266608898207:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.720276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.725387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789265444994872:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40952
2025-11-28T14:52:32.724057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789266239866961:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31269
2025-11-28T14:52:32.724761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789266239866959:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.724895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789266239866961:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.720564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.733498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789266608898122:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.704242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:32.724945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789266239866868:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.729156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789266239866944:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.725723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789265444994761:2049] [node 2] ICP22 created new session: [4:7577789265444994873:2049]
2025-11-28T14:52:32.724360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789268277261560:2108] [node 4] ICP22 created new session: [5:7577789268277261591:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.728921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789267206974812:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.726957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789265444994873:2049] [node 2] ICS09 handshake done sender: [4:7577789265444994859:2135] self: [4:7577789265444994863:0] peer: [2:7577789266230191485:0] socket: 605 qp: -1
2025-11-28T14:52:32.729186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789266239866944:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.724619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789268277261589:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60454
2025-11-28T14:52:32.731240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789266239866944:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:32.733221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789266239866868:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22743) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789266239866959:2120] held: no
2025-11-28T14:52:32.724999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789268277261591:2108] [node 4] ICS09 handshake done sender: [5:7577789268277261574:2115] self: [5:7577789268277261579:0] peer: [4:7577789265444994846:0] socket: 587 qp: -1
2025-11-28T14:52:32.726979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789265444994873:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:32.725013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789266230191508:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55128
2025-11-28T14:52:32.734053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789266239866965:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31269
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.725060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789268277261591:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:32.728953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789267206974812:2133] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.727417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789265444994871:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.725730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266230191481:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.731066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789267206974831:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32606
2025-11-28T14:52:32.734087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789266239866932:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.727907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.725762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789266230191481:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.736642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898215:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.725173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789268277261587:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.728183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789265444994762:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.728215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789266230191481:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.734728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.725421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.733843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789265444994872:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.728329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789266230191508:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.731078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789267206974811:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.735259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789265444994850:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.734785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789266239866932:2107] [node 4] ICP22 created new session: [3:7577789266239866967:2107]
2025-11-28T14:52:32.728501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789266230191475:2108] [node 5] ICP18 handshake (actor [2:7577789266230191508:2122]) is too old
2025-11-28T14:52:32.727382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789268277261578:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:32.736395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789266239866967:2107] [node 4] ICS09 handshake done sender: [3:7577789266239866944:2116] self: [3:7577789266239866937:0] peer: [4:7577789265444994875:0] socket: 589 qp: -1
2025-11-28T14:52:32.735285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789265444994850:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.727595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789268277261497:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:32.731077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789267206974832:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32606
2025-11-28T14:52:32.729834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789266230191411:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.739219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789266608898204:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.727639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789268277261577:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.736626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789266239866942:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.730928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.728734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789268277261498:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.736641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789266239866967:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:32.739777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789266608898120:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789266608898204:2119] held: no
2025-11-28T14:52:32.729175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:32.735305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789265444994876:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40952
2025-11-28T14:52:32.736651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789266239866942:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.731107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789266230191411:2049] [node 3] ICP22 created new session: [2:7577789266230191514:2049]
2025-11-28T14:52:32.731098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789267206974811:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:32.731167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789267206974812:2133] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:32.729196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789268277261499:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.737893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.741378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789266608898120:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.737761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789265444994871:2138] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.731156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789266230191514:2049] [node 3] ICS09 handshake done sender: [2:7577789266230191481:2112] self: [2:7577789266230191479:0] peer: [3:7577789266239866963:0] socket: 594 qp: -1
2025-11-28T14:52:32.729223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.732057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789266230191514:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:32.741880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.738115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789265444994850:2130] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:32.733428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789267206974801:2124] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.738006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789266239866959:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:32.733478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.733236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:32.739582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789266239866868:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.742387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789266608898120:2048] [node 1] ICP22 created new session: [6:7577789266608898219:2048]
2025-11-28T14:52:32.730836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789268277261499:2050] [node 3] ICP22 created new session: [5:7577789268277261596:2050]
2025-11-28T14:52:32.739214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789265444994762:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789265444994871:2138] held: no
2025-11-28T14:52:32.730879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789268277261596:2050] [node 3] ICS09 handshake done sender: [5:7577789268277261577:2117] self: [5:7577789268277261580:0] peer: [3:7577789266239866934:0] socket: 603 qp: -1
2025-11-28T14:52:32.742534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789266608898219:2048] [node 1] ICS09 handshake done sender: [6:7577789266608898204:2119] self: [6:7577789266608898213:0] peer: [1:7577789267206974810:0] socket: 624 qp: -1
2025-11-28T14:52:32.740393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:32.733862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789267206974801:2124] [node 6] ICP22 created new session: [1:7577789267206974835:2124]
2025-11-28T14:52:32.730958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789268277261498:2049] [node 2] ICP22 created new session: [5:7577789268277261595:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.739601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789265444994762:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.742557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789266608898219:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.731054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789268277261596:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:32.742585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:32.740690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.740675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789266239866868:2049] [node 2] ICP22 created new session: [3:7577789266239866970:2049]
2025-11-28T14:52:32.734118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789267206974831:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.731219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:32.741406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789265444994842:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.741414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789265444994762:2050] [node 3] ICP22 created new session: [4:7577789265444994879:2050]
2025-11-28T14:52:32.740721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789266239866970:2049] [node 2] ICS09 handshake done sender: [3:7577789266239866959:2120] self: [3:7577789266239866963:0] peer: [2:7577789266230191479:0] socket: 631 qp: -1
2025-11-28T14:52:32.742613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789266608898183:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63647) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789266608898208:2121] held: no
2025-11-28T14:52:32.731321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789268277261595:2049] [node 2] ICS09 handshake done sender: [5:7577789268277261578:2118] self: [5:7577789268277261581:0] peer: [2:7577789266230191480:0] socket: 608 qp: -1
2025-11-28T14:52:32.741591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.744203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789266608898122:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789266608898207:2120] held: no
2025-11-28T14:52:32.740722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789266239866942:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:32.741625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789265444994842:2126] [node 6] ICP22 created new session: [4:7577789265444994880:2126]
2025-11-28T14:52:32.734122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789267206974835:2124] [node 6] ICS09 handshake done sender: [1:7577789267206974812:2133] self: [1:7577789267206974810:0] peer: [6:7577789266608898213:0] socket: 579 qp: -1
2025-11-28T14:52:32.732992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789268277261595:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:32.742198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789265444994879:2050] [node 3] ICS09 handshake done sender: [4:7577789265444994871:2138] self: [4:7577789265444994875:0] peer: [3:7577789266239866937:0] socket: 598 qp: -1
2025-11-28T14:52:32.744745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898217:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.733027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.741536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789266239866970:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:32.742443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789265444994879:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:32.734160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789267206974835:2124] [node 6] ICS10 traffic start
2025-11-28T14:52:32.741752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:32.742789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789265444994880:2126] [node 6] ICS09 handshake done sender: [4:7577789265444994850:2130] self: [4:7577789265444994843:0] peer: [6:7577789266608898214:0] socket: 634 qp: -1
2025-11-28T14:52:32.741822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789266239866933:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.743014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:32.734196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789267206974811:2132] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:32.737343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789268277261599:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60454
2025-11-28T14:52:32.745052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789266608898218:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18300
2025-11-28T14:52:32.741858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:32.743018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789265444994880:2126] [node 6] ICS10 traffic start
2025-11-28T14:52:32.745163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789266608898208:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:32.734403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.738204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789268277261587:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:32.742590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789266239866933:2109] [node 6] ICP22 created new session: [3:7577789266239866972:2109]
2025-11-28T14:52:32.743375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.740406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789268277261497:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.743182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789266239866972:2109] [node 6] ICS09 handshake done sender: [3:7577789266239866942:2114] self: [3:7577789266239866940:0] peer: [6:7577789266608898216:0] socket: 586 qp: -1
2025-11-28T14:52:32.745336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789266608898207:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:32.741425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:32.743324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789266239866972:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:32.745730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789266608898183:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.743869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:32.741625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789268277261497:2048] [node 1] ICP22 created new session: [5:7577789268277261600:2048]
2025-11-28T14:52:32.735347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789267206974802:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:32.745769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:32.742204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789268277261600:2048] [node 1] ICS09 handshake done sender: [5:7577789268277261587:2120] self: [5:7577789268277261594:0] peer: [1:7577789267206974806:0] socket: 583 qp: -1
2025-11-28T14:52:32.745768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789266608898122:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:32.735377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:32.742559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789268277261600:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:32.745931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789266608898183:2108] [node 4] ICP22 created new session: [6:7577789266608898221:2108]
2025-11-28T14:52:32.742610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:32.746110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789266608898221:2108] [node 4] ICS09 handshake done sender: [6:7577789266608898208:2121] self: [6:7577789266608898214:0] peer: [4:7577789265444994843:0] socket: 590 qp: -1
2025-11-28T14:52:32.735403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789267206974802:2108] [node 5] ICP22 created new session: [1:7577789267206974837:2108]
2025-11-28T14:52:32.746138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789266608898221:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:32.746184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:32.736515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789267206974801:2124] [node 6] ICP18 handshake (actor [1:7577789267206974831:2136]) is too old
2025-11-28T14:52:32.746266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:32.746239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789266608898122:2050] [node 3] ICP22 created new session: [6:7577789266608898222:2050]
2025-11-28T14:52:32.746352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789266608898222:2050] [node 3] ICS09 handshake done sender: [6:7577789266608898207:2120] self: [6:7577789266608898216:0] peer: [3:7577789266239866940:0] socket: 584 qp: -1
2025-11-28T14:52:32.736624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789267206974837:2108] [node 5] ICS09 handshake done sender: [1:7577789267206974811:2132] self: [1:7577789267206974806:0] peer: [5:7577789268277261594:0] socket: 623 qp: -1
2025-11-28T14:52:32.746382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789266608898222:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:32.736623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789267206974832:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:32.746422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:32.737702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789267206974837:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:32.737901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:32.741398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789267206974802:2108] [node 5] ICP18 handshake (actor [1:7577789267206974832:2137]) is too old
2025-11-28T14:52:32.786979Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:32.787152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789266230191501:2048] [node 1] ICS07 socket disconnect 576 reason# EndOfStream
2025-11-28T14:52:32.787177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789266230191501:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.787269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.787319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789266230191501:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:32.788302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:32.788538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789266239866955:2048] [node 1] ICS01 socket: 575 reason# EndOfStream
2025-11-28T14:52:32.788595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789266239866867:2048] [node 1] ICP30 unregister session Session# [3:7577789266239866955:2048] VirtualId# [3:7577789266239866948:0]
2025-11-28T14:52:32.788624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789266239866955:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.788698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.788750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.790016Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789266230191410:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:32.790049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789266230191501:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:32.790975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789266230191410:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:32.791022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789266230191501:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:32.791684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789266230191410:2048] [node 1] ICP30 unregister session Session# [2:7577789266230191501:2048] VirtualId# [2:7577789266230191495:0]
2025-11-28T14:52:32.791723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.791748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789266230191410:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.791757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.792008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:32.792176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789265444994864:2048] [node 1] ICS07 socket disconnect 573 reason# EndOfStream
2025-11-28T14:52:32.792209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789265444994864:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.792282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.792327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789265444994864:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:32.793118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789265444994760:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:32.793307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789265444994864:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:32.793218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:32.793348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789265444994760:2048] [node 1] ICP30 unregister session Session# [4:7577789265444994864:2048] VirtualId# [4:7577789265444994858:0]
2025-11-28T14:52:32.793406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.793446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789265444994760:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.793471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.793424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789268277261600:2048] [node 1] ICS07 socket disconnect 583 reason# EndOfStream
2025-11-28T14:52:32.793456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789268277261600:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.793524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.793578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789268277261600:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:32.794377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:32.794617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789266608898219:2048] [node 1] ICS01 socket: 624 reason# EndOfStream
2025-11-28T14:52:32.794665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789266608898120:2048] [node 1] ICP30 unregister session Session# [6:7577789266608898219:2048] VirtualId# [6:7577789266608898213:0]
2025-11-28T14:52:32.794511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789268277261497:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:32.794626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789268277261600:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:32.794702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789266608898219:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.794674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789268277261497:2048] [node 1] ICP30 unregister session Session# [5:7577789268277261600:2048] VirtualId# [5:7577789268277261594:0]
2025-11-28T14:52:32.794783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:32.794727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.794831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:32.794768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789268277261497:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.794782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.803516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789265444994760:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:32.804464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789268277261497:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:32.846048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:32.846216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789266239866970:2049] [node 2] ICS01 socket: 631 reason# EndOfStream
2025-11-28T14:52:32.846242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789266239866868:2049] [node 2] ICP30 unregister session Session# [3:7577789266239866970:2049] VirtualId# [3:7577789266239866963:0]
2025-11-28T14:52:32.846265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789266239866970:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.846372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:32.846408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:32.849567Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:32.849802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789265444994873:2049] [node 2] ICS01 socket: 605 reason# EndOfStream
2025-11-28T14:52:32.849850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789265444994761:2049] [node 2] ICP30 unregister session Session# [4:7577789265444994873:2049] VirtualId# [4:7577789265444994863:0]
2025-11-28T14:52:32.849882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789265444994873:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.849981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:32.850031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:32.850594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:32.850728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789268277261595:2049] [node 2] ICS07 socket disconnect 608 reason# EndOfStream
2025-11-28T14:52:32.850744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789268277261595:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.850799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:32.850831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789268277261595:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:32.851514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789268277261498:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22743) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:32.851553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789268277261595:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:32.851737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:32.851576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789268277261498:2049] [node 2] ICP30 unregister session Session# [5:7577789268277261595:2049] VirtualId# [5:7577789268277261581:0]
2025-11-28T14:52:32.851609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:32.851633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789268277261498:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22743) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.851643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22743) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:32.852006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789266608898206:2049] [node 2] ICS01 socket: 611 reason# EndOfStream
2025-11-28T14:52:32.852061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789266608898121:2049] [node 2] ICP30 unregister session Session# [6:7577789266608898206:2049] VirtualId# [6:7577789266608898200:0]
2025-11-28T14:52:32.852093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789266608898206:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.852185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:32.852235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:32.861840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789268277261498:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:32.903804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:32.903990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789265444994879:2050] [node 3] ICS01 socket: 598 reason# EndOfStream
2025-11-28T14:52:32.904019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789265444994762:2050] [node 3] ICP30 unregister session Session# [4:7577789265444994879:2050] VirtualId# [4:7577789265444994875:0]
2025-11-28T14:52:32.904040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789265444994879:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.904100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:32.904177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:32.904573Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:32.904710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789268277261596:2050] [node 3] ICS01 socket: 603 reason# EndOfStream
2025-11-28T14:52:32.904732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789268277261499:2050] [node 3] ICP30 unregister session Session# [5:7577789268277261596:2050] VirtualId# [5:7577789268277261580:0]
2025-11-28T14:52:32.904747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789268277261596:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.904800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:32.904827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:32.905313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:32.905467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789266608898222:2050] [node 3] ICS01 socket: 584 reason# EndOfStream
2025-11-28T14:52:32.905506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789266608898122:2050] [node 3] ICP30 unregister session Session# [6:7577789266608898222:2050] VirtualId# [6:7577789266608898216:0]
2025-11-28T14:52:32.905537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789266608898222:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.905633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:32.905676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:32.955055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:32.955241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789268277261591:2108] [node 4] ICS01 socket: 587 reason# EndOfStream
2025-11-28T14:52:32.955272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789268277261560:2108] [node 4] ICP30 unregister session Session# [5:7577789268277261591:2108] VirtualId# [5:7577789268277261579:0]
2025-11-28T14:52:32.955291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789268277261591:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.955353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:32.955401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:32.955814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:32.955984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789266608898221:2108] [node 4] ICS01 socket: 590 reason# EndOfStream
2025-11-28T14:52:32.956030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789266608898183:2108] [node 4] ICP30 unregister session Session# [6:7577789266608898221:2108] VirtualId# [6:7577789266608898214:0]
2025-11-28T14:52:32.956059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789266608898221:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:32.956133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:32.956172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:33.009635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:33.009790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789266608898209:2106] [node 5] ICS01 socket: 606 reason# EndOfStream
2025-11-28T14:52:33.009826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789266608898182:2106] [node 5] ICP30 unregister session Session# [6:7577789266608898209:2106] VirtualId# [6:7577789266608898201:0]
2025-11-28T14:52:33.009863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789266608898209:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.009962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:33.010001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:33.113468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789271466340673:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:33.113508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789271466340674:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:33.136440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789270036713816:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.136480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789270036713817:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:33.158054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789271002499013:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.158099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789271002499014:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:33.180926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789271820531605:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.180974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789271820531606:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:33.181001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789271820531607:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:33.202152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271734422733:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.202194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271734422734:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:33.202219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271734422735:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:33.223283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789269899553141:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.223316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789269899553142:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:33.223342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789269899553143:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:33.307433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789269899553201:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:33.308416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271734422797:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:33.308456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789271466340737:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:33.308836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789270036713879:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:33.308817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789271002499095:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:33.309439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789270036713880:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:33.309321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789271820531668:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:33.309464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271734422798:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:33.309564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789271466340738:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:33.309710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789271820531669:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:33.309725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789269899553205:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:33.309972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789270036713881:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:33.310119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789271466340739:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:33.311269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789271002499098:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:33.313079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789271002499101:2126] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.318080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789271820531683:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.318843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789271466340750:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60170
2025-11-28T14:52:33.319078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789271002499108:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31007
2025-11-28T14:52:33.319163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789271466340751:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60170
2025-11-28T14:52:33.319433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789271002499109:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31007
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.319672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789270036713895:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.320318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789271466340755:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60170
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.325296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789270036713896:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.326360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789270036713897:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.326611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789270036713898:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
2025-11-28T14:52:33.326650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789270036713899:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.326726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789270036713895:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.326660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789271820531684:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-11-28T14:52:33.326753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789271820531685:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-11-28T14:52:33.326960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789271820531683:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.327147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789270036713896:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.327673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789271820531606:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.327805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789270036713816:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.327868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340751:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.328008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789270036713887:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.328031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789270036713887:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.328050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789271820531684:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.328032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340750:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.328194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789271820531605:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.328189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340741:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.328219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789271466340741:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.328759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789271002499113:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31007
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.329318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789271466340756:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60170
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.329463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789271466340757:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60170
2025-11-28T14:52:33.329553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789270036713897:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.329629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789271820531685:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.329714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789271820531607:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.329706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789271002499109:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.329700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789270036713898:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.329834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789271002499108:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.329859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271734422812:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24181
2025-11-28T14:52:33.329983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789271002499114:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31007
2025-11-28T14:52:33.329660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340755:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.329735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340743:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.330033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789270036713887:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:33.329747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789271466340743:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.330566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789271002499103:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.330597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789271002499103:2131] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.331153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789271820531689:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-11-28T14:52:33.331283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340756:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.331307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789270036713879:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.331334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789271002499114:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.331353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:33.331383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789270036713879:2107] [node 4] ICP22 created new session: [2:7577789270036713905:2107]
2025-11-28T14:52:33.331415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789270036713905:2107] [node 4] ICS09 handshake done sender: [2:7577789270036713887:2112] self: [2:7577789270036713886:0] peer: [4:7577789271820531686:0] socket: 580 qp: -1
2025-11-28T14:52:33.331471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789271002499113:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.331499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789270036713905:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:33.331569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:33.331564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340757:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.331659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789271002499014:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.331669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789271820531683:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:33.331761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789271466340741:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:33.331656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271734422812:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.332096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789271820531606:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.332056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789271002499013:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.332191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:33.332256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789271820531606:2049] [node 2] ICP22 created new session: [4:7577789271820531691:2049]
2025-11-28T14:52:33.332309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789271002499103:2131] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:33.332335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789271820531691:2049] [node 2] ICS09 handshake done sender: [4:7577789271820531683:2115] self: [4:7577789271820531686:0] peer: [2:7577789270036713886:0] socket: 582 qp: -1
2025-11-28T14:52:33.332388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789271820531691:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:33.332426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:33.332418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789271466340743:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:33.333173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789271002499095:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.333918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:33.333984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789271002499095:2107] [node 4] ICP22 created new session: [3:7577789271002499120:2107]
2025-11-28T14:52:33.334294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789271002499120:2107] [node 4] ICS09 handshake done sender: [3:7577789271002499103:2131] self: [3:7577789271002499102:0] peer: [4:7577789271820531688:0] socket: 594 qp: -1
2025-11-28T14:52:33.334343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789271002499120:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:33.334408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:33.334634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789270036713885:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.334621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789271466340673:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.334670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789270036713885:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.334723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:33.334773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789271466340673:2048] [node 2] ICP22 created new session: [1:7577789271466340764:2048]
2025-11-28T14:52:33.334856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789271466340764:2048] [node 2] ICS09 handshake done sender: [1:7577789271466340741:2111] self: [1:7577789271466340740:0] peer: [2:7577789270036713901:0] socket: 591 qp: -1
2025-11-28T14:52:33.334888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789271466340764:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:33.334921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:33.336500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789270036713899:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.337002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789271820531690:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-11-28T14:52:33.336982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340745:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.337018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789271466340745:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.337582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789271466340737:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.337666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:33.337725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789271466340737:2107] [node 4] ICP22 created new session: [1:7577789271466340766:2107]
2025-11-28T14:52:33.337667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271734422814:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24181
2025-11-28T14:52:33.337761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789271466340766:2107] [node 4] ICS09 handshake done sender: [1:7577789271466340743:2112] self: [1:7577789271466340742:0] peer: [4:7577789271820531687:0] socket: 593 qp: -1
2025-11-28T14:52:33.337881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789271466340766:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:33.337920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.338270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789270036713907:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.338441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271734422815:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24181
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.338649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271734422814:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.338651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789271002499123:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31007
2025-11-28T14:52:33.339645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789271734422735:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.339892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789271002499105:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.339913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789271002499105:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.340001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789271820531607:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789271820531685:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.340125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789271820531693:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-11-28T14:52:33.341174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789271002499105:2132] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.341350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789271820531684:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:33.341400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271734422817:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24181
2025-11-28T14:52:33.341863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789271002499098:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.341959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.342001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789271002499098:2109] [node 5] ICP22 created new session: [3:7577789271002499125:2109]
2025-11-28T14:52:33.341969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789270036713885:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:33.342063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789271002499125:2109] [node 5] ICS09 handshake done sender: [3:7577789271002499105:2132] self: [3:7577789271002499104:0] peer: [5:7577789271734422816:0] socket: 579 qp: -1
2025-11-28T14:52:33.342088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789271002499125:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:33.342685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:33.342777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271734422814:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:33.342882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789271820531605:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.342936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.343055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789270036713896:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:33.342976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789271466340745:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:33.342974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789271820531605:2048] [node 1] ICP22 created new session: [4:7577789271820531695:2048]
2025-11-28T14:52:33.342998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789271820531695:2048] [node 1] ICS09 handshake done sender: [4:7577789271820531684:2116] self: [4:7577789271820531687:0] peer: [1:7577789271466340742:0] socket: 597 qp: -1
2025-11-28T14:52:33.343300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553219:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
2025-11-28T14:52:33.343026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789271820531695:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:33.343108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:33.343558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789271820531605:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:33.343576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789271466340674:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.343576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789271734422735:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.343624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.343660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271734422735:2050] [node 3] ICP22 created new session: [5:7577789271734422819:2050]
2025-11-28T14:52:33.343674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.343695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271734422819:2050] [node 3] ICS09 handshake done sender: [5:7577789271734422814:2117] self: [5:7577789271734422816:0] peer: [3:7577789271002499104:0] socket: 587 qp: -1
2025-11-28T14:52:33.343676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789270036713816:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789270036713896:2116] held: no
2025-11-28T14:52:33.343721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789271466340674:2049] [node 3] ICP22 created new session: [1:7577789271466340772:2049]
2025-11-28T14:52:33.343723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271734422819:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:33.343769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789271466340772:2049] [node 3] ICS09 handshake done sender: [1:7577789271466340745:2113] self: [1:7577789271466340744:0] peer: [3:7577789271002499119:0] socket: 596 qp: -1
2025-11-28T14:52:33.343775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789270036713816:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.343769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:33.343815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789271466340772:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:33.343851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:33.343845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:33.343919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789270036713816:2048] [node 1] ICP22 created new session: [2:7577789270036713911:2048]
2025-11-28T14:52:33.343990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789270036713911:2048] [node 1] ICS09 handshake done sender: [2:7577789270036713896:2116] self: [2:7577789270036713901:0] peer: [1:7577789271466340740:0] socket: 598 qp: -1
2025-11-28T14:52:33.344012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789270036713911:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:33.344052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:33.344036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789271820531685:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:33.344492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789271820531607:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.344522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.344550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789271820531607:2050] [node 3] ICP22 created new session: [4:7577789271820531698:2050]
2025-11-28T14:52:33.344575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789271820531698:2050] [node 3] ICS09 handshake done sender: [4:7577789271820531685:2117] self: [4:7577789271820531688:0] peer: [3:7577789271002499102:0] socket: 599 qp: -1
2025-11-28T14:52:33.344567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789270036713817:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.344603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789271820531698:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:33.344637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.344618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:33.344679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789270036713817:2049] [node 3] ICP22 created new session: [2:7577789270036713913:2049]
2025-11-28T14:52:33.345643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789270036713913:2049] [node 3] ICS09 handshake done sender: [2:7577789270036713885:2111] self: [2:7577789270036713884:0] peer: [3:7577789271002499117:0] socket: 578 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.345684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789270036713913:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:33.345731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:33.345843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553220:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.346016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789271002499108:2134] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:33.346028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271734422818:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24181
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.346874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271734422821:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24181
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.347808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553221:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
2025-11-28T14:52:33.348040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789271002499014:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.348540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:33.348589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789271002499014:2049] [node 2] ICP22 created new session: [3:7577789271002499129:2049]
2025-11-28T14:52:33.348619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789271002499129:2049] [node 2] ICS09 handshake done sender: [3:7577789271002499108:2134] self: [3:7577789271002499117:0] peer: [2:7577789270036713884:0] socket: 584 qp: -1
2025-11-28T14:52:33.348649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789271002499129:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:33.348672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.349650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789271002499127:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31007
2025-11-28T14:52:33.349783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271734422821:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.349883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789271734422797:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.349879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789271820531700:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-11-28T14:52:33.350069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789271820531677:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.349949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789269899553220:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.350102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789271820531677:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.350248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789271002499013:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789271002499113:2136] held: no
2025-11-28T14:52:33.350249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789269899553205:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.350277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271734422818:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.350350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789271734422733:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.350411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271734422815:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.350497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789269899553219:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.350548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789271734422734:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.350624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340747:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.350659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789271820531700:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.350658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789271466340747:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.351447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789271820531679:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.351467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789271820531679:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.351701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789271466340747:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:33.352190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789269899553141:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.352384Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789271002499014:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.352549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789271002499113:2136] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:33.352527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789270036713889:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.352560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271734422826:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24181
2025-11-28T14:52:33.352695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789271820531677:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:33.352724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789271002499013:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.352793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789271466340738:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.352558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789270036713889:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.352598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553223:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
2025-11-28T14:52:33.352854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.353556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:33.353235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789271466340738:2108] [node 5] ICP22 created new session: [1:7577789271466340775:2108]
2025-11-28T14:52:33.353590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789271002499013:2048] [node 1] ICP22 created new session: [3:7577789271002499131:2048]
2025-11-28T14:52:33.353623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789271466340775:2108] [node 5] ICS09 handshake done sender: [1:7577789271466340747:2114] self: [1:7577789271466340746:0] peer: [5:7577789271734422824:0] socket: 586 qp: -1
2025-11-28T14:52:33.353624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789271002499131:2048] [node 1] ICS09 handshake done sender: [3:7577789271002499113:2136] self: [3:7577789271002499119:0] peer: [1:7577789271466340744:0] socket: 600 qp: -1
2025-11-28T14:52:33.353633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789271820531679:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:33.353651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789271002499131:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:33.353677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:33.353669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789271466340775:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:33.353742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:33.353723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271734422818:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.353955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789270036713889:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:33.353953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789271820531668:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.353830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789269899553223:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.354030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.354067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789271820531668:2108] [node 5] ICP22 created new session: [4:7577789271820531704:2108]
2025-11-28T14:52:33.354148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789271820531704:2108] [node 5] ICS09 handshake done sender: [4:7577789271820531677:2113] self: [4:7577789271820531676:0] peer: [5:7577789271734422823:0] socket: 577 qp: -1
2025-11-28T14:52:33.354066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553225:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
2025-11-28T14:52:33.354169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789271820531704:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:33.354246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789271734422733:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.354247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:33.354322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:33.354354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271734422733:2048] [node 1] ICP22 created new session: [5:7577789271734422829:2048]
2025-11-28T14:52:33.354369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789270036713880:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.354390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271734422829:2048] [node 1] ICS09 handshake done sender: [5:7577789271734422818:2120] self: [5:7577789271734422824:0] peer: [1:7577789271466340746:0] socket: 629 qp: -1
2025-11-28T14:52:33.354377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789271466340749:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.354415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271734422829:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:33.354403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789271466340749:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.354447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.354506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:33.355080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789270036713880:2108] [node 5] ICP22 created new session: [2:7577789270036713917:2108]
2025-11-28T14:52:33.355237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789270036713917:2108] [node 5] ICS09 handshake done sender: [2:7577789270036713889:2113] self: [2:7577789270036713888:0] peer: [5:7577789271734422825:0] socket: 626 qp: -1
2025-11-28T14:52:33.355293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789270036713917:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:33.355338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789269899553142:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.355444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789271820531669:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.355324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:33.355471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.355468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789269899553220:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:33.355492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789271820531669:2109] [node 6] ICP22 created new session: [4:7577789271820531706:2109]
2025-11-28T14:52:33.355517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789271820531706:2109] [node 6] ICS09 handshake done sender: [4:7577789271820531679:2114] self: [4:7577789271820531678:0] peer: [6:7577789269899553222:0] socket: 617 qp: -1
2025-11-28T14:52:33.355544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789271820531706:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:33.355567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.355982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271734422827:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24181
2025-11-28T14:52:33.356108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789269899553205:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.356154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:33.356276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789269899553205:2106] [node 4] ICP22 created new session: [6:7577789269899553227:2106]
2025-11-28T14:52:33.356596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789269899553227:2106] [node 4] ICS09 handshake done sender: [6:7577789269899553220:2116] self: [6:7577789269899553222:0] peer: [4:7577789271820531678:0] socket: 633 qp: -1
2025-11-28T14:52:33.356630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789269899553227:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:33.356662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:33.356713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789270036713891:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.356744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789270036713891:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.357438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789271466340749:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:33.358205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789271466340739:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.358337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.358336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789270036713891:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:33.358415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789271466340739:2109] [node 6] ICP22 created new session: [1:7577789271466340778:2109]
2025-11-28T14:52:33.358460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789271466340778:2109] [node 6] ICS09 handshake done sender: [1:7577789271466340749:2115] self: [1:7577789271466340748:0] peer: [6:7577789269899553224:0] socket: 630 qp: -1
2025-11-28T14:52:33.358494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789271466340778:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:33.358697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:33.358942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789270036713881:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.359000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.359014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789269899553221:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.359033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789270036713881:2109] [node 6] ICP22 created new session: [2:7577789270036713920:2109]
2025-11-28T14:52:33.359097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789270036713920:2109] [node 6] ICS09 handshake done sender: [2:7577789270036713891:2114] self: [2:7577789270036713890:0] peer: [6:7577789269899553226:0] socket: 601 qp: -1
2025-11-28T14:52:33.359125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789270036713920:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:33.359161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.360052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271734422828:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24181
2025-11-28T14:52:33.360326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271734422821:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:33.360827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789271734422797:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14484) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789271734422821:2121] held: no
2025-11-28T14:52:33.360905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789271734422797:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.360967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:33.361005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271734422797:2108] [node 4] ICP22 created new session: [5:7577789271734422831:2108]
2025-11-28T14:52:33.361036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271734422831:2108] [node 4] ICS09 handshake done sender: [5:7577789271734422821:2121] self: [5:7577789271734422823:0] peer: [4:7577789271820531676:0] socket: 606 qp: -1
2025-11-28T14:52:33.361058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271734422831:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:33.361084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:33.361411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789271734422734:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789271734422815:2118] held: no
2025-11-28T14:52:33.361697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271734422815:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:33.361924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789271734422734:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.361958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:33.361997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271734422734:2049] [node 2] ICP22 created new session: [5:7577789271734422834:2049]
2025-11-28T14:52:33.362026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271734422834:2049] [node 2] ICS09 handshake done sender: [5:7577789271734422815:2118] self: [5:7577789271734422825:0] peer: [2:7577789270036713888:0] socket: 627 qp: -1
2025-11-28T14:52:33.362050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271734422834:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:33.362070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:33.362106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789269899553143:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.362186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789269899553141:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789269899553219:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.362336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553229:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.362710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553230:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
2025-11-28T14:52:33.363169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789269899553142:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789269899553223:2118] held: no
2025-11-28T14:52:33.363659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789271002499107:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.363697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789269899553223:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:33.363683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789271002499107:2133] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.364331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789271002499133:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31007
2025-11-28T14:52:33.364476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789269899553219:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:33.364621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789271002499107:2133] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:33.364764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789269899553142:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.364814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:33.364990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789271002499101:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.364986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789269899553142:2049] [node 2] ICP22 created new session: [6:7577789269899553234:2049]
2025-11-28T14:52:33.365058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.365049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789269899553234:2049] [node 2] ICS09 handshake done sender: [6:7577789269899553223:2118] self: [6:7577789269899553226:0] peer: [2:7577789270036713890:0] socket: 636 qp: -1
2025-11-28T14:52:33.365108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789271002499101:2126] [node 6] ICP22 created new session: [3:7577789271002499135:2126]
2025-11-28T14:52:33.365094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789269899553234:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:33.365142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:33.365161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789271002499135:2126] [node 6] ICS09 handshake done sender: [3:7577789271002499107:2133] self: [3:7577789271002499106:0] peer: [6:7577789269899553232:0] socket: 595 qp: -1
2025-11-28T14:52:33.365200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789271002499135:2126] [node 6] ICS10 traffic start
2025-11-28T14:52:33.365241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:33.365623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789269899553141:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.365659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:33.365700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789269899553141:2048] [node 1] ICP22 created new session: [6:7577789269899553237:2048]
2025-11-28T14:52:33.365738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789269899553237:2048] [node 1] ICS09 handshake done sender: [6:7577789269899553219:2115] self: [6:7577789269899553224:0] peer: [1:7577789271466340748:0] socket: 631 qp: -1
2025-11-28T14:52:33.365778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789269899553237:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:33.365809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.366226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553233:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
2025-11-28T14:52:33.367775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789269899553233:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.367918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553236:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
2025-11-28T14:52:33.368446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789269899553221:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:33.368483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789271002499133:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.368629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789271002499101:2126] [node 6] ICP18 handshake (actor [3:7577789271002499133:2140]) is too old
2025-11-28T14:52:33.368711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789269899553201:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.368755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789269899553143:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.368803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.368840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789269899553143:2050] [node 3] ICP22 created new session: [6:7577789269899553240:2050]
2025-11-28T14:52:33.368919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789269899553240:2050] [node 3] ICS09 handshake done sender: [6:7577789269899553221:2117] self: [6:7577789269899553232:0] peer: [3:7577789271002499106:0] socket: 634 qp: -1
2025-11-28T14:52:33.368953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789269899553240:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:33.368981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:33.369602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271734422808:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.369629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789271734422808:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.370364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271734422808:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:33.370622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789271734422798:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.370677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.370719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271734422798:2110] [node 6] ICP22 created new session: [5:7577789271734422837:2110]
2025-11-28T14:52:33.370755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271734422837:2110] [node 6] ICS09 handshake done sender: [5:7577789271734422808:2115] self: [5:7577789271734422807:0] peer: [6:7577789269899553239:0] socket: 605 qp: -1
2025-11-28T14:52:33.370779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271734422837:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:33.370801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.371055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789269899553242:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41082
2025-11-28T14:52:33.372138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789269899553233:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:33.372314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789269899553201:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30046) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789269899553233:2122] held: no
2025-11-28T14:52:33.372426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789269899553201:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.372454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.372487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789269899553201:2105] [node 5] ICP22 created new session: [6:7577789269899553243:2105]
2025-11-28T14:52:33.372532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789269899553243:2105] [node 5] ICS09 handshake done sender: [6:7577789269899553233:2122] self: [6:7577789269899553239:0] peer: [5:7577789271734422807:0] socket: 613 qp: -1
2025-11-28T14:52:33.372564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789269899553243:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:33.372606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:33.385192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:33.385469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789270036713911:2048] [node 1] ICS01 socket: 598 reason# EndOfStream
2025-11-28T14:52:33.385559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789270036713816:2048] [node 1] ICP30 unregister session Session# [2:7577789270036713911:2048] VirtualId# [2:7577789270036713901:0]
2025-11-28T14:52:33.385597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789270036713911:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.385704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.385772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.386125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:33.386310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789271002499131:2048] [node 1] ICS01 socket: 600 reason# EndOfStream
2025-11-28T14:52:33.386343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789271002499013:2048] [node 1] ICP30 unregister session Session# [3:7577789271002499131:2048] VirtualId# [3:7577789271002499119:0]
2025-11-28T14:52:33.386367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789271002499131:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.386457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.386501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.387770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:33.387912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789271820531695:2048] [node 1] ICS01 socket: 597 reason# EndOfStream
2025-11-28T14:52:33.387938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789271820531605:2048] [node 1] ICP30 unregister session Session# [4:7577789271820531695:2048] VirtualId# [4:7577789271820531687:0]
2025-11-28T14:52:33.387958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789271820531695:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.388024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.388067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.388430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:33.388702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789271734422829:2048] [node 1] ICS01 socket: 629 reason# EndOfStream
2025-11-28T14:52:33.388750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789271734422733:2048] [node 1] ICP30 unregister session Session# [5:7577789271734422829:2048] VirtualId# [5:7577789271734422824:0]
2025-11-28T14:52:33.388786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789271734422829:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.388870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.388917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.389084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:33.389323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789269899553237:2048] [node 1] ICS01 socket: 631 reason# EndOfStream
2025-11-28T14:52:33.389372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789269899553141:2048] [node 1] ICP30 unregister session Session# [6:7577789269899553237:2048] VirtualId# [6:7577789269899553224:0]
2025-11-28T14:52:33.389402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789269899553237:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.389474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.389521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.411509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:33.411674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789271002499129:2049] [node 2] ICS01 socket: 584 reason# EndOfStream
2025-11-28T14:52:33.411710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789271002499014:2049] [node 2] ICP30 unregister session Session# [3:7577789271002499129:2049] VirtualId# [3:7577789271002499117:0]
2025-11-28T14:52:33.411737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789271002499129:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.411833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:33.411888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:33.413531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:33.413712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789271820531691:2049] [node 2] ICS01 socket: 582 reason# EndOfStream
2025-11-28T14:52:33.413743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789271820531606:2049] [node 2] ICP30 unregister session Session# [4:7577789271820531691:2049] VirtualId# [4:7577789271820531686:0]
2025-11-28T14:52:33.413761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789271820531691:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.413830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:33.413867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:33.414204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:33.414358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789271734422834:2049] [node 2] ICS01 socket: 627 reason# EndOfStream
2025-11-28T14:52:33.414390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789271734422734:2049] [node 2] ICP30 unregister session Session# [5:7577789271734422834:2049] VirtualId# [5:7577789271734422825:0]
2025-11-28T14:52:33.414416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789271734422834:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.414487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:33.414524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:33.414810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:33.415022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789269899553234:2049] [node 2] ICS01 socket: 636 reason# EndOfStream
2025-11-28T14:52:33.415071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789269899553142:2049] [node 2] ICP30 unregister session Session# [6:7577789269899553234:2049] VirtualId# [6:7577789269899553226:0]
2025-11-28T14:52:33.415101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789269899553234:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.415167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:33.415216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:33.449762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:33.449927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789271820531698:2050] [node 3] ICS01 socket: 599 reason# EndOfStream
2025-11-28T14:52:33.450001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789271820531607:2050] [node 3] ICP30 unregister session Session# [4:7577789271820531698:2050] VirtualId# [4:7577789271820531688:0]
2025-11-28T14:52:33.450043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789271820531698:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.450121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:33.450154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:33.450396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:33.450559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789271734422819:2050] [node 3] ICS01 socket: 587 reason# EndOfStream
2025-11-28T14:52:33.450605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789271734422735:2050] [node 3] ICP30 unregister session Session# [5:7577789271734422819:2050] VirtualId# [5:7577789271734422816:0]
2025-11-28T14:52:33.450658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789271734422819:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.450740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:33.450778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:33.450998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:33.451230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789269899553240:2050] [node 3] ICS07 socket disconnect 634 reason# EndOfStream
2025-11-28T14:52:33.451268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789269899553240:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.451344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:33.451392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789269899553240:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:33.452204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789269899553143:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8057) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:33.452267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789269899553240:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:33.452297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789269899553143:2050] [node 3] ICP30 unregister session Session# [6:7577789269899553240:2050] VirtualId# [6:7577789269899553232:0]
2025-11-28T14:52:33.452342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:33.452423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789269899553143:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8057) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.452454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8057) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.461963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789269899553143:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:33.469034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:33.469204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789271734422831:2108] [node 4] ICS01 socket: 606 reason# EndOfStream
2025-11-28T14:52:33.469240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789271734422797:2108] [node 4] ICP30 unregister session Session# [5:7577789271734422831:2108] VirtualId# [5:7577789271734422823:0]
2025-11-28T14:52:33.469269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789271734422831:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.469379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:33.469443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:33.469763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:33.470027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789269899553227:2106] [node 4] ICS07 socket disconnect 633 reason# EndOfStream
2025-11-28T14:52:33.470061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789269899553227:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.470138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:33.470194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789269899553227:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:33.470941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789269899553205:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14484) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:33.470994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789269899553227:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:33.471053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789269899553205:2106] [node 4] ICP30 unregister session Session# [6:7577789269899553227:2106] VirtualId# [6:7577789269899553222:0]
2025-11-28T14:52:33.471128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:33.471195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789269899553205:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14484) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.471243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14484) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.481399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789269899553205:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:33.501293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:33.501578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789269899553243:2105] [node 5] ICS01 socket: 613 reason# EndOfStream
2025-11-28T14:52:33.501638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789269899553201:2105] [node 5] ICP30 unregister session Session# [6:7577789269899553243:2105] VirtualId# [6:7577789269899553239:0]
2025-11-28T14:52:33.501678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789269899553243:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.501781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:33.501847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:33.561919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789270874340541:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:33.562005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789270874340542:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:33.579253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789273125669877:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.579327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789273125669878:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:33.595891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789272961822513:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.596098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789272961822514:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:33.613305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789269540637899:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:33.613320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789269540637898:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.613341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789269540637900:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:33.629848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271759228804:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.629893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271759228806:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:33.630004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271759228805:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:33.646911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789270318419762:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:33.646920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789270318419761:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:33.646946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789270318419763:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:33.730889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271759228867:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:33.731342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789271759228868:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:33.731104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789273125669940:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:33.731391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789269540637962:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:33.731369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789270318419822:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:33.731967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789273125669941:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:33.732399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789269540637963:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:33.732601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789270318419825:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:33.732868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789273125669942:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:33.734374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789273125669878:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44679169ns) {Configure:104(44676284ns) {ProcessConfigured:135(129527ns) {StartInitialHandshake:157(122436ns) {DropHandshakes:470(3340ns) {DropIncomingHandshake:442(1075ns) DropOutgoingHandshake:457(424ns)} GenerateSessionVirtualId:988(1126ns) PrepareNewSessionHandshake:477(1281ns) {SwitchToState:179(245ns)}}}}}
2025-11-28T14:52:33.734980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789270318419762:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34548819ns) {Configure:104(34543181ns) {ProcessConfigured:135(72927ns) {StartInitialHandshake:157(69960ns) {DropHandshakes:470(2003ns) {DropIncomingHandshake:442(551ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(1065ns) PrepareNewSessionHandshake:477(10389ns) {SwitchToState:179(600ns)}}}}}
2025-11-28T14:52:33.735047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789270874340541:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43663524ns) {Configure:104(43656470ns) {ProcessConfigured:135(49254ns) {StartInitialHandshake:157(47263ns) {DropHandshakes:470(1147ns) {DropIncomingHandshake:442(363ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(598ns) PrepareNewSessionHandshake:477(1045ns) {SwitchToState:179(384ns)}}}}}
2025-11-28T14:52:33.735079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789270874340542:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47106782ns) {Configure:104(43660911ns) {ProcessConfigured:135(61008ns) {StartInitialHandshake:157(56703ns) {DropHandshakes:470(1634ns) {DropIncomingHandshake:442(494ns) DropOutgoingHandshake:457(358ns)} GenerateSessionVirtualId:988(945ns) PrepareNewSessionHandshake:477(3291ns) {SwitchToState:179(676ns)}}}}}
2025-11-28T14:52:33.735449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789270874340608:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:33.735491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789270874340609:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:33.735656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789270874340610:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:33.735711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789269540637898:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43725088ns) {Configure:104(43721030ns) {ProcessConfigured:135(60015ns) {StartInitialHandshake:157(56087ns) {DropHandshakes:470(1177ns) {DropIncomingHandshake:442(420ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(710ns) PrepareNewSessionHandshake:477(5017ns) {SwitchToState:179(366ns)}}}}}
2025-11-28T14:52:33.736169Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789272961822513:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44539810ns) {Configure:104(44530306ns) {ProcessConfigured:135(122127ns) {StartInitialHandshake:157(120701ns) {DropHandshakes:470(911ns) {DropIncomingHandshake:442(207ns) DropOutgoingHandshake:457(142ns)} GenerateSessionVirtualId:988(757ns) PrepareNewSessionHandshake:477(685ns) {SwitchToState:179(158ns)}}}}}
2025-11-28T14:52:33.736203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789272961822514:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44656762ns) {Configure:104(44653174ns) {ProcessConfigured:135(154059ns) {StartInitialHandshake:157(152824ns) {DropHandshakes:470(1486ns) {DropIncomingHandshake:442(648ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(861ns) {SwitchToState:179(443ns)}}}}}
2025-11-28T14:52:33.736606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789272961822581:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:33.736683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789272961822582:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:33.737030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789272961822583:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.739661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789273125669956:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45659
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.740729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789269540637974:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23559
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.741017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789269540637975:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23559
2025-11-28T14:52:33.741346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789270874340618:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42259
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.741545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789270874340619:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42259
2025-11-28T14:52:33.741707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789269540637979:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23559
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.743977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789273125669957:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45659
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.744222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271759228882:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7283
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.744466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789270874340622:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42259
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.746413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789272961822588:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35380
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.747238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271759228883:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7283
2025-11-28T14:52:33.747476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789272961822589:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35380
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.747968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789272961822588:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.747985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789269540637974:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.748317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789269540637979:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.748268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789270874340623:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42259
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.748351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789272961822514:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.748571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340619:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.748233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789273125669958:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45659
2025-11-28T14:52:33.748590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340618:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.748630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789273125669959:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45659
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.748736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789269540637975:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.748789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789273125669957:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.748796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789273125669956:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.749365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789273125669944:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.749389Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789273125669944:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.749388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789273125669960:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45659
2025-11-28T14:52:33.750712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789273125669958:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.750761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340622:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.750989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789273125669944:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:33.750839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340623:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.750844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789269540637899:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.751003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789269540637980:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23559
2025-11-28T14:52:33.751134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789270874340624:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42259
2025-11-28T14:52:33.750998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271759228882:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.751151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789273125669948:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.751145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789271759228867:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.751178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789273125669948:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.752272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789273125669878:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.753663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271759228883:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.752321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.752645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789269540637969:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.752520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789273125669878:2049] [node 3] ICP22 created new session: [2:7577789273125669966:2049]
2025-11-28T14:52:33.753765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789271759228804:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.752668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789269540637969:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.753749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789269540637969:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:33.752572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789273125669966:2049] [node 3] ICS09 handshake done sender: [2:7577789273125669944:2110] self: [2:7577789273125669943:0] peer: [3:7577789272961822595:0] socket: 584 qp: -1
2025-11-28T14:52:33.752669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789273125669966:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:33.754311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340612:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.753385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789273125669959:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.754332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789272961822589:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.753400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:33.754367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789270874340612:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.753792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789273125669948:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:33.754940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789273125669940:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.754976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340624:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.755852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:33.755001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789269540637962:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.755885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789273125669940:2107] [node 4] ICP22 created new session: [2:7577789273125669969:2107]
2025-11-28T14:52:33.755878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789269540637985:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23559
2025-11-28T14:52:33.754969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271759228886:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7283
2025-11-28T14:52:33.755884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.755924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789273125669969:2107] [node 4] ICS09 handshake done sender: [2:7577789273125669948:2112] self: [2:7577789273125669947:0] peer: [4:7577789269540637983:0] socket: 588 qp: -1
2025-11-28T14:52:33.755919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789269540637962:2108] [node 5] ICP22 created new session: [4:7577789269540637986:2108]
2025-11-28T14:52:33.755958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789269540637986:2108] [node 5] ICS09 handshake done sender: [4:7577789269540637969:2113] self: [4:7577789269540637968:0] peer: [5:7577789271759228884:0] socket: 605 qp: -1
2025-11-28T14:52:33.755959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789273125669969:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:33.755985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789269540637986:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:33.755993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:33.756008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.756072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789272961822513:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.756200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789270874340612:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:33.756090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789272961822596:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35380
2025-11-28T14:52:33.756305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340606:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.756554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271759228882:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:33.756324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789270874340606:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.756969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789272961822588:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:33.757007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789271759228867:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.757133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:33.757723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271759228867:2108] [node 4] ICP22 created new session: [5:7577789271759228889:2108]
2025-11-28T14:52:33.757724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789270874340608:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.757710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789273125669960:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.757792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.757781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789272961822514:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.757820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:33.757903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789270874340608:2108] [node 5] ICP22 created new session: [1:7577789270874340631:2108]
2025-11-28T14:52:33.758015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789273125669877:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.757998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271759228889:2108] [node 4] ICS09 handshake done sender: [5:7577789271759228882:2115] self: [5:7577789271759228884:0] peer: [4:7577789269540637968:0] socket: 607 qp: -1
2025-11-28T14:52:33.757839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789269540637980:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.758134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789272961822514:2049] [node 2] ICP22 created new session: [3:7577789272961822598:2049]
2025-11-28T14:52:33.758253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789270874340631:2108] [node 5] ICS09 handshake done sender: [1:7577789270874340612:2112] self: [1:7577789270874340611:0] peer: [5:7577789271759228885:0] socket: 610 qp: -1
2025-11-28T14:52:33.758260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789272961822598:2049] [node 2] ICS09 handshake done sender: [3:7577789272961822588:2115] self: [3:7577789272961822595:0] peer: [2:7577789273125669943:0] socket: 599 qp: -1
2025-11-28T14:52:33.758341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789270874340631:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:33.758155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271759228889:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:33.758351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789272961822598:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:33.758372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:33.758181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:33.758444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:33.758482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789273125669878:2049] [node 3] ICP18 handshake (actor [2:7577789273125669960:2119]) is too old
2025-11-28T14:52:33.759156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789269540637975:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.760533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789269540637899:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789269540637975:2117] held: no
2025-11-28T14:52:33.760778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340607:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.760935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789269540637899:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.760816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789270874340607:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.760879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419839:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-28T14:52:33.761350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:33.761925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789269540637899:2049] [node 2] ICP22 created new session: [4:7577789269540637989:2049]
2025-11-28T14:52:33.761967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789269540637989:2049] [node 2] ICS09 handshake done sender: [4:7577789269540637975:2117] self: [4:7577789269540637983:0] peer: [2:7577789273125669947:0] socket: 594 qp: -1
2025-11-28T14:52:33.761999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789269540637989:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:33.762031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:33.762160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789269540637900:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.762184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789270874340606:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.762317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271759228887:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7283
2025-11-28T14:52:33.762363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271759228888:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7283
2025-11-28T14:52:33.762575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789271759228804:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789271759228883:2116] held: no
2025-11-28T14:52:33.762450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789270874340542:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.762554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.762689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789270874340542:2049] [node 3] ICP22 created new session: [1:7577789270874340636:2049]
2025-11-28T14:52:33.762860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789270874340636:2049] [node 3] ICS09 handshake done sender: [1:7577789270874340606:2110] self: [1:7577789270874340604:0] peer: [3:7577789272961822597:0] socket: 595 qp: -1
2025-11-28T14:52:33.763071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789270874340636:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.763397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271759228883:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:33.763356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.763382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789272961822600:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35380
2025-11-28T14:52:33.763421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271759228888:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.763401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789270874340607:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.763401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789272961822585:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.763929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789273125669973:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45659
2025-11-28T14:52:33.763957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789271759228804:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.763425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789272961822585:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.763991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789271759228805:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.764065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789269540637993:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23559
2025-11-28T14:52:33.764060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789270318419839:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.763923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789272961822601:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35380
2025-11-28T14:52:33.763999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:33.764078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271759228804:2048] [node 1] ICP22 created new session: [5:7577789271759228894:2048]
2025-11-28T14:52:33.764105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271759228894:2048] [node 1] ICS09 handshake done sender: [5:7577789271759228883:2116] self: [5:7577789271759228885:0] peer: [1:7577789270874340611:0] socket: 615 qp: -1
2025-11-28T14:52:33.765327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789273125669950:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.764382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271759228894:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:33.765338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789270874340541:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.765347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789272961822589:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:33.765348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789273125669950:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.764721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:33.765367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789273125669959:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:33.765551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.765991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789270318419762:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.765582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789270874340541:2048] [node 2] ICP22 created new session: [1:7577789270874340638:2048]
2025-11-28T14:52:33.766450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789270874340638:2048] [node 2] ICS09 handshake done sender: [1:7577789270874340607:2111] self: [1:7577789270874340605:0] peer: [2:7577789273125669972:0] socket: 608 qp: -1
2025-11-28T14:52:33.766504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789273125669877:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.766559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789270874340638:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:33.766578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:33.766591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419840:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-28T14:52:33.766937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419841:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-28T14:52:33.766968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789273125669877:2048] [node 1] ICP22 created new session: [2:7577789273125669974:2048]
2025-11-28T14:52:33.766591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:33.767053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789273125669974:2048] [node 1] ICS09 handshake done sender: [2:7577789273125669959:2118] self: [2:7577789273125669972:0] peer: [1:7577789270874340605:0] socket: 613 qp: -1
2025-11-28T14:52:33.767122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789273125669952:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.767157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789270318419840:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.767138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789273125669952:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.767155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789273125669974:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.767296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789272961822513:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.767345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789270318419822:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.767188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:33.767337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:33.767536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789272961822601:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.767560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789272961822513:2048] [node 1] ICP22 created new session: [3:7577789272961822605:2048]
2025-11-28T14:52:33.768467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789269540637973:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.768493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789272961822605:2048] [node 1] ICS09 handshake done sender: [3:7577789272961822589:2116] self: [3:7577789272961822597:0] peer: [1:7577789270874340604:0] socket: 602 qp: -1
2025-11-28T14:52:33.768483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789269540637994:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23559
2025-11-28T14:52:33.769097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271759228896:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7283
2025-11-28T14:52:33.768503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789269540637973:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.768656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789272961822605:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:33.768985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:33.770254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789272961822585:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:33.770307Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789272961822513:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:33.770450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789269540637973:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.770738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789270318419841:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.771116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789269540637993:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.771119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789272961822604:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35380
2025-11-28T14:52:33.771169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789273125669950:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:33.771171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789269540637963:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.771216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.771268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789269540637963:2109] [node 6] ICP22 created new session: [4:7577789269540637996:2109]
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:33.771381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789269540637996:2109] [node 6] ICS09 handshake done sender: [4:7577789269540637973:2115] self: [4:7577789269540637972:0] peer: [6:7577789270318419843:0] socket: 636 qp: -1
2025-11-28T14:52:33.771418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789269540637996:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:33.771516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:33.771524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789272961822581:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.771666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271759228897:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7283
2025-11-28T14:52:33.771667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:33.771518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789269540637980:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:33.771713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419844:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-28T14:52:33.771910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271759228896:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.771715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789272961822581:2107] [node 4] ICP22 created new session: [3:7577789272961822608:2107]
2025-11-28T14:52:33.771539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789273125669952:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:33.771755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789272961822608:2107] [node 4] ICS09 handshake done sender: [3:7577789272961822585:2113] self: [3:7577789272961822584:0] peer: [4:7577789269540637990:0] socket: 621 qp: -1
2025-11-28T14:52:33.772141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789271759228806:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.772142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789273125669942:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.772473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271759228888:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:33.771787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789272961822608:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:33.772148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789273125669941:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.772160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-28T14:52:33.771897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:33.772430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.772530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789270318419825:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.772473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789273125669942:2109] [node 6] ICP22 created new session: [2:7577789273125669978:2109]
2025-11-28T14:52:33.772071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789272961822604:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.772835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789271759228805:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.772807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789272961822587:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.772822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789272961822587:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.772513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.772886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.773563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271759228805:2049] [node 2] ICP22 created new session: [5:7577789271759228899:2049]
2025-11-28T14:52:33.772520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789273125669978:2109] [node 6] ICS09 handshake done sender: [2:7577789273125669952:2114] self: [2:7577789273125669951:0] peer: [6:7577789270318419842:0] socket: 604 qp: -1
2025-11-28T14:52:33.773553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789269540637900:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.773852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789272961822581:2107] [node 4] ICP18 handshake (actor [3:7577789272961822601:2120]) is too old
2025-11-28T14:52:33.773601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.772550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789273125669941:2108] [node 5] ICP22 created new session: [2:7577789273125669979:2108]
2025-11-28T14:52:33.773953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271759228899:2049] [node 2] ICS09 handshake done sender: [5:7577789271759228888:2119] self: [5:7577789271759228893:0] peer: [2:7577789273125669949:0] socket: 633 qp: -1
2025-11-28T14:52:33.773829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789269540637900:2050] [node 3] ICP22 created new session: [4:7577789269540637998:2050]
2025-11-28T14:52:33.773952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-28T14:52:33.772671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789273125669978:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:33.773927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789269540637998:2050] [node 3] ICS09 handshake done sender: [4:7577789269540637980:2119] self: [4:7577789269540637990:0] peer: [3:7577789272961822584:0] socket: 623 qp: -1
2025-11-28T14:52:33.773989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271759228899:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:33.773957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789269540637998:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:33.774024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:33.772822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:33.773985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:33.772758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789273125669979:2108] [node 5] ICS09 handshake done sender: [2:7577789273125669950:2113] self: [2:7577789273125669949:0] peer: [5:7577789271759228893:0] socket: 631 qp: -1
2025-11-28T14:52:33.772855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789273125669979:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:33.776486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789270318419840:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:33.772887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:33.777825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271759228878:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.777864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789271759228878:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.777879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789269540637898:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.778125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789270318419845:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.778392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789272961822587:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:33.778325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789270318419822:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1884) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789270318419840:2116] held: no
2025-11-28T14:52:33.778777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789270318419822:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.778734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340614:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.778769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789270874340614:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.779426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789272961822582:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.779425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:33.779912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.779959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789272961822582:2108] [node 5] ICP22 created new session: [3:7577789272961822612:2108]
2025-11-28T14:52:33.779480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789270318419822:2106] [node 4] ICP22 created new session: [6:7577789270318419849:2106]
2025-11-28T14:52:33.779895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789270318419849:2106] [node 4] ICS09 handshake done sender: [6:7577789270318419840:2116] self: [6:7577789270318419843:0] peer: [4:7577789269540637972:0] socket: 637 qp: -1
2025-11-28T14:52:33.780020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789270318419849:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:33.780045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789272961822612:2108] [node 5] ICS09 handshake done sender: [3:7577789272961822587:2114] self: [3:7577789272961822586:0] peer: [5:7577789271759228898:0] socket: 641 qp: -1
2025-11-28T14:52:33.780061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.780179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789272961822612:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:33.780220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:33.780295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789270318419839:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:33.780307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271759228901:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7283
2025-11-28T14:52:33.780332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789270874340614:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.780644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419848:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-28T14:52:33.780676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789270874340609:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.781405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.781455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789270874340609:2107] [node 4] ICP22 created new session: [1:7577789270874340642:2107]
2025-11-28T14:52:33.781522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789270318419762:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789270318419839:2115] held: no
2025-11-28T14:52:33.781685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271759228878:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:33.781671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789270318419762:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.781703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:33.781712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789269540638001:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23559
2025-11-28T14:52:33.781637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789270874340642:2107] [node 4] ICS09 handshake done sender: [1:7577789270874340614:2113] self: [1:7577789270874340613:0] peer: [4:7577789269540638000:0] socket: 612 qp: -1
2025-11-28T14:52:33.781839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789270318419762:2049] [node 2] ICP22 created new session: [6:7577789270318419851:2049]
2025-11-28T14:52:33.781673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789270874340642:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:33.781876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789270318419851:2049] [node 2] ICS09 handshake done sender: [6:7577789270318419839:2115] self: [6:7577789270318419842:0] peer: [2:7577789273125669951:0] socket: 606 qp: -1
2025-11-28T14:52:33.781942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789269540637993:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:33.781702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:33.781894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789271759228806:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789271759228896:2120] held: no
2025-11-28T14:52:33.782402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789269540637898:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.781897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789270318419851:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:33.782238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789271759228868:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.782351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789271759228896:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:33.782519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:33.781917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:33.782371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.782612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789269540637898:2048] [node 1] ICP22 created new session: [4:7577789269540638002:2048]
2025-11-28T14:52:33.782575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271759228868:2109] [node 6] ICP22 created new session: [5:7577789271759228903:2109]
2025-11-28T14:52:33.782744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789269540638002:2048] [node 1] ICS09 handshake done sender: [4:7577789269540637993:2121] self: [4:7577789269540638000:0] peer: [1:7577789270874340613:0] socket: 635 qp: -1
2025-11-28T14:52:33.782665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271759228903:2109] [node 6] ICS09 handshake done sender: [5:7577789271759228878:2114] self: [5:7577789271759228877:0] peer: [6:7577789270318419847:0] socket: 639 qp: -1
2025-11-28T14:52:33.782810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789269540638002:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:33.782747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271759228903:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:33.782771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:33.783677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:33.783810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789271759228806:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.783843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.783837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789270318419848:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.783879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789271759228806:2050] [node 3] ICP22 created new session: [5:7577789271759228905:2050]
2025-11-28T14:52:33.783975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789271759228905:2050] [node 3] ICS09 handshake done sender: [5:7577789271759228896:2120] self: [5:7577789271759228898:0] peer: [3:7577789272961822586:0] socket: 642 qp: -1
2025-11-28T14:52:33.783997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789271759228905:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:33.784014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:33.784039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789270318419761:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.784276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789270318419763:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:33.784418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789270874340616:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.784413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419853:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-28T14:52:33.784451Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789270874340616:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.784562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789271759228907:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7283
2025-11-28T14:52:33.784557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789272961822614:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35380
2025-11-28T14:52:33.784563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789272961822593:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.784587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789272961822593:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:33.785544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789270874340616:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:33.785753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789270874340610:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.785802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.785945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789270874340610:2109] [node 6] ICP22 created new session: [1:7577789270874340645:2109]
2025-11-28T14:52:33.786010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789270874340645:2109] [node 6] ICS09 handshake done sender: [1:7577789270874340616:2114] self: [1:7577789270874340615:0] peer: [6:7577789270318419856:0] socket: 645 qp: -1
2025-11-28T14:52:33.786025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789272961822593:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:33.786786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789270874340645:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:33.786837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:33.787207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789272961822583:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:33.787265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:33.787503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789272961822583:2109] [node 6] ICP22 created new session: [3:7577789272961822616:2109]
2025-11-28T14:52:33.787559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789272961822616:2109] [node 6] ICS09 handshake done sender: [3:7577789272961822593:2117] self: [3:7577789272961822592:0] peer: [6:7577789270318419857:0] socket: 650 qp: -1
2025-11-28T14:52:33.787588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789272961822616:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:33.787631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:33.787616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789270318419841:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:33.788227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789272961822614:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.788310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789272961822583:2109] [node 6] ICP18 handshake (actor [3:7577789272961822614:2122]) is too old
2025-11-28T14:52:33.788443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789270318419761:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789270318419845:2119] held: no
2025-11-28T14:52:33.788530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789270318419825:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.788562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:33.788605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789270318419825:2107] [node 5] ICP22 created new session: [6:7577789270318419860:2107]
2025-11-28T14:52:33.788638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789270318419860:2107] [node 5] ICS09 handshake done sender: [6:7577789270318419841:2117] self: [6:7577789270318419847:0] peer: [5:7577789271759228877:0] socket: 640 qp: -1
2025-11-28T14:52:33.788667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789270318419860:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:33.788688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:33.788770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789271759228907:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:33.789005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789271759228868:2109] [node 6] ICP18 handshake (actor [5:7577789271759228907:2123]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.790141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419858:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:33.790536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789270318419859:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-28T14:52:33.790696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789270318419763:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789270318419848:2121] held: no
2025-11-28T14:52:33.790854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789270318419848:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:33.790887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789270318419845:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:33.791015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789270318419763:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.791050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:33.791081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789270318419763:2050] [node 3] ICP22 created new session: [6:7577789270318419862:2050]
2025-11-28T14:52:33.791094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789270318419761:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:33.791105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789270318419862:2050] [node 3] ICS09 handshake done sender: [6:7577789270318419848:2121] self: [6:7577789270318419857:0] peer: [3:7577789272961822592:0] socket: 651 qp: -1
2025-11-28T14:52:33.791133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789270318419862:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:33.791151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:33.791152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:33.791191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789270318419761:2048] [node 1] ICP22 created new session: [6:7577789270318419863:2048]
2025-11-28T14:52:33.791232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789270318419863:2048] [node 1] ICS09 handshake done sender: [6:7577789270318419845:2119] self: [6:7577789270318419856:0] peer: [1:7577789270874340615:0] socket: 646 qp: -1
2025-11-28T14:52:33.791265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789270318419863:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:33.791296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:33.808493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:33.808744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789273125669974:2048] [node 1] ICS07 socket disconnect 613 reason# EndOfStream
2025-11-28T14:52:33.808780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789273125669974:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.808888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.808953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789273125669974:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:33.809464Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:33.809593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789272961822605:2048] [node 1] ICS07 socket disconnect 602 reason# EndOfStream
2025-11-28T14:52:33.809617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789272961822605:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.809676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.809719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789272961822605:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:33.810840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789273125669877:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:33.810874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789273125669974:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:33.812603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:33.812676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789272961822513:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:33.812757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789269540638002:2048] [node 1] ICS07 socket disconnect 635 reason# EndOfStream
2025-11-28T14:52:33.812715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789272961822605:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:33.812782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789269540638002:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.812842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.812909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789269540638002:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:33.812920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789273125669877:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:33.813280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789273125669974:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:33.813326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789273125669877:2048] [node 1] ICP30 unregister session Session# [2:7577789273125669974:2048] VirtualId# [2:7577789273125669972:0]
2025-11-28T14:52:33.813365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.813393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789273125669877:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.813404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.813923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789272961822513:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:33.813982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789272961822605:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:33.814012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789272961822513:2048] [node 1] ICP30 unregister session Session# [3:7577789272961822605:2048] VirtualId# [3:7577789272961822597:0]
2025-11-28T14:52:33.814063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:33.814087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789269540637898:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:33.814050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.814118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789269540638002:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:33.814187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789269540637898:2048] [node 1] ICP30 unregister session Session# [4:7577789269540638002:2048] VirtualId# [4:7577789269540638000:0]
2025-11-28T14:52:33.814069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789272961822513:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.814252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789271759228894:2048] [node 1] ICS01 socket: 615 reason# EndOfStream
2025-11-28T14:52:33.814244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.814077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.814311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789271759228804:2048] [node 1] ICP30 unregister session Session# [5:7577789271759228894:2048] VirtualId# [5:7577789271759228885:0]
2025-11-28T14:52:33.814260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789269540637898:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.814333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789271759228894:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.814267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.814391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.814429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.814988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:33.815126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789270318419863:2048] [node 1] ICS01 socket: 646 reason# EndOfStream
2025-11-28T14:52:33.815151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789270318419761:2048] [node 1] ICP30 unregister session Session# [6:7577789270318419863:2048] VirtualId# [6:7577789270318419856:0]
2025-11-28T14:52:33.815168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789270318419863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.815218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:33.815255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:33.824394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789269540637898:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:33.824396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789272961822513:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:33.845857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:33.846032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789272961822598:2049] [node 2] ICS01 socket: 599 reason# EndOfStream
2025-11-28T14:52:33.846071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789272961822514:2049] [node 2] ICP30 unregister session Session# [3:7577789272961822598:2049] VirtualId# [3:7577789272961822595:0]
2025-11-28T14:52:33.846100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789272961822598:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.846197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:33.846266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:33.847981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:33.848112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789269540637989:2049] [node 2] ICS01 socket: 594 reason# EndOfStream
2025-11-28T14:52:33.848144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789269540637899:2049] [node 2] ICP30 unregister session Session# [4:7577789269540637989:2049] VirtualId# [4:7577789269540637983:0]
2025-11-28T14:52:33.848167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789269540637989:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.848222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:33.848267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:33.848736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:33.848856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789271759228899:2049] [node 2] ICS07 socket disconnect 633 reason# EndOfStream
2025-11-28T14:52:33.848878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789271759228899:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.848932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:33.848980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789271759228899:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:33.849390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:33.849533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789271759228805:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23474) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:33.849536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789270318419851:2049] [node 2] ICS01 socket: 606 reason# EndOfStream
2025-11-28T14:52:33.849569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789270318419762:2049] [node 2] ICP30 unregister session Session# [6:7577789270318419851:2049] VirtualId# [6:7577789270318419842:0]
2025-11-28T14:52:33.849591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789271759228899:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:33.849646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789271759228805:2049] [node 2] ICP30 unregister session Session# [5:7577789271759228899:2049] VirtualId# [5:7577789271759228893:0]
2025-11-28T14:52:33.849588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789270318419851:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.849691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:33.849649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:33.849718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789271759228805:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.849691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:33.849727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.859239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789271759228805:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:33.878556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:33.878706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789269540637998:2050] [node 3] ICS01 socket: 623 reason# EndOfStream
2025-11-28T14:52:33.878738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789269540637900:2050] [node 3] ICP30 unregister session Session# [4:7577789269540637998:2050] VirtualId# [4:7577789269540637990:0]
2025-11-28T14:52:33.878774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789269540637998:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.878856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:33.878903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:33.879347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:33.879471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789271759228905:2050] [node 3] ICS01 socket: 642 reason# EndOfStream
2025-11-28T14:52:33.879498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789271759228806:2050] [node 3] ICP30 unregister session Session# [5:7577789271759228905:2050] VirtualId# [5:7577789271759228898:0]
2025-11-28T14:52:33.879514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789271759228905:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.879554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:33.879596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:33.879975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:33.880132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789270318419862:2050] [node 3] ICS01 socket: 651 reason# EndOfStream
2025-11-28T14:52:33.880160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789270318419763:2050] [node 3] ICP30 unregister session Session# [6:7577789270318419862:2050] VirtualId# [6:7577789270318419857:0]
2025-11-28T14:52:33.880202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789270318419862:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.880271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:33.880307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:33.914540Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:33.914707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789271759228889:2108] [node 4] ICS07 socket disconnect 607 reason# EndOfStream
2025-11-28T14:52:33.914729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789271759228889:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.914829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:33.914869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789271759228889:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:33.915228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:33.915434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789271759228867:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1884) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:33.915431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789270318419849:2106] [node 4] ICS01 socket: 637 reason# EndOfStream
2025-11-28T14:52:33.915480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789271759228889:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:33.915535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789270318419822:2106] [node 4] ICP30 unregister session Session# [6:7577789270318419849:2106] VirtualId# [6:7577789270318419843:0]
2025-11-28T14:52:33.915575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789271759228867:2108] [node 4] ICP30 unregister session Session# [5:7577789271759228889:2108] VirtualId# [5:7577789271759228884:0]
2025-11-28T14:52:33.915613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:33.915587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789270318419849:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.915654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789271759228867:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1884) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.915663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1884) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.915671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:33.915722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:33.925741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789271759228867:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:33.962722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:33.962900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789270318419860:2107] [node 5] ICS07 socket disconnect 640 reason# EndOfStream
2025-11-28T14:52:33.962930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789270318419860:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:33.963025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:33.963062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789270318419860:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:33.963752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789270318419825:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29468) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:33.963789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789270318419860:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:33.963821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789270318419825:2107] [node 5] ICP30 unregister session Session# [6:7577789270318419860:2107] VirtualId# [6:7577789270318419847:0]
2025-11-28T14:52:33.963861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:33.963885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789270318419825:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:33.963898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:34.030807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276964891728:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:34.030850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276964891729:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:34.048775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276225015170:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.048818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276225015171:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:34.066304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789275612834462:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.066471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789275612834463:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:34.083681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273524129255:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.083845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273524129257:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:34.083906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273524129256:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:34.105127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277131631850:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.105321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277131631851:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:34.105395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277131631852:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:34.123058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789276247755868:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:34.123083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789276247755866:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.123117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789276247755867:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:34.196503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789276247755925:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:34.196503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276964891791:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:34.202222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789275612834525:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:34.204763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276225015234:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:34.204820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276964891792:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:34.206533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276225015235:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:34.206610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789275612834526:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:34.206676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789276247755930:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:34.207542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276964891793:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:34.207748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789275612834527:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:34.207954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276225015236:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:34.211201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789275612834462:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46925222ns) {Configure:104(46919051ns) {ProcessConfigured:135(178412ns) {StartInitialHandshake:157(172588ns) {DropHandshakes:470(1443ns) {DropIncomingHandshake:442(448ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:477(957ns) {SwitchToState:179(452ns)}}}}}
2025-11-28T14:52:34.211739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789273524129255:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40234248ns) {Configure:104(40229594ns) {ProcessConfigured:135(144215ns) {StartInitialHandshake:157(142123ns) {DropHandshakes:470(1174ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(487ns) PrepareNewSessionHandshake:477(942ns) {SwitchToState:179(325ns)}}}}}
2025-11-28T14:52:34.211753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789276964891729:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48639464ns) {Configure:104(48636436ns) {ProcessConfigured:135(66449ns) {StartInitialHandshake:157(65033ns) {DropHandshakes:470(838ns) {DropIncomingHandshake:442(390ns) DropOutgoingHandshake:457(158ns)} GenerateSessionVirtualId:988(314ns) PrepareNewSessionHandshake:477(483ns) {SwitchToState:179(258ns)}}}}}
2025-11-28T14:52:34.211818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789276964891728:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49893640ns) {Configure:104(49884285ns) {ProcessConfigured:135(184627ns) {StartInitialHandshake:157(182299ns) {DropHandshakes:470(994ns) {DropIncomingHandshake:442(293ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(718ns) PrepareNewSessionHandshake:477(1044ns) {SwitchToState:179(385ns)}}}}}
2025-11-28T14:52:34.211975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789276225015170:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40655843ns) {Configure:104(40651535ns) {ProcessConfigured:135(249846ns) {StartInitialHandshake:157(247431ns) {DropHandshakes:470(3801ns) {DropIncomingHandshake:442(373ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(814ns) PrepareNewSessionHandshake:477(1124ns) {SwitchToState:179(501ns)}}}}}
2025-11-28T14:52:34.211757Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789273524129256:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41323462ns) {Configure:104(40181658ns) {ProcessConfigured:135(127734ns) {StartInitialHandshake:157(125637ns) {DropHandshakes:470(1053ns) {DropIncomingHandshake:442(241ns) DropOutgoingHandshake:457(370ns)} GenerateSessionVirtualId:988(617ns) PrepareNewSessionHandshake:477(694ns) {SwitchToState:179(256ns)}}}}}
2025-11-28T14:52:34.211785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789273524129257:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41077025ns) {Configure:104(41074617ns) {ProcessConfigured:135(69926ns) {StartInitialHandshake:157(68113ns) {DropHandshakes:470(1099ns) {DropIncomingHandshake:442(347ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(554ns) PrepareNewSessionHandshake:477(1017ns) {SwitchToState:179(317ns)}}}}}
2025-11-28T14:52:34.212065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789276225015171:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48538866ns) {Configure:104(48536717ns) {ProcessConfigured:135(185563ns) {StartInitialHandshake:157(183946ns) {DropHandshakes:470(1216ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(248ns)} GenerateSessionVirtualId:988(469ns) PrepareNewSessionHandshake:477(846ns) {SwitchToState:179(381ns)}}}}}
2025-11-28T14:52:34.212476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789276247755866:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31609461ns) {Configure:104(31604859ns) {ProcessConfigured:135(48933ns) {StartInitialHandshake:157(47311ns) {DropHandshakes:470(1016ns) {DropIncomingHandshake:442(265ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(498ns) PrepareNewSessionHandshake:477(707ns) {SwitchToState:179(262ns)}}}}}
2025-11-28T14:52:34.212554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273524129324:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:34.213196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789277131631850:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32270491ns) {Configure:104(32265537ns) {ProcessConfigured:135(364827ns) {StartInitialHandshake:157(363114ns) {DropHandshakes:470(1204ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:477(1297ns) {SwitchToState:179(676ns)}}}}}
2025-11-28T14:52:34.213024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273524129325:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:34.213366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789277131631852:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40751298ns) {Configure:104(32287413ns) {ProcessConfigured:135(361458ns) {StartInitialHandshake:157(359387ns) {DropHandshakes:470(1044ns) {DropIncomingHandshake:442(329ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(505ns) PrepareNewSessionHandshake:477(1037ns) {SwitchToState:179(449ns)}}}}}
2025-11-28T14:52:34.213354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789276247755867:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39774301ns) {Configure:104(39771814ns) {ProcessConfigured:135(354831ns) {StartInitialHandshake:157(353710ns) {DropHandshakes:470(4763ns) {DropIncomingHandshake:442(260ns) DropOutgoingHandshake:457(368ns)} GenerateSessionVirtualId:988(516ns) PrepareNewSessionHandshake:477(1157ns) {SwitchToState:179(647ns)}}}}}
2025-11-28T14:52:34.213778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789277131631851:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33114178ns) {Configure:104(33106503ns) {ProcessConfigured:135(969520ns) {StartInitialHandshake:157(967215ns) {DropHandshakes:470(1259ns) {DropIncomingHandshake:442(299ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(1252ns) {SwitchToState:179(80ns)}}}}}
2025-11-28T14:52:34.213966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277131631919:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:34.214198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277131631920:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:34.218328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789275612834541:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39751
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:34.220623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276225015248:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20591
::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:34.219571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277131631924:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17139
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.221297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789276964891808:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18486
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.221391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276225015250:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20591
2025-11-28T14:52:34.221389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273524129329:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38681
2025-11-28T14:52:34.221393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789276964891806:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18486
2025-11-28T14:52:34.221688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789275612834542:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39751
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.221737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789276964891809:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18486
2025-11-28T14:52:34.221911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789275612834543:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39751
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.224381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273524129330:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38681
2025-11-28T14:52:34.225737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789276964891811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18486
2025-11-28T14:52:34.225674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789275612834544:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39751
2025-11-28T14:52:34.225669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276225015252:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20591
2025-11-28T14:52:34.225746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789276964891810:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18486
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.226674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789275612834541:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.225794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273524129329:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.226293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276225015253:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20591
2025-11-28T14:52:34.227090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789275612834542:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.226809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273524129335:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38681
2025-11-28T14:52:34.226835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891806:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276225015253:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789273524129257:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.227167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789275612834543:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891811:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277131631924:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273524129330:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789275612834463:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.227263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276225015254:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20591
2025-11-28T14:52:34.227514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276225015248:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789275612834531:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789273524129256:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.227607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276225015252:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276225015250:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.227633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789275612834531:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.230168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891809:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.230726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891808:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.230394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789275612834531:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:34.230840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891810:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.228291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276225015240:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.228326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789276225015240:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.229927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276225015241:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.229958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789276225015241:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.231478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273524129335:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.231674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789273524129255:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.231619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789275612834525:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:34.231095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276225015241:2112] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:34.231210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276225015240:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.232249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891799:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.232053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273524129338:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38681
2025-11-28T14:52:34.231702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:34.232283Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276964891799:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.231735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789275612834525:2107] [node 4] ICP22 created new session: [3:7577789275612834550:2107]
2025-11-28T14:52:34.231639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276225015254:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.232621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273524129339:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.231667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789276225015234:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.233539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273524129329:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.232141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789275612834550:2107] [node 4] ICS09 handshake done sender: [3:7577789275612834531:2111] self: [3:7577789275612834529:0] peer: [4:7577789273524129336:0] socket: 591 qp: -1
2025-11-28T14:52:34.231702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:34.234063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273524129330:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:34.233944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276964891799:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:34.232177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789275612834550:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:34.231767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276225015234:2107] [node 4] ICP22 created new session: [2:7577789276225015261:2107]
2025-11-28T14:52:34.232276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:34.231931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789276225015171:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.231981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.237516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789273524129257:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.233395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789275612834544:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.236934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891796:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.236959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276964891796:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.234142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789275612834549:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39751
2025-11-28T14:52:34.236981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276964891791:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.232010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276225015171:2049] [node 3] ICP22 created new session: [2:7577789276225015262:2049]
2025-11-28T14:52:34.237740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789273524129256:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28496) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789273524129330:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.232032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276225015261:2107] [node 4] ICS09 handshake done sender: [2:7577789276225015241:2112] self: [2:7577789276225015239:0] peer: [4:7577789273524129337:0] socket: 595 qp: -1
2025-11-28T14:52:34.237821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789273524129256:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.234586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789275612834462:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.237259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:34.232133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276225015261:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:34.237844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:34.237481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276964891791:2107] [node 4] ICP22 created new session: [1:7577789276964891818:2107]
2025-11-28T14:52:34.237755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789275612834463:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28496) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789275612834542:2116] held: no
2025-11-28T14:52:34.237913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.240079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277131631930:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17139
2025-11-28T14:52:34.232257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:34.238073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276964891818:2107] [node 4] ICS09 handshake done sender: [1:7577789276964891799:2112] self: [1:7577789276964891798:0] peer: [4:7577789273524129340:0] socket: 616 qp: -1
2025-11-28T14:52:34.238461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789275612834542:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:34.239471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276964891818:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:34.232518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276225015262:2049] [node 3] ICS09 handshake done sender: [2:7577789276225015240:2111] self: [2:7577789276225015237:0] peer: [3:7577789275612834547:0] socket: 593 qp: -1
2025-11-28T14:52:34.239653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:34.239805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789275612834463:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.238088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273524129257:2050] [node 3] ICP22 created new session: [4:7577789273524129342:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.240281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891797:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.239834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:34.232688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276225015262:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:34.238365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273524129256:2049] [node 2] ICP22 created new session: [4:7577789273524129343:2049]
2025-11-28T14:52:34.240296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276964891797:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.239910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789275612834463:2049] [node 2] ICP22 created new session: [3:7577789275612834554:2049]
2025-11-28T14:52:34.241203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277131631931:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.238525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273524129342:2050] [node 3] ICS09 handshake done sender: [4:7577789273524129329:2113] self: [4:7577789273524129336:0] peer: [3:7577789275612834529:0] socket: 603 qp: -1
2025-11-28T14:52:34.232717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.241271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276964891796:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.240287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789275612834554:2049] [node 2] ICS09 handshake done sender: [3:7577789275612834542:2116] self: [3:7577789275612834547:0] peer: [2:7577789276225015237:0] socket: 599 qp: -1
2025-11-28T14:52:34.238540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273524129343:2049] [node 2] ICS09 handshake done sender: [4:7577789273524129330:2114] self: [4:7577789273524129337:0] peer: [2:7577789276225015239:0] socket: 606 qp: -1
2025-11-28T14:52:34.240995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789275612834554:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:34.241263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277131631930:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.238554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789276225015170:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.238544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273524129342:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:34.243127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755946:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
2025-11-28T14:52:34.241023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:34.242406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277131631932:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17139
2025-11-28T14:52:34.243716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276964891729:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.239467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273524129343:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:34.242774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789277131631852:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.239479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.243810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277131631931:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.243864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.239486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.243873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276964891797:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:34.243851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277131631932:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:34.241201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789273524129257:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:34.243918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276964891729:2049] [node 3] ICP22 created new session: [1:7577789276964891823:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.243906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789275612834535:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.243962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276964891823:2049] [node 3] ICS09 handshake done sender: [1:7577789276964891796:2110] self: [1:7577789276964891794:0] peer: [3:7577789275612834553:0] socket: 612 qp: -1
2025-11-28T14:52:34.244080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789273524129255:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13896) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789273524129335:2117] held: no
2025-11-28T14:52:34.243970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789276247755946:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.243940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789277131631850:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.243939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789275612834535:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.244288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755947:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
2025-11-28T14:52:34.244276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276964891823:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.243951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789277131631851:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.244812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789276247755867:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.244625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276225015268:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20591
2025-11-28T14:52:34.244302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.244794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789275612834557:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39751
2025-11-28T14:52:34.245282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276225015244:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.246539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789276247755947:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.245011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276964891728:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.246398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273524129341:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38681
2025-11-28T14:52:34.246452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789275612834462:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13896) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789275612834544:2118] held: no
2025-11-28T14:52:34.245208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:34.245301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789276225015244:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.245403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276225015251:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.245336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276964891728:2048] [node 2] ICP22 created new session: [1:7577789276964891825:2048]
2025-11-28T14:52:34.247853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273524129335:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.247859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789275612834535:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:34.247418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789276247755868:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.245420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789276225015251:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.247884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789275612834544:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.246359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276964891825:2048] [node 2] ICS09 handshake done sender: [1:7577789276964891797:2111] self: [1:7577789276964891795:0] peer: [2:7577789276225015265:0] socket: 617 qp: -1
2025-11-28T14:52:34.246444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276225015254:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.248766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789275612834462:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.246470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276225015244:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.250239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789273524129255:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.246503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891801:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.247412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789276225015170:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.250633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.248777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789275612834537:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.246516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276964891825:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:34.248797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:34.250805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273524129255:2048] [node 1] ICP22 created new session: [4:7577789273524129348:2048]
2025-11-28T14:52:34.248799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789275612834537:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.247454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: 2025-11-28T14:52:34.250922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273524129348:2048] [node 1] ICS09 handshake done sender: [4:7577789273524129335:2117] self: [4:7577789273524129340:0] peer: [1:7577789276964891798:0] socket: 624 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:34.246520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276964891801:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.247891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276225015170:2048] [node 1] ICP22 created new session: [2:7577789276225015271:2048]
2025-11-28T14:52:34.251094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273524129348:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:34.251060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277131631938:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17139
2025-11-28T14:52:34.246932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:34.251152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755951:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
2025-11-28T14:52:34.250184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789275612834462:2048] [node 1] ICP22 created new session: [3:7577789275612834559:2048]
2025-11-28T14:52:34.248631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276225015271:2048] [node 1] ICS09 handshake done sender: [2:7577789276225015254:2120] self: [2:7577789276225015265:0] peer: [1:7577789276964891795:0] socket: 623 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.251169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277131631936:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17139
2025-11-28T14:52:34.251473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277131631937:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17139
2025-11-28T14:52:34.250241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789275612834526:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.251224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755950:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
2025-11-28T14:52:34.248657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276225015271:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:34.250277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789275612834559:2048] [node 1] ICS09 handshake done sender: [3:7577789275612834544:2118] self: [3:7577789275612834553:0] peer: [1:7577789276964891794:0] socket: 620 qp: -1
2025-11-28T14:52:34.251357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.251246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276964891801:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:34.250332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.248789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789276225015235:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.250656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789275612834526:2108] [node 5] ICP22 created new session: [3:7577789275612834560:2108]
2025-11-28T14:52:34.248835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.250684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789275612834559:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:34.251951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789276247755950:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.249394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276225015235:2108] [node 5] ICP22 created new session: [2:7577789276225015272:2108]
2025-11-28T14:52:34.250693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789275612834560:2108] [node 5] ICS09 handshake done sender: [3:7577789275612834535:2113] self: [3:7577789275612834534:0] peer: [5:7577789277131631933:0] socket: 607 qp: -1
2025-11-28T14:52:34.252269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276964891792:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.253046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789276247755946:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.250768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.252871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.249405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.251047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789275612834560:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:34.253061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276964891792:2108] [node 5] ICP22 created new session: [1:7577789276964891829:2108]
2025-11-28T14:52:34.251076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.250274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276225015272:2108] [node 5] ICS09 handshake done sender: [2:7577789276225015244:2114] self: [2:7577789276225015243:0] peer: [5:7577789277131631935:0] socket: 592 qp: -1
2025-11-28T14:52:34.254131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789277131631852:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18331) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789277131631930:2116] held: no
2025-11-28T14:52:34.253642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789276247755930:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.252853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789275612834537:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:34.253574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276964891829:2108] [node 5] ICS09 handshake done sender: [1:7577789276964891801:2113] self: [1:7577789276964891800:0] peer: [5:7577789277131631934:0] socket: 589 qp: -1
2025-11-28T14:52:34.250621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276225015272:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:34.254191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789275612834527:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.254196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277131631938:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.254502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755952:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
2025-11-28T14:52:34.254056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276964891829:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:34.254608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789276247755867:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.254512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.250637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276225015251:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:34.254379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789277131631851:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28496) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789277131631932:2118] held: no
2025-11-28T14:52:34.254204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.254691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789275612834527:2109] [node 6] ICP22 created new session: [3:7577789275612834564:2109]
2025-11-28T14:52:34.254642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.254673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789276247755867:2049] [node 2] ICP22 created new session: [6:7577789276247755955:2049]
2025-11-28T14:52:34.251067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.255034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789275612834564:2109] [node 6] ICS09 handshake done sender: [3:7577789275612834537:2114] self: [3:7577789275612834536:0] peer: [6:7577789276247755949:0] socket: 640 qp: -1
2025-11-28T14:52:34.255060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789275612834564:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:34.255137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273524129332:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.255162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.254949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789276247755955:2049] [node 2] ICS09 handshake done sender: [6:7577789276247755946:2117] self: [6:7577789276247755948:0] peer: [2:7577789276225015249:0] socket: 638 qp: -1
2025-11-28T14:52:34.254382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277131631932:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:34.252861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789276225015236:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.252896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:34.255195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789276247755955:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:34.255162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789273524129332:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.255219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:34.254635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277131631930:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.254807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789277131631851:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.253041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276225015236:2110] [node 6] ICP22 created new session: [2:7577789276225015275:2110]
2025-11-28T14:52:34.257500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273524129332:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:34.253067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276225015275:2110] [node 6] ICS09 handshake done sender: [2:7577789276225015251:2117] self: [2:7577789276225015249:0] peer: [6:7577789276247755948:0] socket: 608 qp: -1
2025-11-28T14:52:34.255268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755953:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
2025-11-28T14:52:34.254854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:34.253239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276225015275:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:34.257768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789276247755952:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.255030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277131631851:2049] [node 2] ICP22 created new session: [5:7577789277131631940:2049]
2025-11-28T14:52:34.253327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.255209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277131631940:2049] [node 2] ICS09 handshake done sender: [5:7577789277131631932:2118] self: [5:7577789277131631935:0] peer: [2:7577789276225015243:0] socket: 637 qp: -1
2025-11-28T14:52:34.258018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789273524129325:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.255290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277131631940:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:34.258046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:34.255318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:34.258121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273524129325:2109] [node 6] ICP22 created new session: [4:7577789273524129351:2109]
2025-11-28T14:52:34.258164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789276247755947:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.258155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273524129351:2109] [node 6] ICS09 handshake done sender: [4:7577789273524129332:2115] self: [4:7577789273524129331:0] peer: [6:7577789276247755954:0] socket: 647 qp: -1
2025-11-28T14:52:34.256700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277131631939:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17139
2025-11-28T14:52:34.258171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789277131631852:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.258178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273524129351:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:34.258218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.258217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.258265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277131631852:2050] [node 3] ICP22 created new session: [5:7577789277131631942:2050]
2025-11-28T14:52:34.258298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789277131631850:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13896) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789277131631931:2117] held: no
2025-11-28T14:52:34.258305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277131631942:2050] [node 3] ICS09 handshake done sender: [5:7577789277131631930:2116] self: [5:7577789277131631933:0] peer: [3:7577789275612834534:0] socket: 636 qp: -1
2025-11-28T14:52:34.258384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277131631942:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:34.258414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.258489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789276247755868:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.258437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789277131631919:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.258536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.258574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789276247755868:2050] [node 3] ICP22 created new session: [6:7577789276247755959:2050]
2025-11-28T14:52:34.258620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789276247755959:2050] [node 3] ICS09 handshake done sender: [6:7577789276247755947:2118] self: [6:7577789276247755949:0] peer: [3:7577789275612834536:0] socket: 642 qp: -1
2025-11-28T14:52:34.258632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277131631931:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.258649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789276247755959:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.258682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789276247755866:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.258704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.258823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273524129334:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.258981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789277131631850:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.258843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789273524129334:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:52:34.259354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755957:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
::ffff:127.1.0.0
2025-11-28T14:52:34.259014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.259375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277131631850:2048] [node 1] ICP22 created new session: [5:7577789277131631947:2048]
2025-11-28T14:52:34.259405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277131631947:2048] [node 1] ICS09 handshake done sender: [5:7577789277131631931:2117] self: [5:7577789277131631934:0] peer: [1:7577789276964891800:0] socket: 590 qp: -1
2025-11-28T14:52:34.259430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277131631947:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:34.259467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789275612834566:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39751
2025-11-28T14:52:34.259451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273524129353:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38681
2025-11-28T14:52:34.259456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.259633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273524129334:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.259715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276964891807:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.259751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276964891807:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.261763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755962:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
2025-11-28T14:52:34.262063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276964891807:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:34.261711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789273524129324:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.261814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789276247755950:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:34.262339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789276247755930:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.262142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.262366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276964891793:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.262177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273524129324:2108] [node 5] ICP22 created new session: [4:7577789273524129355:2108]
2025-11-28T14:52:34.262418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:34.262213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273524129355:2108] [node 5] ICS09 handshake done sender: [4:7577789273524129334:2116] self: [4:7577789273524129333:0] peer: [5:7577789277131631946:0] socket: 601 qp: -1
2025-11-28T14:52:34.263177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.262237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273524129355:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:34.263226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789276247755930:2106] [node 4] ICP22 created new session: [6:7577789276247755965:2106]
2025-11-28T14:52:34.262605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276964891793:2109] [node 6] ICP22 created new session: [1:7577789276964891832:2109]
2025-11-28T14:52:34.262448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273524129353:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.263084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276964891832:2109] [node 6] ICS09 handshake done sender: [1:7577789276964891807:2115] self: [1:7577789276964891805:0] peer: [6:7577789276247755960:0] socket: 598 qp: -1
2025-11-28T14:52:34.263527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789276247755965:2106] [node 4] ICS09 handshake done sender: [6:7577789276247755950:2119] self: [6:7577789276247755954:0] peer: [4:7577789273524129331:0] socket: 648 qp: -1
2025-11-28T14:52:34.262477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.263169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276964891832:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:34.263520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789273524129324:2108] [node 5] ICP18 handshake (actor [4:7577789273524129353:2121]) is too old
2025-11-28T14:52:34.263601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789276247755965:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:34.263195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.263583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277131631949:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17139
2025-11-28T14:52:34.263647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.263813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273524129356:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38681
2025-11-28T14:52:34.264028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755964:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
2025-11-28T14:52:34.264086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277131631938:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:34.264430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789277131631919:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6551) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789277131631938:2121] held: no
2025-11-28T14:52:34.264787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789277131631919:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.264671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789276247755866:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13896) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789276247755952:2121] held: no
2025-11-28T14:52:34.265372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:34.265423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789276247755962:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.265427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277131631919:2108] [node 4] ICP22 created new session: [5:7577789277131631950:2108]
2025-11-28T14:52:34.265472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277131631950:2108] [node 4] ICS09 handshake done sender: [5:7577789277131631938:2121] self: [5:7577789277131631946:0] peer: [4:7577789273524129333:0] socket: 651 qp: -1
2025-11-28T14:52:34.265509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277131631950:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:34.265662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:34.265654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789276247755952:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.265779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789276247755866:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.265825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:34.265919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789276247755866:2048] [node 1] ICP22 created new session: [6:7577789276247755967:2048]
2025-11-28T14:52:34.266027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789276247755967:2048] [node 1] ICS09 handshake done sender: [6:7577789276247755952:2121] self: [6:7577789276247755960:0] peer: [1:7577789276964891805:0] socket: 653 qp: -1
2025-11-28T14:52:34.266051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789276247755925:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.266064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789276247755967:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:34.266095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.266266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277131631928:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.266294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789277131631928:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.267215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277131631928:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.267361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789276247755970:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21085
2025-11-28T14:52:34.267396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789277131631920:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.267450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:34.267519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277131631920:2109] [node 6] ICP22 created new session: [5:7577789277131631953:2109]
2025-11-28T14:52:34.267569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277131631953:2109] [node 6] ICS09 handshake done sender: [5:7577789277131631928:2115] self: [5:7577789277131631927:0] peer: [6:7577789276247755968:0] socket: 614 qp: -1
2025-11-28T14:52:34.267572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789276247755962:2124] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:34.267603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277131631953:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:34.267637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.267948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789276247755925:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.268030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.268058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789276247755925:2105] [node 5] ICP22 created new session: [6:7577789276247755971:2105]
2025-11-28T14:52:34.268096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789276247755971:2105] [node 5] ICS09 handshake done sender: [6:7577789276247755962:2124] self: [6:7577789276247755968:0] peer: [5:7577789277131631927:0] socket: 655 qp: -1
2025-11-28T14:52:34.268123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789276247755971:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:34.268199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.268950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789276247755925:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:34.312753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:34.312909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789276225015271:2048] [node 1] ICS07 socket disconnect 623 reason# EndOfStream
2025-11-28T14:52:34.312934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789276225015271:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.313016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.313062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789276225015271:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:34.313718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:34.313855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789275612834559:2048] [node 1] ICS01 socket: 620 reason# EndOfStream
2025-11-28T14:52:34.313890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789275612834462:2048] [node 1] ICP30 unregister session Session# [3:7577789275612834559:2048] VirtualId# [3:7577789275612834553:0]
2025-11-28T14:52:34.313910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789275612834559:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.313976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.314038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.314828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789276225015170:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13896) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:34.314856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789276225015271:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:34.315802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789276225015170:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13896) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.315840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789276225015271:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.315933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789276225015170:2048] [node 1] ICP30 unregister session Session# [2:7577789276225015271:2048] VirtualId# [2:7577789276225015265:0]
2025-11-28T14:52:34.315983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.316009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789276225015170:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13896) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.316021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13896) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.316024Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:34.316158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789273524129348:2048] [node 1] ICS01 socket: 624 reason# EndOfStream
2025-11-28T14:52:34.316187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789273524129255:2048] [node 1] ICP30 unregister session Session# [4:7577789273524129348:2048] VirtualId# [4:7577789273524129340:0]
2025-11-28T14:52:34.316207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789273524129348:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.316273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.316316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.316810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:34.316938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789277131631947:2048] [node 1] ICS01 socket: 590 reason# EndOfStream
2025-11-28T14:52:34.316966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789277131631850:2048] [node 1] ICP30 unregister session Session# [5:7577789277131631947:2048] VirtualId# [5:7577789277131631934:0]
2025-11-28T14:52:34.316984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789277131631947:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.317031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.317058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.317556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:34.317783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789276247755967:2048] [node 1] ICS01 socket: 653 reason# EndOfStream
2025-11-28T14:52:34.317846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789276247755866:2048] [node 1] ICP30 unregister session Session# [6:7577789276247755967:2048] VirtualId# [6:7577789276247755960:0]
2025-11-28T14:52:34.317883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789276247755967:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.317950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.318006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.326177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789276225015170:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:34.378695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:34.378855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789275612834554:2049] [node 2] ICS01 socket: 599 reason# EndOfStream
2025-11-28T14:52:34.378893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789275612834463:2049] [node 2] ICP30 unregister session Session# [3:7577789275612834554:2049] VirtualId# [3:7577789275612834547:0]
2025-11-28T14:52:34.378915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789275612834554:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.378993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:34.379060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:34.380667Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:34.380834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789273524129343:2049] [node 2] ICS01 socket: 606 reason# EndOfStream
2025-11-28T14:52:34.380870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789273524129256:2049] [node 2] ICP30 unregister session Session# [4:7577789273524129343:2049] VirtualId# [4:7577789273524129337:0]
2025-11-28T14:52:34.380898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789273524129343:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.380986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:34.381037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:34.381334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:34.381552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789277131631940:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-11-28T14:52:34.381599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789277131631851:2049] [node 2] ICP30 unregister session Session# [5:7577789277131631940:2049] VirtualId# [5:7577789277131631935:0]
2025-11-28T14:52:34.381639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789277131631940:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.381720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:34.381766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:34.382084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:34.382266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789276247755955:2049] [node 2] ICS07 socket disconnect 638 reason# EndOfStream
2025-11-28T14:52:34.382315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789276247755955:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.382380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:34.382424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789276247755955:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:34.383196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789276247755867:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28496) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.383310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789276247755955:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.383343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789276247755867:2049] [node 2] ICP30 unregister session Session# [6:7577789276247755955:2049] VirtualId# [6:7577789276247755948:0]
2025-11-28T14:52:34.383393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:34.383434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789276247755867:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28496) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.383448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28496) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.393773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789276247755867:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:34.430248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:34.430421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789273524129342:2050] [node 3] ICS07 socket disconnect 603 reason# EndOfStream
2025-11-28T14:52:34.430446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789273524129342:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.430533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:34.430581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789273524129342:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:34.431054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:34.431197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789277131631942:2050] [node 3] ICS01 socket: 636 reason# EndOfStream
2025-11-28T14:52:34.431230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789277131631852:2050] [node 3] ICP30 unregister session Session# [5:7577789277131631942:2050] VirtualId# [5:7577789277131631933:0]
2025-11-28T14:52:34.431253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789273524129257:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18331) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.431251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789277131631942:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.431292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789273524129342:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.431311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:34.431318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789273524129257:2050] [node 3] ICP30 unregister session Session# [4:7577789273524129342:2050] VirtualId# [4:7577789273524129336:0]
2025-11-28T14:52:34.431350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:34.431365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:34.431385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789273524129257:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18331) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.431497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18331) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.431888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:34.432038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789276247755959:2050] [node 3] ICS07 socket disconnect 642 reason# EndOfStream
2025-11-28T14:52:34.432062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789276247755959:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.432119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:34.432165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789276247755959:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:34.432662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789276247755868:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18331) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.432699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789276247755959:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.432724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789276247755868:2050] [node 3] ICP30 unregister session Session# [6:7577789276247755959:2050] VirtualId# [6:7577789276247755949:0]
2025-11-28T14:52:34.432758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:34.432781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789276247755868:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18331) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.432789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18331) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.442912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789276247755868:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:34.451194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:34.451352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789277131631950:2108] [node 4] ICS01 socket: 651 reason# EndOfStream
2025-11-28T14:52:34.451387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789277131631919:2108] [node 4] ICP30 unregister session Session# [5:7577789277131631950:2108] VirtualId# [5:7577789277131631946:0]
2025-11-28T14:52:34.451409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789277131631950:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.451593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:34.451669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:34.452070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:34.452220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789276247755965:2106] [node 4] ICS07 socket disconnect 648 reason# EndOfStream
2025-11-28T14:52:34.452246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789276247755965:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.452331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:34.452398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789276247755965:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:34.452987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789276247755930:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6551) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.453057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789276247755965:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.453105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789276247755930:2106] [node 4] ICP30 unregister session Session# [6:7577789276247755965:2106] VirtualId# [6:7577789276247755954:0]
2025-11-28T14:52:34.453142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:34.453175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789276247755930:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.453193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.463401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789276247755930:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:34.501831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:34.501990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789276247755971:2105] [node 5] ICS07 socket disconnect 655 reason# EndOfStream
2025-11-28T14:52:34.502022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789276247755971:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.502116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:34.502156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789276247755971:2105] [node 5] ICS15 start handshake
2025-11-28T14:52:34.502779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789276247755925:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62274) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.502866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789276247755971:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.502903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789276247755925:2105] [node 5] ICP30 unregister session Session# [6:7577789276247755971:2105] VirtualId# [6:7577789276247755968:0]
2025-11-28T14:52:34.502941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:34.502965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789276247755925:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.502974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:34.588538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276351811526:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:34.588617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276351811527:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:34.610231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276298843562:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.610483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276298843563:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:34.627747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277634533643:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:34.627757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277634533642:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.644556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273486444255:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.644603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273486444256:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:34.644643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273486444257:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:34.665882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277197911267:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:34.665891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277197911268:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:34.665893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277197911266:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.682272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789274108520938:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:34.682311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789274108520939:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:34.682340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789274108520940:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:34.723096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789274108520999:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:34.746623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276351811588:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:34.767751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277197911329:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:34.757626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276351811590:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:34.767771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276298843626:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:34.769652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789277197911330:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:34.767738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789276351811591:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:34.770043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276298843627:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:34.770398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277634533706:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:34.770427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273486444318:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:34.770427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789276298843628:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:34.770402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277634533705:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:34.770325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789273486444319:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:34.770845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277634533707:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:34.771324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789274108521002:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:34.771918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789274108520999:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(48595996ns) {Configure:104(48593349ns) {ProcessConfigured:135(188825ns) {StartInitialHandshake:157(187001ns) {DropHandshakes:470(1208ns) {DropIncomingHandshake:442(307ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:477(836ns) {SwitchToState:179(268ns)}}}}}
2025-11-28T14:52:34.772438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789276351811527:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49327873ns) {Configure:104(49322233ns) {ProcessConfigured:135(288575ns) {StartInitialHandshake:157(286554ns) {DropHandshakes:470(4290ns) {DropIncomingHandshake:442(540ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:477(722ns) {SwitchToState:179(301ns)}}}}}
2025-11-28T14:52:34.772626Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789276351811526:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49293768ns) {Configure:104(49289573ns) {ProcessConfigured:135(281183ns) {StartInitialHandshake:157(279142ns) {DropHandshakes:470(1189ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(3014ns) PrepareNewSessionHandshake:477(629ns) {SwitchToState:179(139ns)}}}}}
2025-11-28T14:52:34.773656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789274108520940:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26404765ns) {Configure:104(26397070ns) {ProcessConfigured:135(240935ns) {StartInitialHandshake:157(238898ns) {DropHandshakes:470(1818ns) {DropIncomingHandshake:442(456ns) DropOutgoingHandshake:457(607ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(833ns) {SwitchToState:179(343ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.774466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789276351811603:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18713
2025-11-28T14:52:34.776642Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789273486444256:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(42578401ns) {Configure:104(42569681ns) {ProcessConfigured:135(184853ns) {StartInitialHandshake:157(182645ns) {DropHandshakes:470(1241ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(664ns) PrepareNewSessionHandshake:477(1664ns) {SwitchToState:179(666ns)}}}}}
2025-11-28T14:52:34.776665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789273486444255:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42707783ns) {Configure:104(42704510ns) {ProcessConfigured:135(176869ns) {StartInitialHandshake:157(174672ns) {DropHandshakes:470(720ns) {DropIncomingHandshake:442(194ns) DropOutgoingHandshake:457(141ns)} GenerateSessionVirtualId:988(318ns) PrepareNewSessionHandshake:477(585ns) {SwitchToState:179(230ns)}}}}}
2025-11-28T14:52:34.779643Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789277197911267:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43691517ns) {Configure:104(43687008ns) {ProcessConfigured:135(528820ns) {StartInitialHandshake:157(526940ns) {DropHandshakes:470(1045ns) {DropIncomingHandshake:442(282ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(484ns) PrepareNewSessionHandshake:477(1046ns) {SwitchToState:179(479ns)}}}}}
2025-11-28T14:52:34.779576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789276298843563:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19305845ns) {Configure:104(19302213ns) {ProcessConfigured:135(305232ns) {StartInitialHandshake:157(302924ns) {DropHandshakes:470(1203ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(442ns) PrepareNewSessionHandshake:477(1031ns) {SwitchToState:179(444ns)}}}}}
2025-11-28T14:52:34.780228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789277634533642:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38663841ns) {Configure:104(38658418ns) {ProcessConfigured:135(570491ns) {StartInitialHandshake:157(568671ns) {DropHandshakes:470(1107ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(433ns) PrepareNewSessionHandshake:477(465633ns) {SwitchToState:179(1916ns)}}}}}
2025-11-28T14:52:34.780545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789277634533643:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(38126611ns) {Configure:104(38120260ns) {ProcessConfigured:135(199173ns) {StartInitialHandshake:157(197340ns) {DropHandshakes:470(908ns) {DropIncomingHandshake:442(270ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(391ns) PrepareNewSessionHandshake:477(2950ns) {SwitchToState:179(106ns)}}}}}
2025-11-28T14:52:34.782108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789277197911266:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44527898ns) {Configure:104(44525240ns) {ProcessConfigured:135(942266ns) {StartInitialHandshake:157(940713ns) {DropHandshakes:470(1231ns) {DropIncomingHandshake:442(250ns) DropOutgoingHandshake:457(405ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:477(289512ns) {SwitchToState:179(401ns)}}}}}
2025-11-28T14:52:34.782140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789277197911268:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(38431596ns) {Configure:104(38421823ns) {ProcessConfigured:135(478687ns) {StartInitialHandshake:157(477517ns) {DropHandshakes:470(745ns) {DropIncomingHandshake:442(225ns) DropOutgoingHandshake:457(196ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(795ns) {SwitchToState:179(411ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.783038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276351811603:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.783084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276298843637:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2654
2025-11-28T14:52:34.783503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277634533717:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49256
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.783747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277634533716:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49256
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.784779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276298843637:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.784252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277634533717:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.785769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276298843641:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2654
2025-11-28T14:52:34.785235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277634533716:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.786120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277197911339:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53850
2025-11-28T14:52:34.785465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277634533718:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49256
2025-11-28T14:52:34.786348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276351811594:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.786141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789276298843562:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.787092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277197911340:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53850
2025-11-28T14:52:34.787930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521016:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
2025-11-28T14:52:34.786386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276351811594:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.785672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789277634533642:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.789450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277197911341:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53850
2025-11-28T14:52:34.787944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276351811595:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.789399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273486444333:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1838
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.789979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276298843644:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2654
2025-11-28T14:52:34.789460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277197911340:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.786115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277634533722:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49256
2025-11-28T14:52:34.790572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276298843641:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.787962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276351811595:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.788161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276351811594:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.789582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277197911339:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:34.790570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277634533718:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:34.792027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789274108521016:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.791305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273486444333:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.790632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277197911341:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.792691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789274108520939:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.792034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789273486444256:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.791396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789277197911329:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.791105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789276351811607:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18713
2025-11-28T14:52:34.794152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277634533728:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49256
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.792790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273486444327:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.796213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277634533722:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.794163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276298843635:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.791503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789277197911266:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.791453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276351811595:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.792807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789273486444327:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.794189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789276298843635:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.796769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277634533727:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49256
2025-11-28T14:52:34.792245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277197911342:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53850
2025-11-28T14:52:34.791637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276351811527:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.793251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273486444334:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1838
2025-11-28T14:52:34.794197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276298843634:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.795287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277197911342:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.791677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.795329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273486444327:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:34.796804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789277634533643:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.794226Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789276298843634:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.795522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789277197911268:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.797055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277634533713:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.792067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276351811527:2049] [node 3] ICP22 created new session: [1:7577789276351811610:2049]
2025-11-28T14:52:34.797083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273486444337:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1838
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.799684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789273486444318:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.797235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276298843635:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.797077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789277634533713:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.792670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276351811610:2049] [node 3] ICS09 handshake done sender: [1:7577789276351811594:2110] self: [1:7577789276351811592:0] peer: [3:7577789277634533723:0] socket: 595 qp: -1
2025-11-28T14:52:34.800563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277634533716:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.799720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.800610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277634533713:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:34.797422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276298843631:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.799846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273486444318:2108] [node 5] ICP22 created new session: [4:7577789273486444339:2108]
2025-11-28T14:52:34.801561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521019:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
2025-11-28T14:52:34.801615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789277634533642:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:34.800121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273486444339:2108] [node 5] ICS09 handshake done sender: [4:7577789273486444327:2113] self: [4:7577789273486444325:0] peer: [5:7577789277197911350:0] socket: 610 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:34.797444Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789276298843631:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.801660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:34.800648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273486444334:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.792807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276351811610:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:34.801947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521018:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.801979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277197911353:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53850
2025-11-28T14:52:34.801981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277634533642:2048] [node 1] ICP22 created new session: [3:7577789277634533732:2048]
2025-11-28T14:52:34.797812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276298843646:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2654
2025-11-28T14:52:34.793262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276351811597:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.801365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273486444339:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:34.802018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277634533732:2048] [node 1] ICS09 handshake done sender: [3:7577789277634533716:2114] self: [3:7577789277634533723:0] peer: [1:7577789276351811592:0] socket: 604 qp: -1
2025-11-28T14:52:34.798548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789276298843647:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2654
2025-11-28T14:52:34.801979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277197911352:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53850
2025-11-28T14:52:34.793264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.801400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.802240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277634533732:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:34.798723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276298843634:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:34.803024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277197911354:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53850
2025-11-28T14:52:34.802040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273486444337:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.804316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789274108521019:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.800536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276298843644:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.802565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273486444338:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1838
2025-11-28T14:52:34.802262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.804558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789274108521016:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:34.801136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789276298843626:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.804403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789273486444255:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.793284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276351811597:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.803010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277634533728:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.806057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277197911341:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:34.801390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789276298843627:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.804648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789274108520938:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.804485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789273486444256:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24074) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789273486444333:2115] held: no
2025-11-28T14:52:34.803043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789277634533705:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.793685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276351811526:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.801811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:34.806459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789274108520939:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.806679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277197911340:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.801840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276298843627:2107] [node 4] ICP22 created new session: [2:7577789276298843652:2107]
2025-11-28T14:52:34.804988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273486444333:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:34.807198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789277197911329:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.806749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:34.794194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.801841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276298843646:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.803349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.806687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789273486444318:2108] [node 5] ICP18 handshake (actor [4:7577789273486444334:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.807423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277197911342:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.807483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789274108520939:2049] [node 2] ICP22 created new session: [6:7577789274108521021:2049]
2025-11-28T14:52:34.795517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276351811526:2048] [node 2] ICP22 created new session: [1:7577789276351811613:2048]
2025-11-28T14:52:34.803764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277634533705:2108] [node 5] ICP22 created new session: [3:7577789277634533734:2108]
2025-11-28T14:52:34.796479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276351811613:2048] [node 2] ICS09 handshake done sender: [1:7577789276351811595:2111] self: [1:7577789276351811593:0] peer: [2:7577789276298843645:0] socket: 597 qp: -1
2025-11-28T14:52:34.801847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276298843637:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.809375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789274108521021:2049] [node 2] ICS09 handshake done sender: [6:7577789274108521016:2115] self: [6:7577789274108521017:0] peer: [2:7577789276298843630:0] socket: 619 qp: -1
2025-11-28T14:52:34.807490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:34.807477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789273486444256:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.796495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276351811607:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.807644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789277197911266:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.801977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:34.804592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277634533734:2108] [node 5] ICS09 handshake done sender: [3:7577789277634533713:2111] self: [3:7577789277634533708:0] peer: [5:7577789277197911351:0] socket: 620 qp: -1
2025-11-28T14:52:34.809678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789274108521021:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:34.807645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:34.796519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276351811613:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:34.804619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277634533734:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:34.807686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:34.805788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.810813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:34.802002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276298843652:2107] [node 4] ICS09 handshake done sender: [2:7577789276298843634:2112] self: [2:7577789276298843632:0] peer: [4:7577789273486444335:0] socket: 613 qp: -1
2025-11-28T14:52:34.809390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273486444256:2049] [node 2] ICP22 created new session: [4:7577789273486444344:2049]
2025-11-28T14:52:34.809885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277634533735:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49256
2025-11-28T14:52:34.807685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277197911329:2108] [node 4] ICP22 created new session: [5:7577789277197911355:2108]
2025-11-28T14:52:34.811199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789277634533705:2108] [node 5] ICP18 handshake (actor [3:7577789277634533728:2120]) is too old
2025-11-28T14:52:34.802258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276298843652:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:34.809925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273486444344:2049] [node 2] ICS09 handshake done sender: [4:7577789273486444333:2115] self: [4:7577789273486444335:0] peer: [2:7577789276298843632:0] socket: 616 qp: -1
2025-11-28T14:52:34.807728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277197911266:2048] [node 1] ICP22 created new session: [5:7577789277197911356:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.802562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276298843626:2110] [node 6] ICP22 created new session: [2:7577789276298843653:2110]
2025-11-28T14:52:34.809418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277197911355:2108] [node 4] ICS09 handshake done sender: [5:7577789277197911341:2116] self: [5:7577789277197911350:0] peer: [4:7577789273486444325:0] socket: 615 qp: -1
2025-11-28T14:52:34.810826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273486444344:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:34.809678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277197911356:2048] [node 1] ICS09 handshake done sender: [5:7577789277197911340:2115] self: [5:7577789277197911349:0] peer: [1:7577789276351811596:0] socket: 607 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:34.811094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:34.796557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:34.814591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521024:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
2025-11-28T14:52:34.802597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:34.809695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277197911356:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:34.814588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273486444345:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1838
2025-11-28T14:52:34.799477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276351811597:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:34.814618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521022:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
2025-11-28T14:52:34.813726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277634533722:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:34.801842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276351811590:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.802012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.809709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277197911355:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:34.803323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276298843653:2110] [node 6] ICS09 handshake done sender: [2:7577789276298843635:2113] self: [2:7577789276298843630:0] peer: [6:7577789274108521017:0] socket: 618 qp: -1
2025-11-28T14:52:34.802335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789276351811527:2049] [node 3] ICP18 handshake (actor [1:7577789276351811607:2116]) is too old
2025-11-28T14:52:34.815072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273486444347:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1838
2025-11-28T14:52:34.814816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789277634533643:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.810835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.802563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276351811590:2108] [node 5] ICP22 created new session: [1:7577789276351811617:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.803731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276298843653:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:34.815060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:34.811453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:34.803713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276351811617:2108] [node 5] ICS09 handshake done sender: [1:7577789276351811597:2112] self: [1:7577789276351811596:0] peer: [5:7577789277197911349:0] socket: 598 qp: -1
2025-11-28T14:52:34.815100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277634533643:2049] [node 2] ICP22 created new session: [3:7577789277634533739:2049]
2025-11-28T14:52:34.812079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789277197911268:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.804453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.815138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277634533739:2049] [node 2] ICS09 handshake done sender: [3:7577789277634533722:2117] self: [3:7577789277634533730:0] peer: [2:7577789276298843629:0] socket: 624 qp: -1
2025-11-28T14:52:34.812991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.817064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789274108521022:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.816823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273486444348:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1838
2025-11-28T14:52:34.815283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277634533739:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:34.813146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277197911268:2050] [node 3] ICP22 created new session: [5:7577789277197911359:2050]
2025-11-28T14:52:34.804303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276351811617:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:34.804595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789276298843562:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.816857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273486444337:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.815419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:34.813393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277197911359:2050] [node 3] ICS09 handshake done sender: [5:7577789277197911342:2117] self: [5:7577789277197911351:0] peer: [3:7577789277634533708:0] socket: 622 qp: -1
2025-11-28T14:52:34.813902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277197911359:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:34.817598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789274108520938:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789274108521019:2117] held: no
2025-11-28T14:52:34.817652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789273486444255:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.814312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.805770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:34.817977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789274108521024:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:34.817973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273486444345:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.804463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.806627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276298843562:2048] [node 1] ICP22 created new session: [2:7577789276298843658:2048]
::ffff:127.1.0.0
2025-11-28T14:52:34.818439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789277634533643:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:34.806665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276298843658:2048] [node 1] ICS09 handshake done sender: [2:7577789276298843637:2115] self: [2:7577789276298843645:0] peer: [1:7577789276351811593:0] socket: 603 qp: -1
2025-11-28T14:52:34.818351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:34.807454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276351811599:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.818408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273486444255:2048] [node 1] ICP22 created new session: [4:7577789273486444349:2048]
2025-11-28T14:52:34.806691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276298843658:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:34.807474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276351811599:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.818413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273486444348:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.806725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.818935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789274108521002:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.806957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276298843631:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.818964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521026:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
2025-11-28T14:52:34.818458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273486444349:2048] [node 1] ICS09 handshake done sender: [4:7577789273486444337:2117] self: [4:7577789273486444342:0] peer: [1:7577789276351811598:0] socket: 641 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:34.807628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276351811605:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:34.812470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789276298843563:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.807652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789276351811605:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.812520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.818798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273486444349:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:34.818838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.819744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277197911361:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53850
2025-11-28T14:52:34.813145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276298843563:2049] [node 3] ICP22 created new session: [2:7577789276298843661:2049]
2025-11-28T14:52:34.819013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789273486444257:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.819754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789274108520940:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.809943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789276351811618:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18713
2025-11-28T14:52:34.819811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277634533714:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.813410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276298843661:2049] [node 3] ICS09 handshake done sender: [2:7577789276298843631:2111] self: [2:7577789276298843629:0] peer: [3:7577789277634533730:0] socket: 614 qp: -1
2025-11-28T14:52:34.819781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789274108521019:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:34.819797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789273486444332:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.819842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789277634533714:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.813440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276298843661:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:34.811418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276351811599:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:34.819819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789273486444332:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.821296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277634533725:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.813100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789276351811605:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:34.813769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.821341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789274108520938:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.821612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277197911361:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.821314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789277634533725:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.821886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273486444332:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:34.813386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276351811588:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.821820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:34.821947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277634533714:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:34.813735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:34.822431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789274108520938:2048] [node 1] ICP22 created new session: [6:7577789274108521030:2048]
2025-11-28T14:52:34.822675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789277197911267:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.822471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789274108521030:2048] [node 1] ICS09 handshake done sender: [6:7577789274108521019:2117] self: [6:7577789274108521020:0] peer: [1:7577789276351811604:0] socket: 649 qp: -1
2025-11-28T14:52:34.822709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789277634533706:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.814122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789276351811618:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:34.822498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789274108521030:2048] [node 1] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:34.822656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:34.814158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789276351811591:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 2025-11-28T14:52:34.822754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
::ffff:127.1.0.0
2025-11-28T14:52:34.814705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276351811588:2107] [node 4] ICP22 created new session: [1:7577789276351811622:2107]
2025-11-28T14:52:34.823117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789276298843643:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.823081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277634533725:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:34.814743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:34.823140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789273486444319:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.823149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789276298843643:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.823206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:34.823173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521029:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
2025-11-28T14:52:34.823083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277634533706:2107] [node 4] ICP22 created new session: [3:7577789277634533743:2107]
2025-11-28T14:52:34.823238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273486444319:2109] [node 6] ICP22 created new session: [4:7577789273486444355:2109]
2025-11-28T14:52:34.815163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276351811622:2107] [node 4] ICS09 handshake done sender: [1:7577789276351811599:2113] self: [1:7577789276351811598:0] peer: [4:7577789273486444342:0] socket: 640 qp: -1
2025-11-28T14:52:34.823277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521031:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
2025-11-28T14:52:34.824201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789273486444354:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1838
2025-11-28T14:52:34.823121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277634533743:2107] [node 4] ICS09 handshake done sender: [3:7577789277634533714:2112] self: [3:7577789277634533711:0] peer: [4:7577789273486444351:0] socket: 609 qp: -1
2025-11-28T14:52:34.815180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789276351811591:2109] [node 6] ICP22 created new session: [1:7577789276351811623:2109]
2025-11-28T14:52:34.824524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273486444355:2109] [node 6] ICS09 handshake done sender: [4:7577789273486444332:2114] self: [4:7577789273486444331:0] peer: [6:7577789274108521027:0] socket: 626 qp: -1
2025-11-28T14:52:34.823167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277634533743:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:34.824552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273486444355:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:34.824593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.824202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789277634533707:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.815417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276351811622:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:34.824607Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789276298843643:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:34.815431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789276351811623:2109] [node 6] ICS09 handshake done sender: [1:7577789276351811605:2115] self: [1:7577789276351811604:0] peer: [6:7577789274108521020:0] socket: 648 qp: -1
2025-11-28T14:52:34.824229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:34.815455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:34.824710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789274108521029:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.824348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:34.825205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789276298843628:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:34.826325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789273486444257:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789273486444345:2119] held: no
2025-11-28T14:52:34.815464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789276351811623:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:34.825146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789274108521022:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.826349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.824555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277634533707:2109] [node 6] ICP22 created new session: [3:7577789277634533745:2109]
2025-11-28T14:52:34.827191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789273486444345:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.816518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.826407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789276298843628:2108] [node 5] ICP22 created new session: [2:7577789276298843665:2108]
2025-11-28T14:52:34.826325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789274108521002:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11783) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789274108521022:2118] held: no
2025-11-28T14:52:34.824609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277634533745:2109] [node 6] ICS09 handshake done sender: [3:7577789277634533725:2118] self: [3:7577789277634533724:0] peer: [6:7577789274108521028:0] socket: 657 qp: -1
2025-11-28T14:52:34.827607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789273486444257:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.828649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789277197911267:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24074) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789277197911361:2122] held: no
2025-11-28T14:52:34.826479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789274108521002:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.827390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789276298843665:2108] [node 5] ICS09 handshake done sender: [2:7577789276298843643:2117] self: [2:7577789276298843642:0] peer: [5:7577789277197911363:0] socket: 629 qp: -1
2025-11-28T14:52:34.817574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789276351811588:2107] [node 4] ICP18 handshake (actor [1:7577789276351811618:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.827240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:34.828657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.827528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789276298843665:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:34.824653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277634533745:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:34.829276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789273486444257:2050] [node 3] ICP22 created new session: [4:7577789273486444357:2050]
2025-11-28T14:52:34.827412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789274108521002:2108] [node 4] ICP22 created new session: [6:7577789274108521034:2108]
2025-11-28T14:52:34.828666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.824674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.829745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789273486444357:2050] [node 3] ICS09 handshake done sender: [4:7577789273486444345:2119] self: [4:7577789273486444351:0] peer: [3:7577789277634533711:0] socket: 634 qp: -1
2025-11-28T14:52:34.827535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789274108521034:2108] [node 4] ICS09 handshake done sender: [6:7577789274108521022:2118] self: [6:7577789274108521027:0] peer: [4:7577789273486444331:0] socket: 627 qp: -1
2025-11-28T14:52:34.830049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789277197911364:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53850
2025-11-28T14:52:34.827557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789274108521034:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:34.829775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789273486444357:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:34.829820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.827576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:34.829611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521033:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
2025-11-28T14:52:34.830018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789274108520999:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:34.830466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277197911361:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:34.830313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789274108520940:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789274108521024:2119] held: no
2025-11-28T14:52:34.830658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789274108521024:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:34.830732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789277197911267:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.830748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789277197911344:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:34.830767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789277197911344:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:34.830835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789274108520940:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.830879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:34.830772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:34.830816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277197911267:2049] [node 2] ICP22 created new session: [5:7577789277197911365:2049]
2025-11-28T14:52:34.830902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277197911365:2049] [node 2] ICS09 handshake done sender: [5:7577789277197911361:2122] self: [5:7577789277197911363:0] peer: [2:7577789276298843642:0] socket: 633 qp: -1
2025-11-28T14:52:34.830967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789274108520940:2050] [node 3] ICP22 created new session: [6:7577789274108521037:2050]
2025-11-28T14:52:34.830941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277197911365:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:34.831579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:34.831649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789274108521037:2050] [node 3] ICS09 handshake done sender: [6:7577789274108521024:2119] self: [6:7577789274108521028:0] peer: [3:7577789277634533724:0] socket: 658 qp: -1
2025-11-28T14:52:34.831691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789274108521037:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:34.831734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:34.831835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789277197911344:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:34.831964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789277197911330:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:34.832064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:34.832119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789277197911330:2109] [node 6] ICP22 created new session: [5:7577789277197911368:2109]
2025-11-28T14:52:34.832116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789274108521039:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5134
2025-11-28T14:52:34.832166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789277197911368:2109] [node 6] ICS09 handshake done sender: [5:7577789277197911344:2118] self: [5:7577789277197911343:0] peer: [6:7577789274108521036:0] socket: 596 qp: -1
2025-11-28T14:52:34.832199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789277197911368:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:34.832235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:34.832303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789274108521029:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:34.832485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789274108520999:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789274108521029:2121] held: no
2025-11-28T14:52:34.832638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789274108520999:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:34.832849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:34.832904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789274108520999:2105] [node 5] ICP22 created new session: [6:7577789274108521040:2105]
2025-11-28T14:52:34.832927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789274108521040:2105] [node 5] ICS09 handshake done sender: [6:7577789274108521029:2121] self: [6:7577789274108521036:0] peer: [5:7577789277197911343:0] socket: 599 qp: -1
2025-11-28T14:52:34.832947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789274108521040:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:34.832971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:34.881758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:34.881921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789276298843658:2048] [node 1] ICS07 socket disconnect 603 reason# EndOfStream
2025-11-28T14:52:34.881947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789276298843658:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.882019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.882055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789276298843658:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:34.883016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:34.883227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789277634533732:2048] [node 1] ICS07 socket disconnect 604 reason# EndOfStream
2025-11-28T14:52:34.883257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789277634533732:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.883330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.883426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789277634533732:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:34.884405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789276298843562:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:34.884440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789276298843658:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:34.885840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789277634533642:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:34.886160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789277634533732:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:34.885834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789276298843562:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.886274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789276298843658:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.886625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789276298843562:2048] [node 1] ICP30 unregister session Session# [2:7577789276298843658:2048] VirtualId# [2:7577789276298843645:0]
2025-11-28T14:52:34.886677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.886685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:34.886716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789276298843562:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.886731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.886956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789273486444349:2048] [node 1] ICS07 socket disconnect 641 reason# EndOfStream
2025-11-28T14:52:34.886979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789273486444349:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.887042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.887083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789273486444349:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:34.887347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789277634533642:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.887436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789277634533732:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.887466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789277634533642:2048] [node 1] ICP30 unregister session Session# [3:7577789277634533732:2048] VirtualId# [3:7577789277634533723:0]
2025-11-28T14:52:34.887508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.887537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789277634533642:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.887545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.887626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789273486444255:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.887692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789273486444349:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.887720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789273486444255:2048] [node 1] ICP30 unregister session Session# [4:7577789273486444349:2048] VirtualId# [4:7577789273486444342:0]
2025-11-28T14:52:34.887772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.887810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789273486444255:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.887823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.887818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:34.888060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789277197911356:2048] [node 1] ICS07 socket disconnect 607 reason# EndOfStream
2025-11-28T14:52:34.888099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789277197911356:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.888195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.888247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789277197911356:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:34.888648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:34.888819Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789277197911266:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.888853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789277197911356:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.888876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789277197911266:2048] [node 1] ICP30 unregister session Session# [5:7577789277197911356:2048] VirtualId# [5:7577789277197911349:0]
2025-11-28T14:52:34.888844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789274108521030:2048] [node 1] ICS01 socket: 649 reason# EndOfStream
2025-11-28T14:52:34.888912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.888942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789274108520938:2048] [node 1] ICP30 unregister session Session# [6:7577789274108521030:2048] VirtualId# [6:7577789274108521020:0]
2025-11-28T14:52:34.888933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789277197911266:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.888984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789274108521030:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.888941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6473) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.889060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:34.889113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:34.897561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789277634533642:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:34.897572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789273486444255:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:34.898583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789277197911266:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:34.932408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:34.932620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789277634533739:2049] [node 2] ICS07 socket disconnect 624 reason# EndOfStream
2025-11-28T14:52:34.932649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789277634533739:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.932727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:34.932771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789277634533739:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:34.934704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:34.934816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789277634533643:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24074) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:34.934875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789273486444344:2049] [node 2] ICS01 socket: 616 reason# EndOfStream
2025-11-28T14:52:34.934851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789277634533739:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:34.934907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789273486444256:2049] [node 2] ICP30 unregister session Session# [4:7577789273486444344:2049] VirtualId# [4:7577789273486444335:0]
2025-11-28T14:52:34.934928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789273486444344:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.935000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:34.935042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:34.935562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789277634533643:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24074) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.935608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789277634533739:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.935634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789277634533643:2049] [node 2] ICP30 unregister session Session# [3:7577789277634533739:2049] VirtualId# [3:7577789277634533730:0]
2025-11-28T14:52:34.935671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:34.935693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789277634533643:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.935705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.935850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:34.935993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789277197911365:2049] [node 2] ICS01 socket: 633 reason# EndOfStream
2025-11-28T14:52:34.936024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789277197911267:2049] [node 2] ICP30 unregister session Session# [5:7577789277197911365:2049] VirtualId# [5:7577789277197911363:0]
2025-11-28T14:52:34.936043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789277197911365:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.936112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:34.936157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:34.936517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:34.936750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789274108521021:2049] [node 2] ICS07 socket disconnect 619 reason# EndOfStream
2025-11-28T14:52:34.936785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789274108521021:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.936850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:34.936898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789274108521021:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:34.937636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789274108520939:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24074) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.937710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789274108521021:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.937748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789274108520939:2049] [node 2] ICP30 unregister session Session# [6:7577789274108521021:2049] VirtualId# [6:7577789274108521017:0]
2025-11-28T14:52:34.937795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:34.937834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789274108520939:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.937851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.947793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789274108520939:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:34.977166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:34.977350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789273486444357:2050] [node 3] ICS01 socket: 634 reason# EndOfStream
2025-11-28T14:52:34.977387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789273486444257:2050] [node 3] ICP30 unregister session Session# [4:7577789273486444357:2050] VirtualId# [4:7577789273486444351:0]
2025-11-28T14:52:34.977413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789273486444357:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.977504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:34.977544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:34.978139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:34.978356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789277197911359:2050] [node 3] ICS07 socket disconnect 622 reason# EndOfStream
2025-11-28T14:52:34.978409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789277197911359:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.978500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:34.978551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789277197911359:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:34.978838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:34.978998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789274108521037:2050] [node 3] ICS01 socket: 658 reason# EndOfStream
2025-11-28T14:52:34.979031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789274108520940:2050] [node 3] ICP30 unregister session Session# [6:7577789274108521037:2050] VirtualId# [6:7577789274108521028:0]
2025-11-28T14:52:34.979053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789274108521037:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:34.979126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:34.979172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:34.979280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789277197911268:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26816) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:34.979324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789277197911359:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:34.979376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789277197911268:2050] [node 3] ICP30 unregister session Session# [5:7577789277197911359:2050] VirtualId# [5:7577789277197911351:0]
2025-11-28T14:52:34.979415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:34.979439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789277197911268:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26816) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.979448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26816) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:34.989744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789277197911268:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:35.044521Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:35.044703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789277197911355:2108] [node 4] ICS07 socket disconnect 615 reason# EndOfStream
2025-11-28T14:52:35.044742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789277197911355:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.044833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:35.044872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789277197911355:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:35.045275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:35.045513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789274108521034:2108] [node 4] ICS01 socket: 627 reason# EndOfStream
2025-11-28T14:52:35.045558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789277197911329:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11783) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:35.045596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789277197911355:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:35.045688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789277197911329:2108] [node 4] ICP30 unregister session Session# [5:7577789277197911355:2108] VirtualId# [5:7577789277197911350:0]
2025-11-28T14:52:35.045578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789274108521002:2108] [node 4] ICP30 unregister session Session# [6:7577789274108521034:2108] VirtualId# [6:7577789274108521027:0]
2025-11-28T14:52:35.045624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789274108521034:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.045718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:35.045751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:35.045778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:35.045776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789277197911329:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.045785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.093327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:35.093544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789274108521040:2105] [node 5] ICS01 socket: 599 reason# EndOfStream
2025-11-28T14:52:35.093605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789274108520999:2105] [node 5] ICP30 unregister session Session# [6:7577789274108521040:2105] VirtualId# [6:7577789274108521036:0]
2025-11-28T14:52:35.093660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789274108521040:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.093744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:35.093790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:35.173549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789278282007556:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:35.173685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789278282007555:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:35.191672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789281213540087:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.191681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789281213540088:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:35.208860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277731171180:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.209090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277731171181:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:35.227212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789281522315586:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:35.227293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789281522315585:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.227489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789281522315587:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:35.247298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789278211446602:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.247484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789278211446603:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:35.247487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789278211446604:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:35.272090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789278941111601:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.272125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789278941111603:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:35.272357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789278941111602:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:35.348270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789278282007618:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:35.362424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789281213540150:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:35.362938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277731171244:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:35.363135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789281522315649:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:35.363150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789278282007619:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:35.363165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277731171245:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:35.363939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789278941111662:2105] [node 4] ICP01 ready to work
2025-11-28T14:52:35.363736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789281213540152:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:35.363795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789278211446665:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:35.363692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789278282007620:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:35.364027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789281522315650:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:35.363946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789278211446666:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:35.363988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789278941111664:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:35.364358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789277731171246:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:35.363692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789281213540151:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:35.368050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789278211446604:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40544537ns) {Configure:104(36187327ns) {ProcessConfigured:135(171896ns) {StartInitialHandshake:157(169947ns) {DropHandshakes:470(1332ns) {DropIncomingHandshake:442(411ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(733ns) PrepareNewSessionHandshake:477(827ns) {SwitchToState:179(356ns)}}}}}
2025-11-28T14:52:35.368089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789278211446602:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(41104560ns) {Configure:104(41098708ns) {ProcessConfigured:135(230602ns) {StartInitialHandshake:157(228480ns) {DropHandshakes:470(1044ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(419ns) PrepareNewSessionHandshake:477(803ns) {SwitchToState:179(339ns)}}}}}
2025-11-28T14:52:35.368102Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789278211446603:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36296946ns) {Configure:104(36291107ns) {ProcessConfigured:135(257828ns) {StartInitialHandshake:157(255648ns) {DropHandshakes:470(1647ns) {DropIncomingHandshake:442(545ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:477(1074ns) {SwitchToState:179(382ns)}}}}}
2025-11-28T14:52:35.373342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789281213540087:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52323154ns) {Configure:104(52314835ns) {ProcessConfigured:135(1375916ns) {StartInitialHandshake:157(1373863ns) {DropHandshakes:470(1633ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(1107ns) {SwitchToState:179(482ns)}}}}}
2025-11-28T14:52:35.374071Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789277731171181:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41830931ns) {Configure:104(41823519ns) {ProcessConfigured:135(1439906ns) {StartInitialHandshake:157(1437551ns) {DropHandshakes:470(1098ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(376ns) PrepareNewSessionHandshake:477(5280ns) {SwitchToState:179(622ns)}}}}}
2025-11-28T14:52:35.375434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789278282007555:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55512455ns) {Configure:104(55509212ns) {ProcessConfigured:135(469474ns) {StartInitialHandshake:157(468015ns) {DropHandshakes:470(863ns) {DropIncomingHandshake:442(245ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(422ns) PrepareNewSessionHandshake:477(954ns) {SwitchToState:179(204ns)}}}}}
2025-11-28T14:52:35.375937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789281522315585:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51605530ns) {Configure:104(51597636ns) {ProcessConfigured:135(2557272ns) {StartInitialHandshake:157(2555075ns) {DropHandshakes:470(1147ns) {DropIncomingHandshake:442(372ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(719ns) PrepareNewSessionHandshake:477(1824ns) {SwitchToState:179(619ns)}}}}}
2025-11-28T14:52:35.373740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789281213540088:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52142186ns) {Configure:104(52136455ns) {ProcessConfigured:135(1349567ns) {StartInitialHandshake:157(1346982ns) {DropHandshakes:470(1324ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(367ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:477(1437ns) {SwitchToState:179(635ns)}}}}}
2025-11-28T14:52:35.376084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789281522315586:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48630168ns) {Configure:104(48622244ns) {ProcessConfigured:135(2064498ns) {StartInitialHandshake:157(2062927ns) {DropHandshakes:470(878ns) {DropIncomingHandshake:442(227ns) DropOutgoingHandshake:457(213ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(1455ns) {SwitchToState:179(390ns)}}}}}
2025-11-28T14:52:35.375705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789278282007556:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17667367ns) {Configure:104(17660328ns) {ProcessConfigured:135(1438253ns) {StartInitialHandshake:157(1436062ns) {DropHandshakes:470(1194ns) {DropIncomingHandshake:442(356ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(817ns) PrepareNewSessionHandshake:477(1679ns) {SwitchToState:179(539ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.377124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789281213540163:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3929
2025-11-28T14:52:35.376643Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789278941111603:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36343399ns) {Configure:104(26890217ns) {ProcessConfigured:135(1077226ns) {StartInitialHandshake:157(1075183ns) {DropHandshakes:470(931ns) {DropIncomingHandshake:442(308ns) DropOutgoingHandshake:457(177ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:477(637ns) {SwitchToState:179(99ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.378013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789278282007630:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65104
2025-11-28T14:52:35.377421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277731171254:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43538
2025-11-28T14:52:35.379254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277731171254:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.378819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789281213540163:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.380959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789278282007630:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.381394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789278282007635:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65104
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.381758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789281522315664:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9524
2025-11-28T14:52:35.381783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277731171261:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43538
2025-11-28T14:52:35.381788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789278211446680:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-11-28T14:52:35.383123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789281213540168:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3929
2025-11-28T14:52:35.382182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111680:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.384339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789281522315665:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9524
2025-11-28T14:52:35.382156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789278211446681:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-11-28T14:52:35.384357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789278282007636:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65104
2025-11-28T14:52:35.382197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111679:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.383457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789281213540167:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3929
2025-11-28T14:52:35.384586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789278282007637:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65104
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.388516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277731171261:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.384389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789281522315666:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9524
2025-11-28T14:52:35.385245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789278941111679:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.383477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789281213540169:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3929
2025-11-28T14:52:35.388866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789278282007627:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.388868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277731171251:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.385806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789281522315666:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.387422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789278941111680:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.385611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789281213540168:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.388912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789277731171251:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.386570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789281522315587:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.388884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789278282007629:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.387388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789281213540087:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.388558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789278941111602:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.387436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789281522315665:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.388901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789278282007627:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.391349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789281213540159:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.392724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789278211446680:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.391312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111681:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.391372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789281213540159:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.388912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789278282007629:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.392240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789278941111601:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.391380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789281213540167:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.393036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789278211446681:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.388025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789281522315585:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.391717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789281213540169:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.391356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789278282007636:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.392289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789278941111681:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.392390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789281213540160:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.389088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789281522315664:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.393048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789278211446603:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.392403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789278282007637:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.395507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277731171251:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.392686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789278941111662:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.392421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789281213540160:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.389396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789281522315586:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.393024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789278282007629:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:35.396050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111685:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.393008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789281213540159:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:35.393963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789281522315658:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.393680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789278282007627:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:35.393938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789281213540158:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.393984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789281522315658:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.393919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789278282007624:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.393960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789281213540158:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.396443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789281522315658:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.393957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789278282007624:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.398587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789278941111679:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:35.398818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789277731171244:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.394615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789281213540160:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:35.397089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789281522315670:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9524
2025-11-28T14:52:35.396337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789281213540151:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.400619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.396118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789278282007620:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.398608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789281522315665:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:35.400836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789278941111602:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.400677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277731171244:2107] [node 4] ICP22 created new session: [3:7577789277731171266:2107]
2025-11-28T14:52:35.396771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:35.396171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789278282007555:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.401754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789281522315585:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789281522315665:2117] held: no
2025-11-28T14:52:35.400925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.401941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277731171266:2107] [node 4] ICS09 handshake done sender: [3:7577789277731171251:2111] self: [3:7577789277731171248:0] peer: [4:7577789281522315667:0] socket: 622 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.404355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789278211446603:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22799) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789278211446680:2115] held: no
2025-11-28T14:52:35.396214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:35.396808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789281213540151:2109] [node 6] ICP22 created new session: [2:7577789281213540178:2109]
2025-11-28T14:52:35.401197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789278941111602:2049] [node 2] ICP22 created new session: [6:7577789278941111688:2049]
2025-11-28T14:52:35.402743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789281522315586:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22799) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789281522315664:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.404295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277731171266:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:35.403211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789281522315650:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.396404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789278282007555:2048] [node 2] ICP22 created new session: [1:7577789278282007644:2048]
2025-11-28T14:52:35.396916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789281213540152:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.405772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789278211446684:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.404274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:35.402731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789278941111688:2049] [node 2] ICS09 handshake done sender: [6:7577789278941111679:2115] self: [6:7577789278941111682:0] peer: [2:7577789281213540153:0] socket: 616 qp: -1
2025-11-28T14:52:35.404312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789281522315650:2109] [node 6] ICP22 created new session: [4:7577789281522315674:2109]
2025-11-28T14:52:35.396539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:35.408572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789278211446685:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-11-28T14:52:35.404318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789281522315585:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.404341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:35.405193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:35.396603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789278282007635:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.397149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.405233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789281522315585:2048] [node 1] ICP22 created new session: [4:7577789281522315675:2048]
2025-11-28T14:52:35.408629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789278211446680:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:35.403280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789278941111688:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:35.396773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789278282007620:2107] [node 4] ICP22 created new session: [1:7577789278282007643:2107]
2025-11-28T14:52:35.408924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277731171267:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43538
2025-11-28T14:52:35.405500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789281522315674:2109] [node 6] ICS09 handshake done sender: [4:7577789281522315658:2113] self: [4:7577789281522315652:0] peer: [6:7577789278941111684:0] socket: 623 qp: -1
2025-11-28T14:52:35.397392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789281213540178:2109] [node 6] ICS09 handshake done sender: [2:7577789281213540159:2111] self: [2:7577789281213540153:0] peer: [6:7577789278941111682:0] socket: 611 qp: -1
2025-11-28T14:52:35.405569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789281522315675:2048] [node 1] ICS09 handshake done sender: [4:7577789281522315665:2117] self: [4:7577789281522315668:0] peer: [1:7577789278282007622:0] socket: 631 qp: -1
2025-11-28T14:52:35.397639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789281213540152:2107] [node 4] ICP22 created new session: [2:7577789281213540179:2107]
2025-11-28T14:52:35.404305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111686:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.405751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789281522315673:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9524
2025-11-28T14:52:35.397925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789281213540178:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:35.397134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789278282007644:2048] [node 2] ICS09 handshake done sender: [1:7577789278282007627:2111] self: [1:7577789278282007621:0] peer: [2:7577789281213540171:0] socket: 612 qp: -1
2025-11-28T14:52:35.405761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789281522315674:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:35.409320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789278211446686:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-11-28T14:52:35.398342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:35.404688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:35.405807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789281522315675:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:35.409645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789278211446603:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.397276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789278282007643:2107] [node 4] ICS09 handshake done sender: [1:7577789278282007629:2113] self: [1:7577789278282007622:0] peer: [4:7577789281522315668:0] socket: 610 qp: -1
2025-11-28T14:52:35.398568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789281213540179:2107] [node 4] ICS09 handshake done sender: [2:7577789281213540160:2112] self: [2:7577789281213540155:0] peer: [4:7577789281522315669:0] socket: 608 qp: -1
2025-11-28T14:52:35.410061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:35.409604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277731171268:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43538
2025-11-28T14:52:35.410108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789278211446603:2049] [node 2] ICP22 created new session: [5:7577789278211446688:2049]
2025-11-28T14:52:35.405789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111687:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.406332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789281522315671:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9524
2025-11-28T14:52:35.398595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789281213540158:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:35.410040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277731171267:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.397303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789278282007644:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:35.410109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789278211446685:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.406347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789278941111601:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789278941111680:2116] held: no
2025-11-28T14:52:35.398599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789281213540179:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:35.405794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.408720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789278941111681:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:35.410752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789277731171180:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.410678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789278211446686:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.406395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:35.397307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789278282007643:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:35.398835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:35.409063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789278941111680:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:35.410864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277731171268:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.410023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789281522315666:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:35.400551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789281213540176:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3929
2025-11-28T14:52:35.410779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789278211446688:2049] [node 2] ICS09 handshake done sender: [5:7577789278211446680:2115] self: [5:7577789278211446682:0] peer: [2:7577789281213540156:0] socket: 609 qp: -1
2025-11-28T14:52:35.397339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:35.410595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789278941111662:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.410958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789278211446688:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:35.410682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789281522315664:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:35.411171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789277731171181:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.410656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:35.411008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:35.401179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789281213540150:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.411473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789281522315587:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.412094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277731171270:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43538
2025-11-28T14:52:35.397340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:35.401230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:35.410664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789278941111601:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.412762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277731171256:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.411218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789278211446604:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.398326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789278282007624:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:35.401985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789281213540150:2108] [node 5] ICP22 created new session: [2:7577789281213540184:2108]
2025-11-28T14:52:35.411783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789281522315586:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.410689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:35.412781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789277731171256:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.411460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789278211446602:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.404308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789281213540184:2108] [node 5] ICS09 handshake done sender: [2:7577789281213540158:2110] self: [2:7577789281213540156:0] peer: [5:7577789278211446682:0] socket: 607 qp: -1
2025-11-28T14:52:35.400569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789278282007619:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.410824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789278941111662:2105] [node 4] ICP22 created new session: [6:7577789278941111691:2105]
2025-11-28T14:52:35.404971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789281213540184:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:35.401947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:35.411793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:35.410825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789278941111601:2048] [node 1] ICP22 created new session: [6:7577789278941111692:2048]
2025-11-28T14:52:35.405222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:35.402841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789278282007619:2109] [node 6] ICP22 created new session: [1:7577789278282007652:2109]
2025-11-28T14:52:35.411446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789278941111692:2048] [node 1] ICS09 handshake done sender: [6:7577789278941111680:2116] self: [6:7577789278941111683:0] peer: [1:7577789278282007623:0] socket: 630 qp: -1
2025-11-28T14:52:35.412772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789281522315587:2050] [node 3] ICP22 created new session: [4:7577789281522315682:2050]
2025-11-28T14:52:35.405523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789281213540168:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:35.411625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789278941111692:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:35.404303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789278282007652:2109] [node 6] ICS09 handshake done sender: [1:7577789278282007624:2110] self: [1:7577789278282007623:0] peer: [6:7577789278941111683:0] socket: 620 qp: -1
2025-11-28T14:52:35.416109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277731171256:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:35.412775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789281522315679:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9524
2025-11-28T14:52:35.415682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789278211446692:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-11-28T14:52:35.407473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789281213540087:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.404326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789278282007652:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:35.408652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:35.411634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789278941111691:2105] [node 4] ICS09 handshake done sender: [6:7577789278941111681:2117] self: [6:7577789278941111684:0] peer: [4:7577789281522315652:0] socket: 640 qp: -1
2025-11-28T14:52:35.409571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789281213540087:2048] [node 1] ICP22 created new session: [2:7577789281213540186:2048]
2025-11-28T14:52:35.405208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:35.413175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:35.411654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.418113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789278211446602:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789278211446685:2118] held: no
2025-11-28T14:52:35.409600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789281213540186:2048] [node 1] ICS09 handshake done sender: [2:7577789281213540168:2117] self: [2:7577789281213540171:0] peer: [1:7577789278282007621:0] socket: 635 qp: -1
2025-11-28T14:52:35.408949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789278282007651:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65104
2025-11-28T14:52:35.418037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789277731171245:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.409631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789281213540186:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:35.411671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789278941111691:2105] [node 4] ICS10 traffic start
2025-11-28T14:52:35.413459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789281522315586:2049] [node 2] ICP22 created new session: [4:7577789281522315683:2049]
2025-11-28T14:52:35.410580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:35.412155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:35.418065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789277731171181:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22799) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789277731171268:2119] held: no
2025-11-28T14:52:35.411232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789278282007631:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.413129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789281213540162:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.411255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789278282007631:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.413481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789281522315682:2050] [node 3] ICS09 handshake done sender: [4:7577789281522315666:2118] self: [4:7577789281522315667:0] peer: [3:7577789277731171248:0] socket: 638 qp: -1
2025-11-28T14:52:35.413147Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789281213540162:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.420026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789278211446693:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-11-28T14:52:35.412160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789278282007628:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.418091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:35.413915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789281522315682:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:35.415186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789281213540162:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:35.413917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789281522315683:2049] [node 2] ICS09 handshake done sender: [4:7577789281522315664:2116] self: [4:7577789281522315669:0] peer: [2:7577789281213540155:0] socket: 619 qp: -1
2025-11-28T14:52:35.420653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789278211446694:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-11-28T14:52:35.412181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789278282007628:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.415516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789281213540088:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.418122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277731171245:2108] [node 5] ICP22 created new session: [3:7577789277731171276:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.415034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789281522315683:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:35.413198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789278282007555:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:35.416104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:35.420260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277731171274:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43538
2025-11-28T14:52:35.420866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789278211446685:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:35.416145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789281213540088:2049] [node 3] ICP22 created new session: [2:7577789281213540189:2049]
2025-11-28T14:52:35.420258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277731171276:2108] [node 5] ICS09 handshake done sender: [3:7577789277731171256:2115] self: [3:7577789277731171249:0] peer: [5:7577789278211446690:0] socket: 662 qp: -1
2025-11-28T14:52:35.415056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:35.415285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789278282007631:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:35.420925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789278211446692:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.420641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277731171276:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:35.415380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:35.416217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789281213540189:2049] [node 3] ICS09 handshake done sender: [2:7577789281213540162:2114] self: [2:7577789281213540157:0] peer: [3:7577789277731171272:0] socket: 663 qp: -1
2025-11-28T14:52:35.421697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789278211446604:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4778) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789278211446686:2119] held: no
2025-11-28T14:52:35.415391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789278282007628:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:35.420678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:35.417204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789281522315650:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:35.422091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789278211446686:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:35.417982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789281522315681:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9524
2025-11-28T14:52:35.416243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789281213540189:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:35.415499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789278282007556:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.421440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789277731171275:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43538
2025-11-28T14:52:35.416159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:35.422198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789278211446602:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.422236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:35.420488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789281522315681:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.416378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789278282007556:2049] [node 3] ICP22 created new session: [1:7577789278282007656:2049]
2025-11-28T14:52:35.421855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789277731171180:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789277731171267:2118] held: no
2025-11-28T14:52:35.416448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:35.422916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277731171268:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:35.416713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789278282007656:2049] [node 3] ICS09 handshake done sender: [1:7577789278282007631:2115] self: [1:7577789278282007626:0] peer: [3:7577789277731171271:0] socket: 657 qp: -1
2025-11-28T14:52:35.422847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789278211446602:2048] [node 1] ICP22 created new session: [5:7577789278211446696:2048]
2025-11-28T14:52:35.423224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277731171267:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:35.416731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789278282007618:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.423537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789277731171181:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.416771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:35.423290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789278211446665:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.424169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789281522315659:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:35.416984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789278282007618:2108] [node 5] ICP22 created new session: [1:7577789278282007657:2108]
2025-11-28T14:52:35.423445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789278211446604:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.423851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789277731171180:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.417242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789278282007656:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:35.424205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789281522315659:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.423869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:35.423545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789278211446696:2048] [node 1] ICS09 handshake done sender: [5:7577789278211446685:2118] self: [5:7577789278211446689:0] peer: [1:7577789278282007625:0] socket: 660 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.417286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:35.423875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:35.423575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:35.424271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111695:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.424012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277731171180:2048] [node 1] ICP22 created new session: [3:7577789277731171280:2048]
2025-11-28T14:52:35.417574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789278282007657:2108] [node 5] ICS09 handshake done sender: [1:7577789278282007628:2112] self: [1:7577789278282007625:0] peer: [5:7577789278211446689:0] socket: 659 qp: -1
2025-11-28T14:52:35.424450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111696:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.423585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789278211446696:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:35.424043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277731171280:2048] [node 1] ICS09 handshake done sender: [3:7577789277731171267:2118] self: [3:7577789277731171271:0] peer: [1:7577789278282007626:0] socket: 658 qp: -1
2025-11-28T14:52:35.417600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789278282007657:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:35.424068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277731171181:2049] [node 2] ICP22 created new session: [3:7577789277731171279:2049]
2025-11-28T14:52:35.419875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:35.423633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789278211446604:2050] [node 3] ICP22 created new session: [5:7577789278211446698:2050]
2025-11-28T14:52:35.424163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277731171280:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:35.424055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789278211446698:2050] [node 3] ICS09 handshake done sender: [5:7577789278211446686:2119] self: [5:7577789278211446690:0] peer: [3:7577789277731171249:0] socket: 665 qp: -1
2025-11-28T14:52:35.424192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:35.424068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:35.424223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277731171279:2049] [node 2] ICS09 handshake done sender: [3:7577789277731171268:2119] self: [3:7577789277731171272:0] peer: [2:7577789281213540157:0] socket: 664 qp: -1
2025-11-28T14:52:35.424176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789278211446698:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:35.424256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277731171279:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:35.424212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:35.424302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:35.425910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789281522315659:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.426125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789278941111695:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.426132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789281522315649:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.426164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789278941111696:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.426188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789278211446701:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-11-28T14:52:35.426213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:35.426239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789278941111664:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.426254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789278941111603:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:35.426274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789281522315649:2108] [node 5] ICP22 created new session: [4:7577789281522315688:2108]
2025-11-28T14:52:35.426321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789281522315688:2108] [node 5] ICS09 handshake done sender: [4:7577789281522315659:2114] self: [4:7577789281522315655:0] peer: [5:7577789278211446697:0] socket: 666 qp: -1
2025-11-28T14:52:35.426365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789281522315688:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:35.426402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:35.426416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789278211446692:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:35.426473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789278211446675:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.427641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789277731171257:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:35.427676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789277731171257:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.426496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789278211446675:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:35.426606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789278211446665:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13349) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789278211446692:2120] held: no
2025-11-28T14:52:35.427645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789278211446665:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.427730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:35.427773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789278211446665:2108] [node 4] ICP22 created new session: [5:7577789278211446703:2108]
2025-11-28T14:52:35.428119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789278211446703:2108] [node 4] ICS09 handshake done sender: [5:7577789278211446692:2120] self: [5:7577789278211446697:0] peer: [4:7577789281522315655:0] socket: 639 qp: -1
2025-11-28T14:52:35.428793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789278211446703:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:35.428840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.428916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789278211446675:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:35.429204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789278211446666:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:35.429194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789277731171257:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:35.429322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:35.429239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111699:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.429368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789278211446666:2109] [node 6] ICP22 created new session: [5:7577789278211446705:2109]
2025-11-28T14:52:35.429437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789277731171246:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:35.429413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789278211446705:2109] [node 6] ICS09 handshake done sender: [5:7577789278211446675:2113] self: [5:7577789278211446673:0] peer: [6:7577789278941111697:0] socket: 602 qp: -1
2025-11-28T14:52:35.429563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789278941111696:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:35.429511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:35.429524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789278211446705:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:35.430001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789278941111700:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28953
2025-11-28T14:52:35.429548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789277731171246:2109] [node 6] ICP22 created new session: [3:7577789277731171284:2109]
2025-11-28T14:52:35.429795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:35.430179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789277731171284:2109] [node 6] ICS09 handshake done sender: [3:7577789277731171257:2116] self: [3:7577789277731171255:0] peer: [6:7577789278941111698:0] socket: 600 qp: -1
2025-11-28T14:52:35.430030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789278941111664:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13701) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789278941111696:2122] held: no
2025-11-28T14:52:35.430701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789278941111664:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.430734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789277731171284:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:35.430764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:35.430790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:35.430847Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789278941111603:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4778) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789278941111695:2121] held: no
2025-11-28T14:52:35.430850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789278941111664:2106] [node 5] ICP22 created new session: [6:7577789278941111701:2106]
2025-11-28T14:52:35.431007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789278941111701:2106] [node 5] ICS09 handshake done sender: [6:7577789278941111696:2122] self: [6:7577789278941111697:0] peer: [5:7577789278211446673:0] socket: 604 qp: -1
2025-11-28T14:52:35.431337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789278941111701:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:35.431425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:35.431592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789278941111695:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:35.431669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789278941111603:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:35.431696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:35.431730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789278941111603:2050] [node 3] ICP22 created new session: [6:7577789278941111703:2050]
2025-11-28T14:52:35.431762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789278941111703:2050] [node 3] ICS09 handshake done sender: [6:7577789278941111695:2121] self: [6:7577789278941111698:0] peer: [3:7577789277731171255:0] socket: 601 qp: -1
2025-11-28T14:52:35.431816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789278941111703:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:35.431844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:35.490324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:35.490492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789281213540186:2048] [node 1] ICS07 socket disconnect 635 reason# EndOfStream
2025-11-28T14:52:35.490519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789281213540186:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.490588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:35.490629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789281213540186:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:35.491251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:35.491471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789277731171280:2048] [node 1] ICS01 socket: 658 reason# EndOfStream
2025-11-28T14:52:35.491581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789277731171180:2048] [node 1] ICP30 unregister session Session# [3:7577789277731171280:2048] VirtualId# [3:7577789277731171271:0]
2025-11-28T14:52:35.491632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789277731171280:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.491714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:35.491776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:35.492428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789281213540087:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20734) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:35.492464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789281213540186:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:35.493867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789281213540087:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20734) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:35.493905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789281213540186:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:35.493962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789281213540087:2048] [node 1] ICP30 unregister session Session# [2:7577789281213540186:2048] VirtualId# [2:7577789281213540171:0]
2025-11-28T14:52:35.494065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:35.494003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:35.494027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789281213540087:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20734) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.494053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20734) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.494191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789281522315675:2048] [node 1] ICS01 socket: 631 reason# EndOfStream
2025-11-28T14:52:35.494242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789281522315585:2048] [node 1] ICP30 unregister session Session# [4:7577789281522315675:2048] VirtualId# [4:7577789281522315668:0]
2025-11-28T14:52:35.494268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789281522315675:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.494325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:35.494360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:35.494837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:35.494981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789278211446696:2048] [node 1] ICS01 socket: 660 reason# EndOfStream
2025-11-28T14:52:35.495014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789278211446602:2048] [node 1] ICP30 unregister session Session# [5:7577789278211446696:2048] VirtualId# [5:7577789278211446689:0]
2025-11-28T14:52:35.495035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789278211446696:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.495105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:35.495143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:35.495439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:35.495569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789278941111692:2048] [node 1] ICS01 socket: 630 reason# EndOfStream
2025-11-28T14:52:35.495599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789278941111601:2048] [node 1] ICP30 unregister session Session# [6:7577789278941111692:2048] VirtualId# [6:7577789278941111683:0]
2025-11-28T14:52:35.495616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789278941111692:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.495683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:35.495724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:35.503757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789281213540087:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:35.552876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:35.553121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789277731171279:2049] [node 2] ICS01 socket: 664 reason# EndOfStream
2025-11-28T14:52:35.553171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789277731171181:2049] [node 2] ICP30 unregister session Session# [3:7577789277731171279:2049] VirtualId# [3:7577789277731171272:0]
2025-11-28T14:52:35.553207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789277731171279:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.553333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:35.553398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:35.554784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:35.554929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789281522315683:2049] [node 2] ICS01 socket: 619 reason# EndOfStream
2025-11-28T14:52:35.554967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789281522315586:2049] [node 2] ICP30 unregister session Session# [4:7577789281522315683:2049] VirtualId# [4:7577789281522315669:0]
2025-11-28T14:52:35.554987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789281522315683:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.555044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:35.555094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:35.555635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:35.555776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789278211446688:2049] [node 2] ICS01 socket: 609 reason# EndOfStream
2025-11-28T14:52:35.555808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789278211446603:2049] [node 2] ICP30 unregister session Session# [5:7577789278211446688:2049] VirtualId# [5:7577789278211446682:0]
2025-11-28T14:52:35.555828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789278211446688:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.555932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:35.555992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:35.556262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:35.556414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789278941111688:2049] [node 2] ICS07 socket disconnect 616 reason# EndOfStream
2025-11-28T14:52:35.556440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789278941111688:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.556489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:35.556525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789278941111688:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:35.557093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789278941111602:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22799) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:35.557131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789278941111688:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:35.557176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789278941111602:2049] [node 2] ICP30 unregister session Session# [6:7577789278941111688:2049] VirtualId# [6:7577789278941111682:0]
2025-11-28T14:52:35.557215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:35.557235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789278941111602:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22799) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.557249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22799) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.567290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789278941111602:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:35.611230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:35.611396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789281522315682:2050] [node 3] ICS07 socket disconnect 638 reason# EndOfStream
2025-11-28T14:52:35.611422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789281522315682:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.611507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:35.611568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789281522315682:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:35.612326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789281522315587:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4778) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:35.612379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789281522315682:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:35.612401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:35.612411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789281522315587:2050] [node 3] ICP30 unregister session Session# [4:7577789281522315682:2050] VirtualId# [4:7577789281522315667:0]
2025-11-28T14:52:35.612478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:35.612509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789281522315587:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.612518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.612590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789278211446698:2050] [node 3] ICS01 socket: 665 reason# EndOfStream
2025-11-28T14:52:35.612626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789278211446604:2050] [node 3] ICP30 unregister session Session# [5:7577789278211446698:2050] VirtualId# [5:7577789278211446690:0]
2025-11-28T14:52:35.612657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789278211446698:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.612712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:35.612752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:35.613277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:35.613419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789278941111703:2050] [node 3] ICS01 socket: 601 reason# EndOfStream
2025-11-28T14:52:35.613450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789278941111603:2050] [node 3] ICP30 unregister session Session# [6:7577789278941111703:2050] VirtualId# [6:7577789278941111698:0]
2025-11-28T14:52:35.613473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789278941111703:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.613528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:35.613566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:35.658277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:35.658427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789278211446703:2108] [node 4] ICS01 socket: 639 reason# EndOfStream
2025-11-28T14:52:35.658480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789278211446665:2108] [node 4] ICP30 unregister session Session# [5:7577789278211446703:2108] VirtualId# [5:7577789278211446697:0]
2025-11-28T14:52:35.658505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789278211446703:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.658573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:35.658622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:35.659029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:35.659225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789278941111691:2105] [node 4] ICS07 socket disconnect 640 reason# EndOfStream
2025-11-28T14:52:35.659270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789278941111691:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.659358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:35.659409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789278941111691:2105] [node 4] ICS15 start handshake
2025-11-28T14:52:35.660010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789278941111662:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13349) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:35.660055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789278941111691:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:35.660126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789278941111662:2105] [node 4] ICP30 unregister session Session# [6:7577789278941111691:2105] VirtualId# [6:7577789278941111684:0]
2025-11-28T14:52:35.660162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:35.660194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789278941111662:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13349) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.660208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13349) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:35.669671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789278941111662:2105] [node 4] ICP33 wake up from error state
2025-11-28T14:52:35.735836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:35.735995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789278941111701:2106] [node 5] ICS01 socket: 604 reason# EndOfStream
2025-11-28T14:52:35.736029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789278941111664:2106] [node 5] ICP30 unregister session Session# [6:7577789278941111701:2106] VirtualId# [6:7577789278941111697:0]
2025-11-28T14:52:35.736051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789278941111701:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:35.736120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:35.736163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:35.835353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789277818405241:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:35.835393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789277818405242:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:35.851164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789280941472666:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.851203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789280941472667:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:35.867102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789279120590992:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.867146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789279120590993:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:35.884278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789280330373597:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.884313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789280330373598:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:35.884341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789280330373599:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:35.900384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789281596559383:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.900413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789281596559384:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:35.900432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789281596559385:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:35.917138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789280929759072:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:35.917170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789280929759073:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:35.917202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789280929759074:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:35.991413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789280929759134:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:36.000221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789280941472729:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:36.001029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789285891526742:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:36.000982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789285236440026:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.000886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789282113372609:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:36.000784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789285224726433:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:36.001156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284625340956:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.001304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789282113372610:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.001309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789285236440027:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.001288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789283415558351:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:36.001429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789285891526743:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.001481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284625340957:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.001523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789282113372611:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.001694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789283415558352:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.002056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789283415558353:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.009329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285236440038:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54541
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.010329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284625340971:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28654
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.010836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285236440042:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54541
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.010895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285236440043:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54541
2025-11-28T14:52:36.010940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282113372622:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29006
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.011128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282113372626:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29006
2025-11-28T14:52:36.011177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282113372627:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29006
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.014602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789285891526754:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42854
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.014936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789283415558364:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19475
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.015004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789283415558365:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19475
2025-11-28T14:52:36.015947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789283415558366:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19475
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.017725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789283415558370:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19475
2025-11-28T14:52:36.017931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789285891526754:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.018700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284625340972:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28654
2025-11-28T14:52:36.018775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372622:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.018968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284625340971:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.018956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285236440038:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.019405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284625340973:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28654
2025-11-28T14:52:36.019320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285236440042:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.019477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372627:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.019517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285236440043:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.019543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372626:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.019549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789280330373598:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.019684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285236440044:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54541
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.019749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282113372628:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29006
2025-11-28T14:52:36.019751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284625340972:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.019742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789283415558364:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.019782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285236440045:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54541
2025-11-28T14:52:36.019832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282113372629:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29006
2025-11-28T14:52:36.019859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789280330373597:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.019862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789283415558365:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.019956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789283415558366:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.019975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285236440033:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.020003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789285236440033:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.020084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372617:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.020136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282113372617:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.021279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282113372617:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:36.021357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789280941472666:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.021507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284625340976:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28654
2025-11-28T14:52:36.021634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789282113372609:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.021685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.021714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789282113372609:2107] [node 4] ICP22 created new session: [1:7577789282113372634:2107]
2025-11-28T14:52:36.021742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282113372634:2107] [node 4] ICS09 handshake done sender: [1:7577789282113372617:2121] self: [1:7577789282113372616:0] peer: [4:7577789284625340975:0] socket: 625 qp: -1
2025-11-28T14:52:36.021762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282113372634:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:36.021803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:36.021856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285236440033:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:36.022165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284625340972:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:36.022598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789280941472729:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.022755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.022815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789280941472729:2107] [node 4] ICP22 created new session: [2:7577789285236440050:2107]
2025-11-28T14:52:36.022868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285236440050:2107] [node 4] ICS09 handshake done sender: [2:7577789285236440033:2112] self: [2:7577789285236440032:0] peer: [4:7577789284625340974:0] socket: 617 qp: -1
2025-11-28T14:52:36.022901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285236440050:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:36.022929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:36.022955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372613:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.022984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282113372613:2119] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.023015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789280330373597:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.023058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:36.023552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789280330373597:2048] [node 1] ICP22 created new session: [4:7577789284625340978:2048]
2025-11-28T14:52:36.023582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284625340978:2048] [node 1] ICS09 handshake done sender: [4:7577789284625340972:2116] self: [4:7577789284625340975:0] peer: [1:7577789282113372616:0] socket: 629 qp: -1
2025-11-28T14:52:36.023612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284625340978:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.023634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.024230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372628:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.024645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789279120590993:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.024740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285236440044:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.024747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789279120590992:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.024856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789283415558370:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.025275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372629:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.025499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282113372613:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.025594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285236440031:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.025624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789285236440031:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.025713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789285891526759:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42854
2025-11-28T14:52:36.026801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789285891526760:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42854
No RDMA context found for GID: 2025-11-28T14:52:36.026872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285236440045:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:36.026992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372615:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.026981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284625340977:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28654
2025-11-28T14:52:36.027010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282113372615:2120] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.027234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285236440053:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54541
2025-11-28T14:52:36.027600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789277818405241:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.027652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:36.027692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789277818405241:2048] [node 2] ICP22 created new session: [1:7577789282113372639:2048]
2025-11-28T14:52:36.027731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282113372639:2048] [node 2] ICS09 handshake done sender: [1:7577789282113372613:2119] self: [1:7577789282113372612:0] peer: [2:7577789285236440048:0] socket: 618 qp: -1
2025-11-28T14:52:36.027725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284625340973:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.027782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789280330373598:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3541) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789284625340971:2115] held: no
2025-11-28T14:52:36.027806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282113372639:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:36.027849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:36.028061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284625340971:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.028444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789283415558375:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19475
2025-11-28T14:52:36.028677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789280330373598:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.028730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.028767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789280330373598:2049] [node 2] ICP22 created new session: [4:7577789284625340982:2049]
2025-11-28T14:52:36.028846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284625340982:2049] [node 2] ICS09 handshake done sender: [4:7577789284625340971:2115] self: [4:7577789284625340974:0] peer: [2:7577789285236440032:0] socket: 619 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.028890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284625340982:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:36.028923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:36.028869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789285891526761:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42854
2025-11-28T14:52:36.028926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726447:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
2025-11-28T14:52:36.029443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285236440043:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:36.029556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789280330373599:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.029959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789283415558359:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.029991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789283415558359:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.030024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285236440031:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.030143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789285891526759:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.030456Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282113372615:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.030665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789281596559383:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.030948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726448:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
2025-11-28T14:52:36.031488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789277818405242:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.031539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:36.031814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789277818405242:2049] [node 3] ICP22 created new session: [1:7577789282113372644:2049]
2025-11-28T14:52:36.031844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282113372644:2049] [node 3] ICS09 handshake done sender: [1:7577789282113372615:2120] self: [1:7577789282113372614:0] peer: [3:7577789283415558373:0] socket: 621 qp: -1
2025-11-28T14:52:36.031862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282113372644:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:36.031886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:36.032178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789280941472666:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.032222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:36.032259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789280941472666:2048] [node 1] ICP22 created new session: [2:7577789285236440057:2048]
2025-11-28T14:52:36.032272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789285891526760:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.032306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285236440057:2048] [node 1] ICS09 handshake done sender: [2:7577789285236440043:2117] self: [2:7577789285236440048:0] peer: [1:7577789282113372612:0] socket: 622 qp: -1
2025-11-28T14:52:36.032333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285236440057:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.032334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372619:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.032360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.032377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282113372619:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.032400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789281596559384:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.032396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789283415558364:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:36.032886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789280941472666:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.033150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789283415558376:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19475
2025-11-28T14:52:36.033396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789280941472667:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.033462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:36.033381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789285891526761:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.033572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789285224726447:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.034310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789280941472667:2049] [node 3] ICP22 created new session: [2:7577789285236440059:2049]
2025-11-28T14:52:36.034315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789285891526742:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.034383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285236440059:2049] [node 3] ICS09 handshake done sender: [2:7577789285236440031:2111] self: [2:7577789285236440030:0] peer: [3:7577789283415558371:0] socket: 614 qp: -1
2025-11-28T14:52:36.034418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285236440059:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:36.034420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789285224726433:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.034454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:36.034546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284625340965:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.034577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789283415558359:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:36.034578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789284625340965:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.035247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789279120590993:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.035264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284625340985:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28654
2025-11-28T14:52:36.035308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285236440035:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.035297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:36.035302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789285891526765:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42854
2025-11-28T14:52:36.035356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789279120590993:2049] [node 2] ICP22 created new session: [3:7577789283415558378:2049]
2025-11-28T14:52:36.035333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789285236440035:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.035399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789283415558378:2049] [node 2] ICS09 handshake done sender: [3:7577789283415558364:2115] self: [3:7577789283415558371:0] peer: [2:7577789285236440030:0] socket: 631 qp: -1
2025-11-28T14:52:36.035435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789283415558378:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:36.035466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:36.036554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284625340967:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.036543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282113372619:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.036597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789284625340967:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.037414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789285891526759:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:36.037430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284625340973:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.037431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789282113372610:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.037497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789285224726448:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.037486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.037531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789282113372610:2108] [node 5] ICP22 created new session: [1:7577789282113372647:2108]
2025-11-28T14:52:36.037832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789280330373599:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.037707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282113372647:2108] [node 5] ICS09 handshake done sender: [1:7577789282113372619:2122] self: [1:7577789282113372618:0] peer: [5:7577789285891526762:0] socket: 624 qp: -1
2025-11-28T14:52:36.037873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:36.037805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282113372647:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:36.037913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789280330373599:2050] [node 3] ICP22 created new session: [4:7577789284625340988:2050]
2025-11-28T14:52:36.037963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284625340988:2050] [node 3] ICS09 handshake done sender: [4:7577789284625340973:2117] self: [4:7577789284625340984:0] peer: [3:7577789283415558358:0] socket: 644 qp: -1
2025-11-28T14:52:36.037875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.037994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284625340988:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:36.038017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:36.038512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789281596559383:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.038519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789280929759072:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.038585Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285236440035:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.038563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.038662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789281596559383:2048] [node 1] ICP22 created new session: [5:7577789285891526767:2048]
2025-11-28T14:52:36.038808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726450:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
2025-11-28T14:52:36.038734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789285891526767:2048] [node 1] ICS09 handshake done sender: [5:7577789285891526759:2116] self: [5:7577789285891526762:0] peer: [1:7577789282113372618:0] socket: 652 qp: -1
2025-11-28T14:52:36.038773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789285891526767:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.038838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284625340965:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.038803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.038926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789279120590992:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789283415558366:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.039007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789285236440026:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.039228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.039269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789285236440026:2108] [node 5] ICP22 created new session: [2:7577789285236440063:2108]
2025-11-28T14:52:36.039240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789283415558351:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.039256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726452:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
2025-11-28T14:52:36.039297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.039312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285236440063:2108] [node 5] ICS09 handshake done sender: [2:7577789285236440035:2113] self: [2:7577789285236440034:0] peer: [5:7577789285891526763:0] socket: 653 qp: -1
2025-11-28T14:52:36.039343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285236440063:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:36.039345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789283415558351:2107] [node 4] ICP22 created new session: [3:7577789283415558380:2107]
2025-11-28T14:52:36.039327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284625340967:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:36.039371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.039400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789283415558380:2107] [node 4] ICS09 handshake done sender: [3:7577789283415558359:2112] self: [3:7577789283415558358:0] peer: [4:7577789284625340984:0] socket: 640 qp: -1
2025-11-28T14:52:36.039451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789283415558380:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:36.039516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:36.039741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789285224726447:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:36.040465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789285224726433:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.040494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282113372621:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.040502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.041388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789284625340956:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.040526Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282113372621:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.040550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789285224726433:2108] [node 4] ICP22 created new session: [6:7577789285224726453:2108]
2025-11-28T14:52:36.041431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.041408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789285224726453:2108] [node 4] ICS09 handshake done sender: [6:7577789285224726447:2116] self: [6:7577789285224726449:0] peer: [4:7577789284625340966:0] socket: 658 qp: -1
2025-11-28T14:52:36.041539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284625340956:2108] [node 5] ICP22 created new session: [4:7577789284625340990:2108]
2025-11-28T14:52:36.041584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284625340990:2108] [node 5] ICS09 handshake done sender: [4:7577789284625340965:2113] self: [4:7577789284625340964:0] peer: [5:7577789285891526764:0] socket: 656 qp: -1
2025-11-28T14:52:36.041535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789285224726453:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:36.041580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:36.041613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284625340990:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:36.041648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.041727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282113372621:2123] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.042058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789282113372611:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.042101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.042098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789285891526766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42854
2025-11-28T14:52:36.042130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789282113372611:2109] [node 6] ICP22 created new session: [1:7577789282113372650:2109]
2025-11-28T14:52:36.042175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789284625340957:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.042165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282113372650:2109] [node 6] ICS09 handshake done sender: [1:7577789282113372621:2123] self: [1:7577789282113372620:0] peer: [6:7577789285224726451:0] socket: 660 qp: -1
2025-11-28T14:52:36.042200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.042184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282113372650:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:36.042370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284625340957:2109] [node 6] ICP22 created new session: [4:7577789284625340992:2109]
2025-11-28T14:52:36.042204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.042936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284625340992:2109] [node 6] ICS09 handshake done sender: [4:7577789284625340967:2114] self: [4:7577789284625340966:0] peer: [6:7577789285224726449:0] socket: 657 qp: -1
2025-11-28T14:52:36.042960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284625340992:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:36.042976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.043529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789283415558366:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:36.044964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789285224726452:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.044970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789281596559384:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3541) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789285891526760:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.045211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789285891526769:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42854
2025-11-28T14:52:36.045193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789279120590992:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.045226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:36.045274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789279120590992:2048] [node 1] ICP22 created new session: [3:7577789283415558382:2048]
2025-11-28T14:52:36.045303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789283415558382:2048] [node 1] ICS09 handshake done sender: [3:7577789283415558366:2117] self: [3:7577789283415558373:0] peer: [1:7577789282113372614:0] socket: 633 qp: -1
2025-11-28T14:52:36.045324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789283415558382:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.045345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.046456Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789285891526761:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.046682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
2025-11-28T14:52:36.046810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789285891526742:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.046853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.046897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789285891526742:2108] [node 4] ICP22 created new session: [5:7577789285891526772:2108]
2025-11-28T14:52:36.046938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789285891526772:2108] [node 4] ICS09 handshake done sender: [5:7577789285891526761:2118] self: [5:7577789285891526764:0] peer: [4:7577789284625340964:0] socket: 623 qp: -1
2025-11-28T14:52:36.047007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789285891526772:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:36.047060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:36.047292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789280929759072:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20081) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789285224726448:2117] held: no
2025-11-28T14:52:36.047480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789280929759073:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.047581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789285224726448:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:36.047946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789280929759072:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.048039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.048076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789280929759072:2048] [node 1] ICP22 created new session: [6:7577789285224726459:2048]
2025-11-28T14:52:36.048111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789285224726459:2048] [node 1] ICS09 handshake done sender: [6:7577789285224726448:2117] self: [6:7577789285224726451:0] peer: [1:7577789282113372620:0] socket: 661 qp: -1
2025-11-28T14:52:36.048250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789285224726459:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.048271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.048232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284625340994:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28654
2025-11-28T14:52:36.048316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789285891526760:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:36.048233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285236440037:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.048262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789285236440037:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.048956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285236440037:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:36.049470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789285236440027:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.049521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.049567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789285236440027:2109] [node 6] ICP22 created new session: [2:7577789285236440066:2109]
2025-11-28T14:52:36.049609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285236440066:2109] [node 6] ICS09 handshake done sender: [2:7577789285236440037:2114] self: [2:7577789285236440036:0] peer: [6:7577789285224726458:0] socket: 669 qp: -1
2025-11-28T14:52:36.049641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285236440066:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:36.049662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.049690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789281596559384:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.049738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:36.049775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789281596559384:2049] [node 2] ICP22 created new session: [5:7577789285891526775:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.049819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789285891526775:2049] [node 2] ICS09 handshake done sender: [5:7577789285891526760:2117] self: [5:7577789285891526763:0] peer: [2:7577789285236440034:0] socket: 655 qp: -1
2025-11-28T14:52:36.049863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726457:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
2025-11-28T14:52:36.049852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789285891526775:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:36.049950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.050651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789283415558386:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19475
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.051152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789285891526774:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42854
2025-11-28T14:52:36.052236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789280929759073:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3541) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789285224726452:2119] held: no
2025-11-28T14:52:36.052241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789283415558386:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.052658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789285224726457:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.052739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726461:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.052814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789285891526774:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.052870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726462:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
2025-11-28T14:52:36.053034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789281596559385:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.053312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789283415558361:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.053348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789283415558361:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.053363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789285224726461:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.053520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789285224726452:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:36.053768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789280929759074:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.053822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789280929759073:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.053884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:36.053926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789280929759073:2049] [node 2] ICP22 created new session: [6:7577789285224726464:2049]
2025-11-28T14:52:36.053916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789283415558361:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.053957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789285224726464:2049] [node 2] ICS09 handshake done sender: [6:7577789285224726452:2119] self: [6:7577789285224726458:0] peer: [2:7577789285236440036:0] socket: 637 qp: -1
2025-11-28T14:52:36.053977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789285224726464:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:36.054008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.054221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789283415558352:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.054242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789285891526778:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42854
2025-11-28T14:52:36.054261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.054289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789283415558352:2108] [node 5] ICP22 created new session: [3:7577789283415558389:2108]
2025-11-28T14:52:36.054317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789283415558389:2108] [node 5] ICS09 handshake done sender: [3:7577789283415558361:2113] self: [3:7577789283415558360:0] peer: [5:7577789285891526777:0] socket: 607 qp: -1
2025-11-28T14:52:36.054348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789283415558389:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:36.054371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.054472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789280929759134:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.054779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789285891526753:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.054806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789285891526753:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.055364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789285891526774:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.055513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789283415558363:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.055540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789283415558363:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.056177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789283415558363:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.056980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789281596559385:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.057019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:36.057022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789283415558353:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.057008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726467:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
2025-11-28T14:52:36.057051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789281596559385:2050] [node 3] ICP22 created new session: [5:7577789285891526780:2050]
2025-11-28T14:52:36.057061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.057093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789285891526780:2050] [node 3] ICS09 handshake done sender: [5:7577789285891526774:2122] self: [5:7577789285891526777:0] peer: [3:7577789283415558360:0] socket: 673 qp: -1
2025-11-28T14:52:36.057092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789283415558353:2109] [node 6] ICP22 created new session: [3:7577789283415558392:2109]
2025-11-28T14:52:36.057127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789285891526780:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:36.057124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789283415558392:2109] [node 6] ICS09 handshake done sender: [3:7577789283415558363:2114] self: [3:7577789283415558362:0] peer: [6:7577789285224726463:0] socket: 611 qp: -1
2025-11-28T14:52:36.057153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:36.057145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789283415558392:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:36.057162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.057398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789281596559385:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:36.057409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789285224726457:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.057599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789280929759074:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.057721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:36.057780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789280929759074:2050] [node 3] ICP22 created new session: [6:7577789285224726468:2050]
2025-11-28T14:52:36.057813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789285224726468:2050] [node 3] ICS09 handshake done sender: [6:7577789285224726457:2121] self: [6:7577789285224726463:0] peer: [3:7577789283415558362:0] socket: 639 qp: -1
2025-11-28T14:52:36.057835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789285224726468:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:36.057854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:36.058035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789285891526753:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:36.058320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789285891526743:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.058356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.058382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789285891526743:2109] [node 6] ICP22 created new session: [5:7577789285891526782:2109]
2025-11-28T14:52:36.058412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789285891526782:2109] [node 6] ICS09 handshake done sender: [5:7577789285891526753:2114] self: [5:7577789285891526752:0] peer: [6:7577789285224726466:0] socket: 671 qp: -1
2025-11-28T14:52:36.058433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789285891526782:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:36.058453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.060517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789280929759134:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26279) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789285224726461:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.060663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789285224726470:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33368
2025-11-28T14:52:36.060975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789285224726461:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.061123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789280929759134:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.061165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.061191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789280929759134:2106] [node 5] ICP22 created new session: [6:7577789285224726471:2106]
2025-11-28T14:52:36.061216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789285224726471:2106] [node 5] ICS09 handshake done sender: [6:7577789285224726461:2122] self: [6:7577789285224726466:0] peer: [5:7577789285891526752:0] socket: 642 qp: -1
2025-11-28T14:52:36.061234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789285224726471:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:36.061257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.080203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:36.080386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789285236440057:2048] [node 1] ICS01 socket: 622 reason# EndOfStream
2025-11-28T14:52:36.080437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789280941472666:2048] [node 1] ICP30 unregister session Session# [2:7577789285236440057:2048] VirtualId# [2:7577789285236440048:0]
2025-11-28T14:52:36.080475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789285236440057:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.080557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.080615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.081637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:36.081865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789283415558382:2048] [node 1] ICS01 socket: 633 reason# EndOfStream
2025-11-28T14:52:36.081913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789279120590992:2048] [node 1] ICP30 unregister session Session# [3:7577789283415558382:2048] VirtualId# [3:7577789283415558373:0]
2025-11-28T14:52:36.081946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789283415558382:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.082024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.082083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.083829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:36.083985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789284625340978:2048] [node 1] ICS01 socket: 629 reason# EndOfStream
2025-11-28T14:52:36.084018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789280330373597:2048] [node 1] ICP30 unregister session Session# [4:7577789284625340978:2048] VirtualId# [4:7577789284625340975:0]
2025-11-28T14:52:36.084041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789284625340978:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.084107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.084146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.084874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:36.085072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789285891526767:2048] [node 1] ICS01 socket: 652 reason# EndOfStream
2025-11-28T14:52:36.085114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789281596559383:2048] [node 1] ICP30 unregister session Session# [5:7577789285891526767:2048] VirtualId# [5:7577789285891526762:0]
2025-11-28T14:52:36.085144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789285891526767:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.085216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.085268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.085808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:36.085971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789285224726459:2048] [node 1] ICS01 socket: 661 reason# EndOfStream
2025-11-28T14:52:36.086000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789280929759072:2048] [node 1] ICP30 unregister session Session# [6:7577789285224726459:2048] VirtualId# [6:7577789285224726451:0]
2025-11-28T14:52:36.086018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789285224726459:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.086068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.086102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.119737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:36.119897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789283415558378:2049] [node 2] ICS01 socket: 631 reason# EndOfStream
2025-11-28T14:52:36.119930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789279120590993:2049] [node 2] ICP30 unregister session Session# [3:7577789283415558378:2049] VirtualId# [3:7577789283415558371:0]
2025-11-28T14:52:36.119952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789283415558378:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.120044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:36.120086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:36.122543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:36.122741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789284625340982:2049] [node 2] ICS01 socket: 619 reason# EndOfStream
2025-11-28T14:52:36.122781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789280330373598:2049] [node 2] ICP30 unregister session Session# [4:7577789284625340982:2049] VirtualId# [4:7577789284625340974:0]
2025-11-28T14:52:36.122818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789284625340982:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.122906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:36.122954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:36.123532Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:36.123666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789285891526775:2049] [node 2] ICS01 socket: 655 reason# EndOfStream
2025-11-28T14:52:36.123695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789281596559384:2049] [node 2] ICP30 unregister session Session# [5:7577789285891526775:2049] VirtualId# [5:7577789285891526763:0]
2025-11-28T14:52:36.123713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789285891526775:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.123765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:36.123798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:36.124357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:36.124486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789285224726464:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-11-28T14:52:36.124511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789280929759073:2049] [node 2] ICP30 unregister session Session# [6:7577789285224726464:2049] VirtualId# [6:7577789285224726458:0]
2025-11-28T14:52:36.124535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789285224726464:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.124596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:36.124642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:36.148953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:36.149113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789284625340988:2050] [node 3] ICS01 socket: 644 reason# EndOfStream
2025-11-28T14:52:36.149146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789280330373599:2050] [node 3] ICP30 unregister session Session# [4:7577789284625340988:2050] VirtualId# [4:7577789284625340984:0]
2025-11-28T14:52:36.149186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789284625340988:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.149285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:36.149347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:36.150137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:36.150355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789285891526780:2050] [node 3] ICS01 socket: 673 reason# EndOfStream
2025-11-28T14:52:36.150401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789281596559385:2050] [node 3] ICP30 unregister session Session# [5:7577789285891526780:2050] VirtualId# [5:7577789285891526777:0]
2025-11-28T14:52:36.150427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789285891526780:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.150499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:36.150549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:36.151073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:36.151197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789285224726468:2050] [node 3] ICS07 socket disconnect 639 reason# EndOfStream
2025-11-28T14:52:36.151219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789285224726468:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.151269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:36.151304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789285224726468:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:36.152041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789280929759074:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4940) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:36.152074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789285224726468:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:36.152094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789280929759074:2050] [node 3] ICP30 unregister session Session# [6:7577789285224726468:2050] VirtualId# [6:7577789285224726463:0]
2025-11-28T14:52:36.152125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:36.152149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789280929759074:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4940) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.152173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4940) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.162216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789280929759074:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:36.189324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:36.189545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789285891526772:2108] [node 4] ICS07 socket disconnect 623 reason# EndOfStream
2025-11-28T14:52:36.189576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789285891526772:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.189671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:36.189720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789285891526772:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:36.190037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:36.190169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789285224726453:2108] [node 4] ICS01 socket: 658 reason# EndOfStream
2025-11-28T14:52:36.190211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789285224726433:2108] [node 4] ICP30 unregister session Session# [6:7577789285224726453:2108] VirtualId# [6:7577789285224726449:0]
2025-11-28T14:52:36.190234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789285224726453:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.190280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:36.190312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:36.190533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789285891526742:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61039) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:36.190588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789285891526772:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:36.190625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789285891526742:2108] [node 4] ICP30 unregister session Session# [5:7577789285891526772:2108] VirtualId# [5:7577789285891526764:0]
2025-11-28T14:52:36.190666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:36.190699Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789285891526742:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61039) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.190714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61039) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.212918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:36.213063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789285224726471:2106] [node 5] ICS01 socket: 642 reason# EndOfStream
2025-11-28T14:52:36.213097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789280929759134:2106] [node 5] ICP30 unregister session Session# [6:7577789285224726471:2106] VirtualId# [6:7577789285224726466:0]
2025-11-28T14:52:36.213119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789285224726471:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.213189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:36.213227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:36.270199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789282603068269:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:36.270200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789282603068270:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:36.286517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789285633791916:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.286568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789285633791917:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:36.302232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789282262745251:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.302278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789282262745252:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:36.318712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284712748898:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.318754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284712748900:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:36.318751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284712748899:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:36.334964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283035324143:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.335016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283035324145:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:36.335016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283035324144:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:36.351494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282849594470:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.351493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282849594471:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:36.351531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282849594472:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:36.428998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282849594533:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:36.427518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789282603068332:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:36.429386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789282603068333:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.429418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284712748961:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.429608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282849594535:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:36.429632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283035324206:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:36.429724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789282603068334:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.429899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284712748962:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.430209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283035324207:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.431030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789283035324143:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36083098ns) {Configure:104(36079941ns) {ProcessConfigured:135(53483ns) {StartInitialHandshake:157(51547ns) {DropHandshakes:470(1955ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(337ns)} GenerateSessionVirtualId:988(593ns) PrepareNewSessionHandshake:477(837ns) {SwitchToState:179(265ns)}}}}}
2025-11-28T14:52:36.431210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789282603068269:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44159065ns) {Configure:104(44148520ns) {ProcessConfigured:135(96973ns) {StartInitialHandshake:157(95315ns) {DropHandshakes:470(1195ns) {DropIncomingHandshake:442(363ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(447ns) PrepareNewSessionHandshake:477(724ns) {SwitchToState:179(282ns)}}}}}
2025-11-28T14:52:36.431734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789284712748898:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38329002ns) {Configure:104(38325394ns) {ProcessConfigured:135(126183ns) {StartInitialHandshake:157(123917ns) {DropHandshakes:470(6767ns) {DropIncomingHandshake:442(499ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:477(685ns) {SwitchToState:179(261ns)}}}}}
2025-11-28T14:52:36.433423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789282262745252:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31934696ns) {Configure:104(31927126ns) {ProcessConfigured:135(120609ns) {StartInitialHandshake:157(119092ns) {DropHandshakes:470(818ns) {DropIncomingHandshake:442(230ns) DropOutgoingHandshake:457(196ns)} GenerateSessionVirtualId:988(404ns) PrepareNewSessionHandshake:477(4467ns) {SwitchToState:179(309ns)}}}}}
2025-11-28T14:52:36.433507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789282262745251:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40077027ns) {Configure:104(40068597ns) {ProcessConfigured:135(170988ns) {StartInitialHandshake:157(168069ns) {DropHandshakes:470(1242ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(451ns) PrepareNewSessionHandshake:477(932ns) {SwitchToState:179(520ns)}}}}}
2025-11-28T14:52:36.433700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789285633791916:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32472734ns) {Configure:104(32457484ns) {ProcessConfigured:135(125044ns) {StartInitialHandshake:157(123401ns) {DropHandshakes:470(981ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(397ns) PrepareNewSessionHandshake:477(604ns) {SwitchToState:179(114ns)}}}}}
2025-11-28T14:52:36.433922Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789285633791917:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32579530ns) {Configure:104(32574748ns) {ProcessConfigured:135(229159ns) {StartInitialHandshake:157(227457ns) {DropHandshakes:470(964ns) {DropIncomingHandshake:442(204ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(870ns) {SwitchToState:179(336ns)}}}}}
2025-11-28T14:52:36.434067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789285633792001:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:36.434066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789282262745318:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:36.434072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789282262745319:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.434431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789285633792002:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.434528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789282262745320:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.434611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789285633792003:2117] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.436090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282603068348:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57969
2025-11-28T14:52:36.436162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284712748971:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45055
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.442683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284712748971:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.443505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282603068349:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57969
2025-11-28T14:52:36.443877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789284712748898:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.444389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068348:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.444788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068340:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.444827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282603068340:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.444909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068349:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.445614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282603068350:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57969
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.446532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282603068340:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.446761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285633792004:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28008
2025-11-28T14:52:36.446886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284712748978:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45055
2025-11-28T14:52:36.446884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789282603068332:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:36.446983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.447536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284712748971:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:36.447093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789282603068332:2107] [node 4] ICP22 created new session: [1:7577789282603068354:2107]
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.447973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789282262745325:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11794
2025-11-28T14:52:36.448017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285633792010:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28008
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.447557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282603068354:2107] [node 4] ICS09 handshake done sender: [1:7577789282603068340:2112] self: [1:7577789282603068339:0] peer: [4:7577789284712748977:0] socket: 614 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:36.448019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285633792009:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28008
::ffff:127.1.0.0
2025-11-28T14:52:36.447925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789284712748898:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.448196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789282262745326:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11794
2025-11-28T14:52:36.447662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282603068354:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:36.447870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.447966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:36.448195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284712748898:2048] [node 1] ICP22 created new session: [4:7577789284712748979:2048]
2025-11-28T14:52:36.448324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789282262745328:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11794
2025-11-28T14:52:36.448327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284712748979:2048] [node 1] ICS09 handshake done sender: [4:7577789284712748971:2114] self: [4:7577789284712748977:0] peer: [1:7577789282603068339:0] socket: 616 qp: -1
2025-11-28T14:52:36.448324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285633792011:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28008
2025-11-28T14:52:36.448399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284712748979:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.448425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.448462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789282262745327:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11794
2025-11-28T14:52:36.448699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285633792004:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.449136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285633792009:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.449358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789282262745326:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.449705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789282262745328:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.449804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285633792011:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.449867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789282262745334:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11794
2025-11-28T14:52:36.449929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789285633791916:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-28T14:52:36.450063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285633792016:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28008
2025-11-28T14:52:36.450310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789282262745325:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.450148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283035324221:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12351
2025-11-28T14:52:36.450462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789282262745327:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.450512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285633792010:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.450673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789282262745251:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.451385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284712748981:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45055
2025-11-28T14:52:36.450655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068350:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.451434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068336:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:36.451457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282603068336:2110] [node 2] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.451663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282603068356:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57969
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.451693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283035324222:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12351
2025-11-28T14:52:36.453073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789282603068357:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57969
2025-11-28T14:52:36.453063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283035324223:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12351
2025-11-28T14:52:36.453245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789283035324222:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.453588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068338:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.454737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789283035324206:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.453616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282603068338:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.455191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282603068336:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:36.455219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284712748970:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.455242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789284712748970:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.455685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284712748981:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.455994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789283035324221:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.455847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789282603068269:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.456066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789284712748899:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.456014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:36.456121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789282603068269:2048] [node 2] ICP22 created new session: [1:7577789282603068362:2048]
2025-11-28T14:52:36.456154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789283035324145:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.456172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282603068362:2048] [node 2] ICS09 handshake done sender: [1:7577789282603068336:2110] self: [1:7577789282603068335:0] peer: [2:7577789285633792018:0] socket: 612 qp: -1
2025-11-28T14:52:36.456205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282603068362:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:36.456219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285633792016:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.456241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:36.456208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789282262745334:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.456272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285633792006:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.456285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789285633792006:2131] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.456308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284712748970:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.456397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789282262745322:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:36.456337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068356:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:36.456475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282603068338:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.456426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789282262745322:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.456787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789285633792022:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28008
2025-11-28T14:52:36.457032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789284712748961:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.457011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789282262745339:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11794
2025-11-28T14:52:36.457075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.457419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284712748961:2108] [node 5] ICP22 created new session: [4:7577789284712748986:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.457651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284712748986:2108] [node 5] ICS09 handshake done sender: [4:7577789284712748970:2113] self: [4:7577789284712748968:0] peer: [5:7577789283035324224:0] socket: 654 qp: -1
2025-11-28T14:52:36.457700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284712748986:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:36.457704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285633792006:2131] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:36.457685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283035324226:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12351
2025-11-28T14:52:36.457730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.457775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789283035324223:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.457759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789282603068270:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.457790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789282262745252:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.457791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: 2025-11-28T14:52:36.457989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789282262745325:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-28T14:52:36.457793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068357:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.457819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789282603068270:2049] [node 3] ICP22 created new session: [1:7577789282603068365:2049]
2025-11-28T14:52:36.458215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789283035324143:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.458243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789285633792001:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.458398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789282262745251:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.458411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594549:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:36.458180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282603068365:2049] [node 3] ICS09 handshake done sender: [1:7577789282603068338:2111] self: [1:7577789282603068337:0] peer: [3:7577789282262745337:0] socket: 613 qp: -1
2025-11-28T14:52:36.458420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284712748987:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45055
2025-11-28T14:52:36.458449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.458201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282603068365:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:36.458579Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285633792004:2130] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:36.458730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:36.458246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789282603068269:2048] [node 2] ICP18 handshake (actor [1:7577789282603068356:2118]) is too old
2025-11-28T14:52:36.458633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789285633792001:2107] [node 4] ICP22 created new session: [2:7577789285633792025:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.459131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789282262745251:2048] [node 1] ICP22 created new session: [3:7577789282262745342:2048]
2025-11-28T14:52:36.458874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285633792025:2107] [node 4] ICS09 handshake done sender: [2:7577789285633792006:2131] self: [2:7577789285633792005:0] peer: [4:7577789284712748983:0] socket: 649 qp: -1
2025-11-28T14:52:36.459309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789283035324222:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:36.458401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:36.459456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594550:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.459152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285633792025:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:36.459562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789282262745342:2048] [node 1] ICS09 handshake done sender: [3:7577789282262745325:2114] self: [3:7577789282262745337:0] peer: [1:7577789282603068337:0] socket: 631 qp: -1
2025-11-28T14:52:36.459448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:36.461620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789283035324206:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.460052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789282262745322:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.460496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284712748981:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:36.461676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.460861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789282262745319:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.461773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789282603068270:2049] [node 3] ICP18 handshake (actor [1:7577789282603068357:2119]) is too old
2025-11-28T14:52:36.460923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.461690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283035324228:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12351
2025-11-28T14:52:36.462066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068342:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.460967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789282262745319:2108] [node 5] ICP22 created new session: [3:7577789282262745343:2108]
2025-11-28T14:52:36.464767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789284712748899:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.462009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283035324206:2108] [node 4] ICP22 created new session: [5:7577789283035324229:2108]
2025-11-28T14:52:36.464801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789282849594549:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.462110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282603068342:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.461012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789282262745343:2108] [node 5] ICS09 handshake done sender: [3:7577789282262745322:2112] self: [3:7577789282262745321:0] peer: [5:7577789283035324225:0] socket: 647 qp: -1
2025-11-28T14:52:36.464921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789283035324229:2108] [node 4] ICS09 handshake done sender: [5:7577789283035324222:2116] self: [5:7577789283035324224:0] peer: [4:7577789284712748968:0] socket: 655 qp: -1
2025-11-28T14:52:36.464841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:36.465182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282849594535:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.461046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789282262745343:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:36.464996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284712748899:2049] [node 2] ICP22 created new session: [4:7577789284712748989:2049]
2025-11-28T14:52:36.464963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789283035324229:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:36.465226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284712748989:2049] [node 2] ICS09 handshake done sender: [4:7577789284712748981:2117] self: [4:7577789284712748983:0] peer: [2:7577789285633792005:0] socket: 653 qp: -1
2025-11-28T14:52:36.465377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284712748989:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:36.465191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:36.461089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.465622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285633792000:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.465419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:36.461673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789282262745342:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.465658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789285633792000:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.465975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284712748975:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.461756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.465882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789285633791916:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.465998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789284712748975:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.465993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:36.466060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789284712748899:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:36.466028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789285633791916:2048] [node 1] ICP22 created new session: [2:7577789285633792027:2048]
2025-11-28T14:52:36.466057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285633792027:2048] [node 1] ICS09 handshake done sender: [2:7577789285633792004:2130] self: [2:7577789285633792018:0] peer: [1:7577789282603068335:0] socket: 615 qp: -1
2025-11-28T14:52:36.467006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282603068342:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.466077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285633792027:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.467576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.468265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789282849594550:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.468950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282849594470:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:52:36.469219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789285633791916:2048] [node 1] ICP27 obsolete handshake fail ignored
::ffff:127.1.0.0
2025-11-28T14:52:36.469196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789282603068333:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.469065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789283035324145:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4654) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789283035324221:2115] held: no
2025-11-28T14:52:36.469264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.469229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789283035324221:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.469343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789282603068333:2108] [node 5] ICP22 created new session: [1:7577789282603068368:2108]
2025-11-28T14:52:36.469390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282603068368:2108] [node 5] ICS09 handshake done sender: [1:7577789282603068342:2113] self: [1:7577789282603068341:0] peer: [5:7577789283035324227:0] socket: 656 qp: -1
2025-11-28T14:52:36.469347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594552:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:36.469421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282603068368:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:36.469470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.469542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789282603068344:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.469558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789282603068344:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.470128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789283035324145:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.470169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.470373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283035324145:2050] [node 3] ICP22 created new session: [5:7577789283035324233:2050]
2025-11-28T14:52:36.470452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789283035324233:2050] [node 3] ICS09 handshake done sender: [5:7577789283035324221:2115] self: [5:7577789283035324225:0] peer: [3:7577789282262745321:0] socket: 648 qp: -1
2025-11-28T14:52:36.470480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789283035324233:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:36.470629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284712748975:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:36.470517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.470648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284712748991:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45055
2025-11-28T14:52:36.470883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285633792000:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.471355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594554:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:36.471476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789284712748962:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.471492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789282603068344:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.471620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.471790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284712748962:2109] [node 6] ICP22 created new session: [4:7577789284712748993:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.471858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284712748991:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.472500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789285633791917:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.472268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789282603068334:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.471822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283035324231:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12351
2025-11-28T14:52:36.472340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.472713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:36.472732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789282849594552:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.472745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789285633791917:2049] [node 3] ICP22 created new session: [2:7577789285633792032:2049]
2025-11-28T14:52:36.472562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789282262745348:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11794
2025-11-28T14:52:36.472540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789282603068334:2109] [node 6] ICP22 created new session: [1:7577789282603068371:2109]
2025-11-28T14:52:36.472536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789283035324143:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789283035324223:2117] held: no
2025-11-28T14:52:36.472234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284712748993:2109] [node 6] ICS09 handshake done sender: [4:7577789284712748975:2115] self: [4:7577789284712748974:0] peer: [6:7577789282849594551:0] socket: 665 qp: -1
2025-11-28T14:52:36.472665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789282262745324:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.472774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789282603068371:2109] [node 6] ICS09 handshake done sender: [1:7577789282603068344:2114] self: [1:7577789282603068343:0] peer: [6:7577789282849594553:0] socket: 669 qp: -1
2025-11-28T14:52:36.473557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285633792032:2049] [node 3] ICS09 handshake done sender: [2:7577789285633792000:2129] self: [2:7577789285633791998:0] peer: [3:7577789282262745341:0] socket: 642 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:36.473616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285633792032:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:36.472336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789284712748900:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.472691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789282262745324:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.473112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789282603068371:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:36.473837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789282262745334:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:36.473388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.472599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284712748993:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:36.473825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789283035324223:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:36.473844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:36.472647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.473990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594555:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:36.474211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789282849594549:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:36.474582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789282262745324:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:36.474621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789283035324143:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.474655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.475479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283035324143:2048] [node 1] ICP22 created new session: [5:7577789283035324236:2048]
2025-11-28T14:52:36.476302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789282262745252:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.476046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789283035324236:2048] [node 1] ICS09 handshake done sender: [5:7577789283035324223:2117] self: [5:7577789283035324227:0] peer: [1:7577789282603068341:0] socket: 657 qp: -1
2025-11-28T14:52:36.476146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789282849594470:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789282849594550:2117] held: no
2025-11-28T14:52:36.476070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789283035324236:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.476572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282849594535:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.476492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284712748996:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45055
2025-11-28T14:52:36.476560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:36.476628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.476093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.476611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789282262745252:2049] [node 2] ICP22 created new session: [3:7577789282262745350:2049]
2025-11-28T14:52:36.476680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282849594535:2109] [node 4] ICP22 created new session: [6:7577789282849594558:2109]
2025-11-28T14:52:36.476558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283035324235:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12351
2025-11-28T14:52:36.476649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789282262745318:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.476661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789282262745350:2049] [node 2] ICS09 handshake done sender: [3:7577789282262745334:2119] self: [3:7577789282262745341:0] peer: [2:7577789285633791998:0] socket: 646 qp: -1
2025-11-28T14:52:36.476692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789282262745350:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:36.476740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789282849594558:2109] [node 4] ICS09 handshake done sender: [6:7577789282849594549:2116] self: [6:7577789282849594551:0] peer: [4:7577789284712748974:0] socket: 667 qp: -1
2025-11-28T14:52:36.476697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:36.476772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594557:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:36.476738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:36.476764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284712748991:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.476741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789282262745318:2107] [node 4] ICP22 created new session: [3:7577789282262745351:2107]
2025-11-28T14:52:36.476780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789282849594558:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:36.476781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789282262745351:2107] [node 4] ICS09 handshake done sender: [3:7577789282262745324:2113] self: [3:7577789282262745323:0] peer: [4:7577789284712748994:0] socket: 622 qp: -1
2025-11-28T14:52:36.477027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:36.476962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789283035324235:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.476806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789282262745351:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:36.477171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789283035324144:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.477171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789284712748900:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.476844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:36.477215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:36.477193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789282262745252:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:36.477249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284712748900:2050] [node 3] ICP22 created new session: [4:7577789284712748997:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.477351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789284712748997:2050] [node 3] ICS09 handshake done sender: [4:7577789284712748991:2119] self: [4:7577789284712748994:0] peer: [3:7577789282262745323:0] socket: 635 qp: -1
2025-11-28T14:52:36.477389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789284712748997:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:36.477416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:36.477452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594556:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:36.477439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285633792008:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.477465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789285633792008:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.477559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282849594472:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.478964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789282849594555:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.478980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789282262745330:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.479022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789282849594550:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:36.479006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789282262745330:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.479030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285633792008:2132] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.479026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789282849594556:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.479190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789285633792002:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.479691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.479679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282849594470:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.479688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283035324239:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12351
2025-11-28T14:52:36.479716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282849594533:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.479735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789285633792002:2108] [node 5] ICP22 created new session: [2:7577789285633792035:2108]
2025-11-28T14:52:36.479740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:36.479767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282849594470:2048] [node 1] ICP22 created new session: [6:7577789282849594563:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.479814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285633792035:2108] [node 5] ICS09 handshake done sender: [2:7577789285633792008:2132] self: [2:7577789285633792007:0] peer: [5:7577789283035324238:0] socket: 673 qp: -1
2025-11-28T14:52:36.479807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789282849594563:2048] [node 1] ICS09 handshake done sender: [6:7577789282849594550:2117] self: [6:7577789282849594553:0] peer: [1:7577789282603068343:0] socket: 670 qp: -1
2025-11-28T14:52:36.479824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789282849594563:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:36.479849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285633792035:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:36.479906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.479864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283035324240:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12351
2025-11-28T14:52:36.479841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282849594471:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.479844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:36.479889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789282262745330:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:36.479968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789283035324235:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:36.480686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789282262745320:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.480976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789285633792015:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.481007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789285633792015:2136] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.480923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.481181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789283035324144:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26733) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789283035324235:2121] held: no
2025-11-28T14:52:36.481185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789283035324216:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.481051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789282262745320:2109] [node 6] ICP22 created new session: [3:7577789282262745355:2109]
2025-11-28T14:52:36.481206Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789283035324216:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:36.481513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789283035324144:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.481186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789282262745355:2109] [node 6] ICS09 handshake done sender: [3:7577789282262745330:2118] self: [3:7577789282262745329:0] peer: [6:7577789282849594560:0] socket: 623 qp: -1
2025-11-28T14:52:36.481549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:36.481294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789282262745355:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:36.481518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.481586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283035324144:2049] [node 2] ICP22 created new session: [5:7577789283035324241:2049]
2025-11-28T14:52:36.481636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789283035324241:2049] [node 2] ICS09 handshake done sender: [5:7577789283035324235:2121] self: [5:7577789283035324238:0] peer: [2:7577789285633792007:0] socket: 679 qp: -1
2025-11-28T14:52:36.481904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789283035324241:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:36.481941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:36.481993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789285633792015:2136] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:36.482113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789283035324216:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:36.482225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789285633792003:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.482587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.482643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789285633792003:2117] [node 6] ICP22 created new session: [2:7577789285633792038:2117]
2025-11-28T14:52:36.482827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789285633792038:2117] [node 6] ICS09 handshake done sender: [2:7577789285633792015:2136] self: [2:7577789285633792014:0] peer: [6:7577789282849594564:0] socket: 676 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.482859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789285633792038:2117] [node 6] ICS10 traffic start
2025-11-28T14:52:36.482926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.482926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594566:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:36.483104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789283035324207:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:36.483144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:36.483183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283035324207:2109] [node 6] ICP22 created new session: [5:7577789283035324244:2109]
2025-11-28T14:52:36.483180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789282849594472:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4654) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789282849594552:2118] held: no
2025-11-28T14:52:36.483258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789283035324244:2109] [node 6] ICS09 handshake done sender: [5:7577789283035324216:2113] self: [5:7577789283035324215:0] peer: [6:7577789282849594562:0] socket: 644 qp: -1
2025-11-28T14:52:36.483299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789283035324244:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:36.483337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:36.483647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789282849594552:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:36.483667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789283035324240:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:36.483763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789283035324207:2109] [node 6] ICP18 handshake (actor [5:7577789283035324240:2123]) is too old
2025-11-28T14:52:36.483864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789282849594471:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26733) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789282849594556:2121] held: no
2025-11-28T14:52:36.484033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282849594472:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.484087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:36.484115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282849594472:2050] [node 3] ICP22 created new session: [6:7577789282849594569:2050]
2025-11-28T14:52:36.484146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789282849594569:2050] [node 3] ICS09 handshake done sender: [6:7577789282849594552:2118] self: [6:7577789282849594560:0] peer: [3:7577789282262745329:0] socket: 626 qp: -1
2025-11-28T14:52:36.484166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789282849594569:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:36.484185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:36.484523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789282849594533:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16176) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789282849594555:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.484660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594567:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.484974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789282849594568:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:36.485038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789282849594556:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:36.485160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282849594471:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.485187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:36.485241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282849594471:2049] [node 2] ICP22 created new session: [6:7577789282849594571:2049]
2025-11-28T14:52:36.485273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789282849594571:2049] [node 2] ICS09 handshake done sender: [6:7577789282849594556:2121] self: [6:7577789282849594564:0] peer: [2:7577789285633792014:0] socket: 617 qp: -1
2025-11-28T14:52:36.485292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789282849594571:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:36.485320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789282849594555:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:36.485324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:36.485590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282849594533:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:36.485641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:36.485679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282849594533:2106] [node 5] ICP22 created new session: [6:7577789282849594573:2106]
2025-11-28T14:52:36.485718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789282849594573:2106] [node 5] ICS09 handshake done sender: [6:7577789282849594555:2120] self: [6:7577789282849594562:0] peer: [5:7577789283035324215:0] socket: 672 qp: -1
2025-11-28T14:52:36.485747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789282849594573:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:36.485800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:36.498453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:36.498607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789285633792027:2048] [node 1] ICS07 socket disconnect 615 reason# EndOfStream
2025-11-28T14:52:36.498632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789285633792027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.498738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.498774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789285633792027:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:36.499576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:36.499765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789282262745342:2048] [node 1] ICS07 socket disconnect 631 reason# EndOfStream
2025-11-28T14:52:36.499798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789282262745342:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.499886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.499944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789282262745342:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:36.500485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789285633791916:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:36.500532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789285633792027:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:36.500906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789282262745251:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:36.500974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789282262745342:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:36.501021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789282262745251:2048] [node 1] ICP30 unregister session Session# [3:7577789282262745342:2048] VirtualId# [3:7577789282262745337:0]
2025-11-28T14:52:36.501071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.501094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789282262745251:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.501104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.501735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789285633791916:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:36.501795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789285633792027:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:36.501829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789285633791916:2048] [node 1] ICP30 unregister session Session# [2:7577789285633792027:2048] VirtualId# [2:7577789285633792018:0]
2025-11-28T14:52:36.501860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.501879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789285633791916:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.501886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.502059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:36.502287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789284712748979:2048] [node 1] ICS07 socket disconnect 616 reason# EndOfStream
2025-11-28T14:52:36.502316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789284712748979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.502393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.502439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789284712748979:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:36.502750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:36.502907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789283035324236:2048] [node 1] ICS01 socket: 657 reason# EndOfStream
2025-11-28T14:52:36.502975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789284712748898:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:36.502948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789283035324143:2048] [node 1] ICP30 unregister session Session# [5:7577789283035324236:2048] VirtualId# [5:7577789283035324227:0]
2025-11-28T14:52:36.503026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789284712748979:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:36.503058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789284712748898:2048] [node 1] ICP30 unregister session Session# [4:7577789284712748979:2048] VirtualId# [4:7577789284712748977:0]
2025-11-28T14:52:36.503094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.502981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789283035324236:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.503119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789284712748898:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.503102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.503137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.503162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.503518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:36.503670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789282849594563:2048] [node 1] ICS01 socket: 670 reason# EndOfStream
2025-11-28T14:52:36.503699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282849594470:2048] [node 1] ICP30 unregister session Session# [6:7577789282849594563:2048] VirtualId# [6:7577789282849594553:0]
2025-11-28T14:52:36.503722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789282849594563:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.503776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:36.503819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:36.511373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789285633791916:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:36.511386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789282262745251:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:36.513416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789284712748898:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:36.529135Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:36.529402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789282262745350:2049] [node 2] ICS07 socket disconnect 646 reason# EndOfStream
2025-11-28T14:52:36.529441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789282262745350:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.529532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:36.529574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789282262745350:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:36.531557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:36.531575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789282262745252:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26733) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:36.531782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789284712748989:2049] [node 2] ICS01 socket: 653 reason# EndOfStream
2025-11-28T14:52:36.531834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789284712748899:2049] [node 2] ICP30 unregister session Session# [4:7577789284712748989:2049] VirtualId# [4:7577789284712748983:0]
2025-11-28T14:52:36.531604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789282262745350:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:36.531865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789284712748989:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.531942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:36.532002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:36.532320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789282262745252:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26733) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:36.532584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789282262745350:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:36.532659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789282262745252:2049] [node 2] ICP30 unregister session Session# [3:7577789282262745350:2049] VirtualId# [3:7577789282262745341:0]
2025-11-28T14:52:36.532720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:36.532757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789282262745252:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.532776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:36.533026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:36.533181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789283035324241:2049] [node 2] ICS01 socket: 679 reason# EndOfStream
2025-11-28T14:52:36.533215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789283035324144:2049] [node 2] ICP30 unregister session Session# [5:7577789283035324241:2049] VirtualId# [5:7577789283035324238:0]
2025-11-28T14:52:36.533235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789283035324241:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.533289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:36.533324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:36.534101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:36.534325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789282849594571:2049] [node 2] ICS01 socket: 617 reason# EndOfStream
2025-11-28T14:52:36.534375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282849594471:2049] [node 2] ICP30 unregister session Session# [6:7577789282849594571:2049] VirtualId# [6:7577789282849594564:0]
2025-11-28T14:52:36.534409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789282849594571:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.534486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:36.534534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:36.542124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789282262745252:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:36.586070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:36.586288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789284712748997:2050] [node 3] ICS01 socket: 635 reason# EndOfStream
2025-11-28T14:52:36.586339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789284712748900:2050] [node 3] ICP30 unregister session Session# [4:7577789284712748997:2050] VirtualId# [4:7577789284712748994:0]
2025-11-28T14:52:36.586390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789284712748997:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.586493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:36.586545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:36.586851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:36.586980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789283035324233:2050] [node 3] ICS01 socket: 648 reason# EndOfStream
2025-11-28T14:52:36.587009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789283035324145:2050] [node 3] ICP30 unregister session Session# [5:7577789283035324233:2050] VirtualId# [5:7577789283035324225:0]
2025-11-28T14:52:36.587029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789283035324233:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.587089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:36.587125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:36.587639Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:36.587781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789282849594569:2050] [node 3] ICS01 socket: 626 reason# EndOfStream
2025-11-28T14:52:36.587810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282849594472:2050] [node 3] ICP30 unregister session Session# [6:7577789282849594569:2050] VirtualId# [6:7577789282849594560:0]
2025-11-28T14:52:36.587830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789282849594569:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.587889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:36.587934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:36.624618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:36.624862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789283035324229:2108] [node 4] ICS01 socket: 655 reason# EndOfStream
2025-11-28T14:52:36.624959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789283035324206:2108] [node 4] ICP30 unregister session Session# [5:7577789283035324229:2108] VirtualId# [5:7577789283035324224:0]
2025-11-28T14:52:36.625012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789283035324229:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.625137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:36.625205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:36.625461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:36.625731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789282849594558:2109] [node 4] ICS01 socket: 667 reason# EndOfStream
2025-11-28T14:52:36.625782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282849594535:2109] [node 4] ICP30 unregister session Session# [6:7577789282849594558:2109] VirtualId# [6:7577789282849594551:0]
2025-11-28T14:52:36.625812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789282849594558:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.625891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:36.625955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:36.685424Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:36.685797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789282849594573:2106] [node 5] ICS01 socket: 672 reason# EndOfStream
2025-11-28T14:52:36.685860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282849594533:2106] [node 5] ICP30 unregister session Session# [6:7577789282849594573:2106] VirtualId# [6:7577789282849594562:0]
2025-11-28T14:52:36.685893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789282849594573:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:36.686005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:36.686067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:36.764762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789283168613111:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:36.764814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789283168613112:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:36.783519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789282353857897:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.783562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789282353857898:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:36.805066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789285802750069:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.805077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789285802750070:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:36.826925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284985293220:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.826971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284985293222:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:36.827054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284985293221:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:36.859609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283683432253:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.859793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283683432254:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:36.859815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283683432255:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:36.886599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282509333877:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:36.886640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282509333878:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:36.886675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282509333879:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:36.957482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282509333939:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:36.985914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789282353857963:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:36.982878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789283168613174:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:36.985298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283683432316:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:36.985301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284985293283:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.985775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789285802750133:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:36.985764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789283168613175:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.986338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789282353857961:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:36.986388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789282509333941:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:36.986669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789285802750135:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:36.986759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789284985293284:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:36.987056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789283168613176:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.986877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789285802750134:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:36.986591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789282353857962:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:36.987061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789283683432317:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:36.989371Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789283168613111:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52354786ns) {Configure:104(52349052ns) {ProcessConfigured:135(47851ns) {StartInitialHandshake:157(45284ns) {DropHandshakes:470(1080ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(4444ns) {SwitchToState:179(211ns)}}}}}
2025-11-28T14:52:36.989384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789283168613112:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52361409ns) {Configure:104(52358285ns) {ProcessConfigured:135(65901ns) {StartInitialHandshake:157(63807ns) {DropHandshakes:470(2208ns) {DropIncomingHandshake:442(580ns) DropOutgoingHandshake:457(311ns)} GenerateSessionVirtualId:988(614ns) PrepareNewSessionHandshake:477(1025ns) {SwitchToState:179(130ns)}}}}}
2025-11-28T14:52:36.990665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789285802750069:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(45485647ns) {Configure:104(45478581ns) {ProcessConfigured:135(161250ns) {StartInitialHandshake:157(158875ns) {DropHandshakes:470(1040ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(412ns) PrepareNewSessionHandshake:477(4478ns) {SwitchToState:179(470ns)}}}}}
2025-11-28T14:52:36.992252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789283683432254:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45648284ns) {Configure:104(26417224ns) {ProcessConfigured:135(147349ns) {StartInitialHandshake:157(145278ns) {DropHandshakes:470(1089ns) {DropIncomingHandshake:442(395ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:477(567ns) {SwitchToState:179(100ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.992585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789283683432255:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43774061ns) {Configure:104(43771560ns) {ProcessConfigured:135(161459ns) {StartInitialHandshake:157(159570ns) {DropHandshakes:470(2127ns) {DropIncomingHandshake:442(588ns) DropOutgoingHandshake:457(347ns)} GenerateSessionVirtualId:988(537ns) PrepareNewSessionHandshake:477(1231ns) {SwitchToState:179(548ns)}}}}}
2025-11-28T14:52:36.993797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789285802750142:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57703
2025-11-28T14:52:36.997355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789285802750142:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.997936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789285802750069:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:36.998209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789283168613190:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40050
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:36.998941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789283168613191:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40050
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:37.000994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789283168613192:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40050
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.001805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789285802750150:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57703
2025-11-28T14:52:37.001828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284985293291:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63000
2025-11-28T14:52:37.001458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789283168613178:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.001491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789283168613178:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.004195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789283683432331:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16937
2025-11-28T14:52:37.004093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284985293291:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.004550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789282353857970:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36717
2025-11-28T14:52:37.003849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789283168613191:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.003905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789283168613178:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:37.005061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789285802750150:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.005006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789282353857971:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36717
2025-11-28T14:52:37.005338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789284985293294:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63000
2025-11-28T14:52:37.004607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789283168613190:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.005838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789283683432331:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.005126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789285802750152:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57703
2025-11-28T14:52:37.005750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789282353857971:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.004654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789283168613192:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.006201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789283168613112:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.006467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789282353857970:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.006354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789284985293220:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.006257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.006513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789286648825272:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36717
2025-11-28T14:52:37.006372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789283168613112:2049] [node 3] ICP22 created new session: [1:7577789287463580494:2049]
2025-11-28T14:52:37.006534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287463580494:2049] [node 3] ICS09 handshake done sender: [1:7577789283168613178:2110] self: [1:7577789283168613177:0] peer: [3:7577789285802750151:0] socket: 618 qp: -1
2025-11-28T14:52:37.006633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287463580494:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:37.006644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284985293294:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.006687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789282353857897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.006715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.006833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289280260596:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63000
2025-11-28T14:52:37.006835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789284985293222:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.006950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789286648825276:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36717
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.006953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789290097717449:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57703
2025-11-28T14:52:37.007221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789286648825272:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.006980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789285802750152:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.008601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789290097717450:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57703
2025-11-28T14:52:37.009536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789285802750139:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.008766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789283168613182:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.009550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789285802750139:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.008802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789283168613182:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.010517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789290097717451:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57703
2025-11-28T14:52:37.010597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789287463580493:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40050
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.011821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789285802750069:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789285802750142:2114] held: no
2025-11-28T14:52:37.010666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789283168613181:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.011896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789285802750139:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.010685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789283168613181:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.013507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789285802750142:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.013471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789283168613182:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.014210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789285802750069:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.013521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789287463580495:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40050
2025-11-28T14:52:37.014271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789289280260596:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.014258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789290097717451:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.014260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.013686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789283168613181:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.014720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789284985293221:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.014280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789286648825276:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:37.014430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789285802750133:2107] [node 4] ICP20 outgoing handshake succeeded
::ffff:127.1.0.02025-11-28T14:52:37.014591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789283168613111:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.014840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789286648825280:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36717
2025-11-28T14:52:37.014711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789285802750069:2048] [node 1] ICP22 created new session: [3:7577789290097717456:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.014781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:37.014907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287978399629:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16937
2025-11-28T14:52:37.014887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289280260599:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63000
2025-11-28T14:52:37.014761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789290097717456:2048] [node 1] ICS09 handshake done sender: [3:7577789285802750142:2114] self: [3:7577789285802750151:0] peer: [1:7577789283168613177:0] socket: 621 qp: -1
2025-11-28T14:52:37.015000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289280260600:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63000
2025-11-28T14:52:37.014975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789282353857966:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.014833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789283168613174:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.014990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301251:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.014763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.015009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789282353857966:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.014889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789283168613111:2048] [node 2] ICP22 created new session: [1:7577789287463580500:2048]
2025-11-28T14:52:37.014844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789290097717456:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.014893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.014858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789285802750133:2107] [node 4] ICP22 created new session: [3:7577789290097717457:2107]
2025-11-28T14:52:37.014928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789283168613174:2107] [node 4] ICP22 created new session: [1:7577789287463580501:2107]
2025-11-28T14:52:37.015678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789282353857970:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.015937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284985293294:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:37.014884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.017063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789282353857966:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.014960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287463580501:2107] [node 4] ICS09 handshake done sender: [1:7577789283168613182:2112] self: [1:7577789283168613179:0] peer: [4:7577789289280260597:0] socket: 620 qp: -1
2025-11-28T14:52:37.014980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287463580501:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:37.015958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284985293291:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.014900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789290097717457:2107] [node 4] ICS09 handshake done sender: [3:7577789285802750139:2112] self: [3:7577789285802750138:0] peer: [4:7577789289280260598:0] socket: 625 qp: -1
2025-11-28T14:52:37.017367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789282353857897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.016941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789284985293220:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.014988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287463580500:2048] [node 2] ICS09 handshake done sender: [1:7577789283168613181:2111] self: [1:7577789283168613180:0] peer: [2:7577789286648825278:0] socket: 619 qp: -1
2025-11-28T14:52:37.017923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.014980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789290097717457:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:37.017049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789284985293222:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.017969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789282353857897:2048] [node 1] ICP22 created new session: [2:7577789286648825283:2048]
2025-11-28T14:52:37.015005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.015007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.017092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.018446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789286648825283:2048] [node 1] ICS09 handshake done sender: [2:7577789282353857970:2114] self: [2:7577789286648825278:0] peer: [1:7577789283168613180:0] socket: 626 qp: -1
2025-11-28T14:52:37.015017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287463580500:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:37.018803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789290097717450:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.018537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789286648825283:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.018802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.015052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.017201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284985293222:2050] [node 3] ICP22 created new session: [4:7577789289280260602:2050]
2025-11-28T14:52:37.019477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789286804301251:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.019028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789282353857961:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.019651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287978399629:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.017870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287463580495:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.019832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789285802750070:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.017369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.019062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.018811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287463580493:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.019088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789282353857961:2107] [node 4] ICP22 created new session: [2:7577789286648825285:2107]
2025-11-28T14:52:37.020158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789283683432253:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.020454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282509333879:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.017531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289280260602:2050] [node 3] ICS09 handshake done sender: [4:7577789284985293294:2114] self: [4:7577789289280260598:0] peer: [3:7577789285802750138:0] socket: 643 qp: -1
2025-11-28T14:52:37.019473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789286648825285:2107] [node 4] ICS09 handshake done sender: [2:7577789282353857966:2111] self: [2:7577789282353857965:0] peer: [4:7577789289280260601:0] socket: 653 qp: -1
2025-11-28T14:52:37.020407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789285802750147:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.020407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789283168613174:2107] [node 4] ICP18 handshake (actor [1:7577789287463580493:2118]) is too old
2025-11-28T14:52:37.020027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789286648825285:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:37.017556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289280260602:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:37.020447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789285802750147:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.020168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.017815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284985293220:2048] [node 1] ICP22 created new session: [4:7577789289280260604:2048]
2025-11-28T14:52:37.020625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789283168613111:2048] [node 2] ICP18 handshake (actor [1:7577789287463580495:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.020624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789285802750133:2107] [node 4] ICP18 handshake (actor [3:7577789290097717450:2120]) is too old
2025-11-28T14:52:37.020728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789283168613184:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.017928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289280260604:2048] [node 1] ICS09 handshake done sender: [4:7577789284985293291:2113] self: [4:7577789289280260597:0] peer: [1:7577789283168613179:0] socket: 623 qp: -1
2025-11-28T14:52:37.018012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:37.020756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789283168613184:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.022746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287978399631:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16937
2025-11-28T14:52:37.022594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301252:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.021953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789285802750147:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:37.018133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289280260604:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.018578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.023225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789282353857969:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.023256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789282353857969:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.023549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789285802750135:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.023780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301254:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.023923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287978399631:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.023895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.023937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789284985293221:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789289280260596:2117] held: no
2025-11-28T14:52:37.024021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789283168613184:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:37.024446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289280260603:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63000
2025-11-28T14:52:37.024538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789283683432255:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.024491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789285802750135:2110] [node 6] ICP22 created new session: [3:7577789290097717463:2110]
2025-11-28T14:52:37.025098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789282353857969:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:37.025261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789283168613175:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.024764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789290097717463:2110] [node 6] ICS09 handshake done sender: [3:7577789285802750147:2115] self: [3:7577789285802750143:0] peer: [6:7577789286804301253:0] socket: 665 qp: -1
2025-11-28T14:52:37.025956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789289280260596:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.026074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789282353857898:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.026363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.028859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789282509333879:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26593) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789286804301251:2115] held: no
2025-11-28T14:52:37.027855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789284985293221:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.024917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789290097717463:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:37.027368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.027401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789283168613175:2108] [node 5] ICP22 created new session: [1:7577789287463580506:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:37.028042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-28T14:52:37.025433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:37.028084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284985293221:2049] [node 2] ICP22 created new session: [4:7577789289280260609:2049]
2025-11-28T14:52:37.029456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287978399634:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16937
2025-11-28T14:52:37.029304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287463580506:2108] [node 5] ICS09 handshake done sender: [1:7577789283168613184:2113] self: [1:7577789283168613183:0] peer: [5:7577789287978399630:0] socket: 624 qp: -1
2025-11-28T14:52:37.028897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789282353857898:2049] [node 3] ICP22 created new session: [2:7577789286648825290:2049]
2025-11-28T14:52:37.026625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789285802750149:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.029327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289280260609:2049] [node 2] ICS09 handshake done sender: [4:7577789289280260596:2117] self: [4:7577789289280260601:0] peer: [2:7577789282353857965:0] socket: 656 qp: -1
2025-11-28T14:52:37.029511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287463580506:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:37.029734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789286804301251:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:37.029340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789286648825290:2049] [node 3] ICS09 handshake done sender: [2:7577789282353857969:2113] self: [2:7577789282353857968:0] peer: [3:7577789290097717461:0] socket: 655 qp: -1
2025-11-28T14:52:37.026660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789285802750149:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.030600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287978399632:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16937
2025-11-28T14:52:37.029977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289280260609:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:37.029571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:37.030624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287978399635:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16937
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.029504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789286648825290:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:37.030941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.030723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789285802750070:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789290097717451:2121] held: no
2025-11-28T14:52:37.030862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282509333879:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:37.031063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789283683432253:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789287978399629:2116] held: no
2025-11-28T14:52:37.029727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0
2025-11-28T14:52:37.031075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789285802750149:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:37.030883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789286804301252:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.031915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287978399634:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.031403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.032137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789290097717465:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57703
2025-11-28T14:52:37.031440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282509333879:2050] [node 3] ICP22 created new session: [6:7577789286804301258:2050]
2025-11-28T14:52:37.031474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789286804301258:2050] [node 3] ICS09 handshake done sender: [6:7577789286804301251:2115] self: [6:7577789286804301253:0] peer: [3:7577789285802750143:0] socket: 666 qp: -1
2025-11-28T14:52:37.032197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789285802750134:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.031599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789286804301258:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:37.032525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789287978399629:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.032560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:37.031733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301255:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.032532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287978399635:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.032604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789285802750134:2109] [node 5] ICP22 created new session: [3:7577789290097717467:2109]
2025-11-28T14:52:37.031740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.032332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789286804301255:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.032629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789290097717467:2109] [node 5] ICS09 handshake done sender: [3:7577789285802750149:2116] self: [3:7577789285802750148:0] peer: [5:7577789287978399633:0] socket: 654 qp: -1
2025-11-28T14:52:37.032673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287978399636:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16937
2025-11-28T14:52:37.032654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789290097717467:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:37.032415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301256:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.032719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789283683432254:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.033077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789283168613189:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.032967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789286648825292:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36717
2025-11-28T14:52:37.032715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:37.032969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789283683432316:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.032538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282509333877:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.033104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789283168613189:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.033132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789290097717451:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.034306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789283168613189:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:37.034050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789285802750070:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.033003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789283683432253:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.032902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282509333878:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.034206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:37.033112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.034172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789282353857973:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.033224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283683432253:2048] [node 1] ICP22 created new session: [5:7577789287978399640:2048]
2025-11-28T14:52:37.034572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284985293296:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.034207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789282353857973:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.032912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789286804301256:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.033351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287978399640:2048] [node 1] ICS09 handshake done sender: [5:7577789287978399629:2116] self: [5:7577789287978399630:0] peer: [1:7577789283168613183:0] socket: 662 qp: -1
2025-11-28T14:52:37.034606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789284985293296:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.034965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789285802750070:2049] [node 2] ICP22 created new session: [3:7577789290097717469:2049]
2025-11-28T14:52:37.034199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287978399640:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.034842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789282353857973:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:37.033345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282509333941:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.034627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.035289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789290097717469:2049] [node 2] ICS09 handshake done sender: [3:7577789290097717451:2121] self: [3:7577789290097717461:0] peer: [2:7577789282353857968:0] socket: 657 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.035239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789282353857975:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.035290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789284985293295:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.035260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789282353857975:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.036488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789290097717469:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:37.035283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789283683432255:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26593) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789287978399631:2117] held: no
2025-11-28T14:52:37.036462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789283168613176:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.037179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789287978399631:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:37.036537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.035310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789284985293295:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.036518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789282353857962:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.036509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.035480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284985293296:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:37.036556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.037538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301263:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.036549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789283168613176:2109] [node 6] ICP22 created new session: [1:7577789287463580509:2109]
2025-11-28T14:52:37.037913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789283683432255:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.036711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789282353857962:2110] [node 6] ICP22 created new session: [2:7577789286648825294:2110]
2025-11-28T14:52:37.037508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289280260611:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63000
2025-11-28T14:52:37.037107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287463580509:2109] [node 6] ICS09 handshake done sender: [1:7577789283168613189:2114] self: [1:7577789283168613188:0] peer: [6:7577789286804301260:0] socket: 649 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.037021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789286648825292:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.038097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789286804301252:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.037544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789284985293295:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:37.037925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789286648825294:2110] [node 6] ICS09 handshake done sender: [2:7577789282353857973:2116] self: [2:7577789282353857972:0] peer: [6:7577789286804301261:0] socket: 675 qp: -1
2025-11-28T14:52:37.038115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.037143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287463580509:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:37.037549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789284985293283:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.037962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789286648825294:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:37.038615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301265:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.038616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283683432255:2050] [node 3] ICP22 created new session: [5:7577789287978399643:2050]
2025-11-28T14:52:37.038140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789284985293284:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.038044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:37.038003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:37.038171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789282353857975:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:37.038159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:37.038833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287978399643:2050] [node 3] ICS09 handshake done sender: [5:7577789287978399631:2117] self: [5:7577789287978399633:0] peer: [3:7577789285802750148:0] socket: 672 qp: -1
2025-11-28T14:52:37.038786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789282353857963:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.038869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301264:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.038096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.038855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287978399643:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:37.038853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:37.038974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:37.038211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284985293283:2109] [node 6] ICP22 created new session: [4:7577789289280260614:2109]
2025-11-28T14:52:37.038904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789282353857963:2109] [node 5] ICP22 created new session: [2:7577789286648825297:2109]
2025-11-28T14:52:37.038672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789284985293284:2108] [node 5] ICP22 created new session: [4:7577789289280260615:2108]
2025-11-28T14:52:37.039164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789282353857962:2110] [node 6] ICP18 handshake (actor [2:7577789286648825292:2121]) is too old
2025-11-28T14:52:37.039333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789286648825297:2109] [node 5] ICS09 handshake done sender: [2:7577789282353857975:2117] self: [2:7577789282353857974:0] peer: [5:7577789287978399637:0] socket: 659 qp: -1
2025-11-28T14:52:37.038714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289280260615:2108] [node 5] ICS09 handshake done sender: [4:7577789284985293295:2115] self: [4:7577789284985293293:0] peer: [5:7577789287978399639:0] socket: 646 qp: -1
2025-11-28T14:52:37.039359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789286648825297:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:37.039954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282509333877:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.038716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289280260614:2109] [node 6] ICS09 handshake done sender: [4:7577789284985293296:2116] self: [4:7577789284985293292:0] peer: [6:7577789286804301262:0] socket: 679 qp: -1
2025-11-28T14:52:37.040132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.039462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.038742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289280260615:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:37.040708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287978399642:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16937
2025-11-28T14:52:37.040510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282509333877:2048] [node 1] ICP22 created new session: [6:7577789286804301266:2048]
2025-11-28T14:52:37.038742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289280260614:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:37.040674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789286804301266:2048] [node 1] ICS09 handshake done sender: [6:7577789286804301252:2116] self: [6:7577789286804301260:0] peer: [1:7577789283168613188:0] socket: 650 qp: -1
2025-11-28T14:52:37.038882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.040735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789286804301255:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.038978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:37.040786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789286804301256:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.041955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287978399644:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16937
2025-11-28T14:52:37.040887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789286804301266:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.040928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.042240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789283683432254:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789287978399634:2119] held: no
2025-11-28T14:52:37.042113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282509333878:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.042293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:37.042373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282509333878:2049] [node 2] ICP22 created new session: [6:7577789286804301269:2049]
2025-11-28T14:52:37.042416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789286804301269:2049] [node 2] ICS09 handshake done sender: [6:7577789286804301255:2118] self: [6:7577789286804301261:0] peer: [2:7577789282353857972:0] socket: 676 qp: -1
2025-11-28T14:52:37.042426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789282509333877:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:37.042456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789286804301269:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:37.042477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282509333941:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.042491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.042504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789287978399634:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.042513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.042547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282509333941:2108] [node 4] ICP22 created new session: [6:7577789286804301271:2108]
2025-11-28T14:52:37.042517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789287978399635:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.042571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789289280260611:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.042809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789284985293284:2108] [node 5] ICP18 handshake (actor [4:7577789289280260611:2121]) is too old
2025-11-28T14:52:37.042638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789286804301271:2108] [node 4] ICS09 handshake done sender: [6:7577789286804301256:2119] self: [6:7577789286804301262:0] peer: [4:7577789284985293292:0] socket: 628 qp: -1
2025-11-28T14:52:37.042679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789286804301271:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:37.042699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.042980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789283683432254:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.043058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:37.043087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789283683432316:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.043117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283683432254:2049] [node 2] ICP22 created new session: [5:7577789287978399647:2049]
2025-11-28T14:52:37.043119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.043152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283683432316:2108] [node 4] ICP22 created new session: [5:7577789287978399648:2108]
2025-11-28T14:52:37.043155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287978399647:2049] [node 2] ICS09 handshake done sender: [5:7577789287978399634:2119] self: [5:7577789287978399637:0] peer: [2:7577789282353857974:0] socket: 674 qp: -1
2025-11-28T14:52:37.043248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287978399647:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:37.043260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287978399648:2108] [node 4] ICS09 handshake done sender: [5:7577789287978399635:2120] self: [5:7577789287978399639:0] peer: [4:7577789284985293293:0] socket: 678 qp: -1
2025-11-28T14:52:37.043281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.043297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287978399648:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:37.043346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.043428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789282509333878:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.043824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301267:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.045097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789286804301267:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.045223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789282509333939:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.045420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789283683432330:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.045446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789283683432330:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.046093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789283683432330:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.046286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789283683432317:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.046271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789286804301275:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49230
2025-11-28T14:52:37.046359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.046410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789283683432317:2109] [node 6] ICP22 created new session: [5:7577789287978399652:2109]
2025-11-28T14:52:37.046458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287978399652:2109] [node 6] ICS09 handshake done sender: [5:7577789283683432330:2114] self: [5:7577789283683432329:0] peer: [6:7577789286804301274:0] socket: 636 qp: -1
2025-11-28T14:52:37.046492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287978399652:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:37.046506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789286804301267:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:37.046530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:37.047474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789282509333939:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789286804301267:2123] held: no
2025-11-28T14:52:37.047595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789282509333939:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.047654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:37.047712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789282509333939:2105] [node 5] ICP22 created new session: [6:7577789286804301276:2105]
2025-11-28T14:52:37.047763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789286804301276:2105] [node 5] ICS09 handshake done sender: [6:7577789286804301267:2123] self: [6:7577789286804301274:0] peer: [5:7577789283683432329:0] socket: 637 qp: -1
2025-11-28T14:52:37.047815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789286804301276:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:37.047869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:37.091090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:37.091326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789286648825283:2048] [node 1] ICS07 socket disconnect 626 reason# EndOfStream
2025-11-28T14:52:37.091358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789286648825283:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.091486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.091543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789286648825283:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:37.091997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:37.092212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789290097717456:2048] [node 1] ICS01 socket: 621 reason# EndOfStream
2025-11-28T14:52:37.092270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789285802750069:2048] [node 1] ICP30 unregister session Session# [3:7577789290097717456:2048] VirtualId# [3:7577789285802750151:0]
2025-11-28T14:52:37.092308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789290097717456:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.092399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.092450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.094288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789282353857897:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:37.094330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789286648825283:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:37.094328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:37.094701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789289280260604:2048] [node 1] ICS07 socket disconnect 623 reason# EndOfStream
2025-11-28T14:52:37.094732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789289280260604:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.094799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.094829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789289280260604:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:37.095251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789282353857897:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.095330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789286648825283:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.095366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789282353857897:2048] [node 1] ICP30 unregister session Session# [2:7577789286648825283:2048] VirtualId# [2:7577789286648825278:0]
2025-11-28T14:52:37.095378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789284985293220:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.095412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.095335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:37.095530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789289280260604:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.095452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789282353857897:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.095554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789284985293220:2048] [node 1] ICP30 unregister session Session# [4:7577789289280260604:2048] VirtualId# [4:7577789289280260597:0]
2025-11-28T14:52:37.095463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.095582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.095611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789284985293220:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.095651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789287978399640:2048] [node 1] ICS01 socket: 662 reason# EndOfStream
2025-11-28T14:52:37.095709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789283683432253:2048] [node 1] ICP30 unregister session Session# [5:7577789287978399640:2048] VirtualId# [5:7577789287978399630:0]
2025-11-28T14:52:37.095621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.095731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789287978399640:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.095817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.095862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.096314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:37.096548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789286804301266:2048] [node 1] ICS07 socket disconnect 650 reason# EndOfStream
2025-11-28T14:52:37.096590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789286804301266:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.096657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.096706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789286804301266:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:37.097325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789282509333877:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.097372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789286804301266:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.097393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282509333877:2048] [node 1] ICP30 unregister session Session# [6:7577789286804301266:2048] VirtualId# [6:7577789286804301260:0]
2025-11-28T14:52:37.097459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.097486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789282509333877:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.097504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.105303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789284985293220:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:37.107369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789282509333877:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:37.133826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:37.134058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789290097717469:2049] [node 2] ICS01 socket: 657 reason# EndOfStream
2025-11-28T14:52:37.134106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789285802750070:2049] [node 2] ICP30 unregister session Session# [3:7577789290097717469:2049] VirtualId# [3:7577789290097717461:0]
2025-11-28T14:52:37.134137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789290097717469:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.134260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:37.134326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:37.135497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:37.135682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789289280260609:2049] [node 2] ICS01 socket: 656 reason# EndOfStream
2025-11-28T14:52:37.135723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789284985293221:2049] [node 2] ICP30 unregister session Session# [4:7577789289280260609:2049] VirtualId# [4:7577789289280260601:0]
2025-11-28T14:52:37.135761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789289280260609:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.135835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:37.135869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:37.136216Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:37.136373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789287978399647:2049] [node 2] ICS01 socket: 674 reason# EndOfStream
2025-11-28T14:52:37.136419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789283683432254:2049] [node 2] ICP30 unregister session Session# [5:7577789287978399647:2049] VirtualId# [5:7577789287978399637:0]
2025-11-28T14:52:37.136446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789287978399647:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.136525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:37.136563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:37.136813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:37.137001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789286804301269:2049] [node 2] ICS07 socket disconnect 676 reason# EndOfStream
2025-11-28T14:52:37.137046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789286804301269:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.137130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:37.137176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789286804301269:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:37.137844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789282509333878:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28047) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.137945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789286804301269:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.137986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282509333878:2049] [node 2] ICP30 unregister session Session# [6:7577789286804301269:2049] VirtualId# [6:7577789286804301261:0]
2025-11-28T14:52:37.138044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:37.138083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789282509333878:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.138101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.148338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789282509333878:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:37.198296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:37.198465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789289280260602:2050] [node 3] ICS07 socket disconnect 643 reason# EndOfStream
2025-11-28T14:52:37.198494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789289280260602:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.198562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:37.198594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789289280260602:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:37.199164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789284985293222:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26593) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.199097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:37.199242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789289280260602:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.199278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789284985293222:2050] [node 3] ICP30 unregister session Session# [4:7577789289280260602:2050] VirtualId# [4:7577789289280260598:0]
2025-11-28T14:52:37.199317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:37.199294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789287978399643:2050] [node 3] ICS01 socket: 672 reason# EndOfStream
2025-11-28T14:52:37.199343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789284985293222:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26593) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.199359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26593) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.199327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789283683432255:2050] [node 3] ICP30 unregister session Session# [5:7577789287978399643:2050] VirtualId# [5:7577789287978399633:0]
2025-11-28T14:52:37.199354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789287978399643:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.199436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:37.199474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:37.199830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:37.200062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789286804301258:2050] [node 3] ICS01 socket: 666 reason# EndOfStream
2025-11-28T14:52:37.200109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282509333879:2050] [node 3] ICP30 unregister session Session# [6:7577789286804301258:2050] VirtualId# [6:7577789286804301253:0]
2025-11-28T14:52:37.200151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789286804301258:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.200224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:37.200271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:37.236080Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:37.236262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789287978399648:2108] [node 4] ICS07 socket disconnect 678 reason# EndOfStream
2025-11-28T14:52:37.236291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789287978399648:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.236414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:37.236480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789287978399648:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:37.236964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:37.237260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789286804301271:2108] [node 4] ICS07 socket disconnect 628 reason# EndOfStream
2025-11-28T14:52:37.237034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789283683432316:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6390) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.237130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789287978399648:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.237311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789286804301271:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.237166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789283683432316:2108] [node 4] ICP30 unregister session Session# [5:7577789287978399648:2108] VirtualId# [5:7577789287978399639:0]
2025-11-28T14:52:37.237238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:37.237255Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789283683432316:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6390) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.237398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:37.237263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6390) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.237485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789286804301271:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:37.238174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789282509333941:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6390) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.238297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789286804301271:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.238344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282509333941:2108] [node 4] ICP30 unregister session Session# [6:7577789286804301271:2108] VirtualId# [6:7577789286804301262:0]
2025-11-28T14:52:37.238399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:37.238435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789282509333941:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6390) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.238457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6390) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.246618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789283683432316:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:37.248667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789282509333941:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:37.299888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:37.300139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789286804301276:2105] [node 5] ICS01 socket: 637 reason# EndOfStream
2025-11-28T14:52:37.300188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789282509333939:2105] [node 5] ICP30 unregister session Session# [6:7577789286804301276:2105] VirtualId# [6:7577789286804301274:0]
2025-11-28T14:52:37.300230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789286804301276:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.300317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:37.300379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:37.402305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789287766158146:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:37.402319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789287766158147:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:37.423909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789287616925455:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:37.423916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789287616925454:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:37.445251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789286422262722:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:37.445448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789286422262723:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:37.467234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789289981688542:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:37.467284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789289981688543:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:37.467342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789289981688544:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:37.489874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789287688380619:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:37.489926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789287688380621:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:37.489940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789287688380620:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:37.514360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789288941320956:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:37.514413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789288941320957:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:37.514519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789288941320958:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:37.598128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789287766158209:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:37.606637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789286422262787:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:37.605698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789287766158210:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:37.607086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789289981688605:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:37.607075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789286422262786:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:37.607804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789287688380682:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:37.607901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789288941321019:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:37.607937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789287616925535:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:37.607475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789289981688606:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:37.607341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789287766158211:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:37.607920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789286422262788:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:37.608248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789288941321018:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:37.608585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789287688380683:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:37.608089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789287616925536:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:37.609037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789287616925537:2118] [node 6] ICP01 ready to work
2025-11-28T14:52:37.612678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789287766158147:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(53074716ns) {Configure:104(53069476ns) {ProcessConfigured:135(236583ns) {StartInitialHandshake:157(234053ns) {DropHandshakes:470(2203ns) {DropIncomingHandshake:442(747ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(3072ns) PrepareNewSessionHandshake:477(754ns) {SwitchToState:179(320ns)}}}}}
2025-11-28T14:52:37.612795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789287766158146:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49969357ns) {Configure:104(49964977ns) {ProcessConfigured:135(67838ns) {StartInitialHandshake:157(65898ns) {DropHandshakes:470(1368ns) {DropIncomingHandshake:442(277ns) DropOutgoingHandshake:457(578ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:477(1073ns) {SwitchToState:179(376ns)}}}}}
2025-11-28T14:52:37.613833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789289981688543:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41552614ns) {Configure:104(34921010ns) {ProcessConfigured:135(443414ns) {StartInitialHandshake:157(440032ns) {DropHandshakes:470(963ns) {DropIncomingHandshake:442(282ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(696ns) PrepareNewSessionHandshake:477(1634ns) {SwitchToState:179(143ns)}}}}}
2025-11-28T14:52:37.613757Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789287688380620:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41911349ns) {Configure:104(41908452ns) {ProcessConfigured:135(351160ns) {StartInitialHandshake:157(274593ns) {DropHandshakes:470(1648ns) {DropIncomingHandshake:442(469ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(905ns) PrepareNewSessionHandshake:477(1263ns) {SwitchToState:179(522ns)}}}}}
2025-11-28T14:52:37.613907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789289981688542:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(41583287ns) {Configure:104(41578606ns) {ProcessConfigured:135(485108ns) {StartInitialHandshake:157(483646ns) {DropHandshakes:470(816ns) {DropIncomingHandshake:442(210ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:477(614ns) {SwitchToState:179(124ns)}}}}}
2025-11-28T14:52:37.613827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789287688380619:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43397421ns) {Configure:104(43392388ns) {ProcessConfigured:135(393351ns) {StartInitialHandshake:157(385329ns) {DropHandshakes:470(1253ns) {DropIncomingHandshake:442(336ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(3263ns) PrepareNewSessionHandshake:477(1166ns) {SwitchToState:179(639ns)}}}}}
2025-11-28T14:52:37.614216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789289981688544:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47405092ns) {Configure:104(47400854ns) {ProcessConfigured:135(528939ns) {StartInitialHandshake:157(449784ns) {DropHandshakes:470(1112ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(577ns) PrepareNewSessionHandshake:477(1123ns) {SwitchToState:179(491ns)}}}}}
2025-11-28T14:52:37.615805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789286422262722:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52080777ns) {Configure:104(52079056ns) {ProcessConfigured:135(691455ns) {StartInitialHandshake:157(689739ns) {DropHandshakes:470(141056ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(713ns) PrepareNewSessionHandshake:477(861ns) {SwitchToState:179(345ns)}}}}}
2025-11-28T14:52:37.615828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789287616925454:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52633666ns) {Configure:104(52625969ns) {ProcessConfigured:135(621906ns) {StartInitialHandshake:157(620878ns) {DropHandshakes:470(589ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(144ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:477(798ns) {SwitchToState:179(450ns)}}}}}
2025-11-28T14:52:37.616298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789287616925455:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36003057ns) {Configure:104(16605110ns) {ProcessConfigured:135(286405ns) {StartInitialHandshake:157(285123ns) {DropHandshakes:470(1102ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:477(1309ns) {SwitchToState:179(719ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.618921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287688380695:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52582
2025-11-28T14:52:37.618939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789287766158223:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60010
2025-11-28T14:52:37.618604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789288941320957:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(37047810ns) {Configure:104(37037250ns) {ProcessConfigured:135(255198ns) {StartInitialHandshake:157(254091ns) {DropHandshakes:470(841ns) {DropIncomingHandshake:442(305ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(594ns) {SwitchToState:179(118ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.619885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789286422262797:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64009
2025-11-28T14:52:37.621384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789287616925543:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-28T14:52:37.620594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287688380698:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52582
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.623869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789286422262798:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64009
2025-11-28T14:52:37.624761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287688380699:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52582
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.624754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289981688620:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63848
2025-11-28T14:52:37.624798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789286422262797:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.625948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287688380695:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.624888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789287616925543:2129] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.627147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789287766158226:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60010
2025-11-28T14:52:37.626082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789289981688620:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.628068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789287616925554:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-28T14:52:37.626071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789287688380619:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.625647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789286422262798:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.628169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289981688621:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63848
2025-11-28T14:52:37.628548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789287616925547:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:37.627767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287766158217:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.626443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789286422262722:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.630362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789287616925454:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.628196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287688380698:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.631549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789286422262793:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.627793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789287766158217:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.629216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287688380699:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.632350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789289981688612:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.629129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287766158218:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.631027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789287616925554:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.631575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789286422262793:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.629455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789287688380682:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.632403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789289981688612:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.632321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789286422262804:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64009
2025-11-28T14:52:37.629160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789287766158218:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.633930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789289981688621:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.631620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789287616925547:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.630364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789287688380621:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.635407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789286422262793:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:37.631473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287688380700:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52582
2025-11-28T14:52:37.630328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789287766158217:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.636313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287688380700:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.632064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789287616925548:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-28T14:52:37.635368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789289981688612:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:37.632334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789287766158227:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60010
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.632388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789287766158228:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60010
2025-11-28T14:52:37.637025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789287688380620:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.633418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789287616925555:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.637374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789286422262804:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.637584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789289981688605:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.637236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287688380704:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52582
2025-11-28T14:52:37.633514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789287766158218:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:37.637738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789286422262787:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.633931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789287616925548:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.637763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:37.637784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.633520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287766158216:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.637840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789286422262723:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.636602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789287616925555:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.637821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789289981688605:2108] [node 5] ICP22 created new session: [4:7577789289981688625:2108]
2025-11-28T14:52:37.638586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287688380705:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52582
2025-11-28T14:52:37.638233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289981688625:2108] [node 5] ICS09 handshake done sender: [4:7577789289981688612:2111] self: [4:7577789289981688607:0] peer: [5:7577789287688380702:0] socket: 637 qp: -1
2025-11-28T14:52:37.638039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789286422262807:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64009
2025-11-28T14:52:37.633551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789287766158216:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.638759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789287688380695:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.637212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789287616925541:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.633765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789287766158209:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.638639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289981688625:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:37.637241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789287616925541:2128] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.643233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321034:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.638077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789286422262787:2109] [node 5] ICP22 created new session: [3:7577789286422262809:2109]
2025-11-28T14:52:37.638873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:37.638771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287688380707:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52582
2025-11-28T14:52:37.640739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789287688380619:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.638808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789286422262809:2109] [node 5] ICS09 handshake done sender: [3:7577789286422262793:2111] self: [3:7577789286422262791:0] peer: [5:7577789287688380703:0] socket: 646 qp: -1
2025-11-28T14:52:37.640772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.633818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:37.638837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789286422262809:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:37.641391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789287688380619:2048] [node 1] ICP22 created new session: [5:7577789287688380708:2048]
::ffff:127.1.0.0
2025-11-28T14:52:37.639498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789287616925544:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.634662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789287766158209:2108] [node 5] ICP22 created new session: [1:7577789287766158231:2108]
2025-11-28T14:52:37.639514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:37.642820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287688380708:2048] [node 1] ICS09 handshake done sender: [5:7577789287688380695:2114] self: [5:7577789287688380701:0] peer: [1:7577789287766158213:0] socket: 630 qp: -1
2025-11-28T14:52:37.639534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789287616925544:2130] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.645788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789288941321034:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.639850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789286422262797:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.636306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287766158231:2108] [node 5] ICS09 handshake done sender: [1:7577789287766158217:2111] self: [1:7577789287766158213:0] peer: [5:7577789287688380701:0] socket: 625 qp: -1
2025-11-28T14:52:37.646474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289981688628:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63848
2025-11-28T14:52:37.643356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287688380708:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.643249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789286422262722:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.639582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789287616925541:2128] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.636323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287766158223:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.643400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.646135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789288941320958:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.643418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.643645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789287616925544:2130] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:37.636387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789287766158147:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.643405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789287688380682:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26873) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789287688380698:2116] held: no
2025-11-28T14:52:37.643465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789286422262722:2048] [node 1] ICP22 created new session: [3:7577789286422262814:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.647964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789289981688628:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.636596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.647326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321035:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.646099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789287616925561:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-28T14:52:37.636624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287766158231:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:37.648927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789289981688542:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.645761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789286422262814:2048] [node 1] ICS09 handshake done sender: [3:7577789286422262797:2115] self: [3:7577789286422262805:0] peer: [1:7577789287766158212:0] socket: 628 qp: -1
2025-11-28T14:52:37.645552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789287688380699:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:37.649086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789288941321035:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.636675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:37.646507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789286422262814:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.637029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789287766158147:2049] [node 3] ICP22 created new session: [1:7577789287766158232:2049]
2025-11-28T14:52:37.646762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789287616925535:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.646125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789287688380621:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2554) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789287688380699:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.637178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287766158232:2049] [node 3] ICS09 handshake done sender: [1:7577789287766158218:2112] self: [1:7577789287766158212:0] peer: [3:7577789286422262805:0] socket: 626 qp: -1
2025-11-28T14:52:37.646812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:37.647045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789287688380621:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.646818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.649373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789288941320956:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.646859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789287616925535:2108] [node 5] ICP22 created new session: [2:7577789287616925564:2108]
2025-11-28T14:52:37.637202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287766158232:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:37.647927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789286422262813:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64009
2025-11-28T14:52:37.647082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.637230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:37.647959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789286422262795:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.647061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789287616925455:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.637243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287766158227:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.637539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287766158226:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.647997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789286422262795:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.647316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789287688380698:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.649206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789286422262813:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.639184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287766158228:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.647343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789287616925564:2108] [node 5] ICS09 handshake done sender: [2:7577789287616925541:2128] self: [2:7577789287616925540:0] peer: [5:7577789287688380706:0] socket: 654 qp: -1
2025-11-28T14:52:37.649382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789286422262815:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64009
2025-11-28T14:52:37.647788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789287688380621:2050] [node 3] ICP22 created new session: [5:7577789287688380711:2050]
2025-11-28T14:52:37.639267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789287766158216:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.649814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789286422262795:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:37.652461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789286422262786:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.647959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287688380711:2050] [node 3] ICS09 handshake done sender: [5:7577789287688380699:2117] self: [5:7577789287688380703:0] peer: [3:7577789286422262791:0] socket: 650 qp: -1
2025-11-28T14:52:37.647360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.652496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.648733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287688380711:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:37.642850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789287766158146:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.653571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789286422262804:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.648798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789287688380682:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.643386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789287766158147:2049] [node 3] ICP18 handshake (actor [1:7577789287766158227:2117]) is too old
2025-11-28T14:52:37.653545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321038:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.647575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789287616925564:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.653585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789286422262786:2110] [node 6] ICP22 created new session: [3:7577789286422262819:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.648895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:37.643420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:37.649124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.647810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:37.643463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789287766158146:2048] [node 2] ICP22 created new session: [1:7577789287766158240:2048]
2025-11-28T14:52:37.654191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789286422262819:2110] [node 6] ICS09 handshake done sender: [3:7577789286422262795:2113] self: [3:7577789286422262789:0] peer: [6:7577789288941321036:0] socket: 678 qp: -1
2025-11-28T14:52:37.649166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789287688380682:2108] [node 4] ICP22 created new session: [5:7577789287688380712:2108]
2025-11-28T14:52:37.643670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287766158240:2048] [node 2] ICS09 handshake done sender: [1:7577789287766158216:2110] self: [1:7577789287766158214:0] peer: [2:7577789287616925556:0] socket: 624 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.647818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789287616925455:2049] [node 3] ICP22 created new session: [2:7577789287616925565:2049]
2025-11-28T14:52:37.655891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321039:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.654217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789286422262723:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789286422262804:2118] held: no
2025-11-28T14:52:37.655900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289981688630:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63848
2025-11-28T14:52:37.649175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287688380709:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52582
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.644533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287766158240:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:37.648716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789287616925565:2049] [node 3] ICS09 handshake done sender: [2:7577789287616925544:2130] self: [2:7577789287616925539:0] peer: [3:7577789286422262810:0] socket: 659 qp: -1
2025-11-28T14:52:37.654380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789286422262819:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:37.649216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287688380712:2108] [node 4] ICS09 handshake done sender: [5:7577789287688380698:2116] self: [5:7577789287688380702:0] peer: [4:7577789289981688607:0] socket: 638 qp: -1
2025-11-28T14:52:37.647054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.648741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789287616925565:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:37.657636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289981688631:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63848
2025-11-28T14:52:37.654420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:37.657011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789288941321034:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:37.649677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287766158219:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.649710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:37.658062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789288941320958:2050] [node 3] ICP19 incoming handshake succeeded
::ffff:127.1.0.02025-11-28T14:52:37.658010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289981688632:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63848
2025-11-28T14:52:37.655639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789286422262723:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.652147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789287616925543:2129] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.658093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.649367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287688380712:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:37.649700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789287766158219:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.655676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:37.658801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789289981688630:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.649598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.649851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789287766158146:2048] [node 2] ICP18 handshake (actor [1:7577789287766158228:2118]) is too old
2025-11-28T14:52:37.658588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789288941320958:2050] [node 3] ICP22 created new session: [6:7577789288941321042:2050]
2025-11-28T14:52:37.652639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789287616925454:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.655881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789286422262723:2049] [node 2] ICP22 created new session: [3:7577789286422262822:2049]
2025-11-28T14:52:37.650342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789287766158225:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.656233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789286422262822:2049] [node 2] ICS09 handshake done sender: [3:7577789286422262804:2118] self: [3:7577789286422262810:0] peer: [2:7577789287616925539:0] socket: 662 qp: -1
2025-11-28T14:52:37.658979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789289981688628:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.656566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789286422262822:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:37.658968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789288941321042:2050] [node 3] ICS09 handshake done sender: [6:7577789288941321034:2115] self: [6:7577789288941321036:0] peer: [3:7577789286422262789:0] socket: 679 qp: -1
2025-11-28T14:52:37.654385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789287688380620:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789287688380700:2118] held: no
2025-11-28T14:52:37.652819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.650369Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789287766158225:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.659062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789289981688543:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.655836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789287688380700:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.657037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.653577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789287616925454:2048] [node 1] ICP22 created new session: [2:7577789287616925570:2048]
2025-11-28T14:52:37.654341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789287766158219:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.656740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789287688380620:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.659001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789288941321042:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:37.659404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789289981688542:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789289981688628:2117] held: no
2025-11-28T14:52:37.654172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789287616925570:2048] [node 1] ICS09 handshake done sender: [2:7577789287616925543:2129] self: [2:7577789287616925556:0] peer: [1:7577789287766158214:0] socket: 627 qp: -1
2025-11-28T14:52:37.654338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789287616925570:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.659037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789288941321039:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.656777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:37.661070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789289981688542:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.655570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789287766158225:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:37.654365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.661586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.657015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789287688380620:2049] [node 2] ICP22 created new session: [5:7577789287688380716:2049]
2025-11-28T14:52:37.659127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.657710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287688380716:2049] [node 2] ICS09 handshake done sender: [5:7577789287688380700:2118] self: [5:7577789287688380706:0] peer: [2:7577789287616925540:0] socket: 656 qp: -1
2025-11-28T14:52:37.661848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789289981688542:2048] [node 1] ICP22 created new session: [4:7577789289981688634:2048]
2025-11-28T14:52:37.656720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789287766158210:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.662413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289981688634:2048] [node 1] ICS09 handshake done sender: [4:7577789289981688628:2117] self: [4:7577789289981688629:0] peer: [1:7577789287766158215:0] socket: 677 qp: -1
2025-11-28T14:52:37.659175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321040:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.657747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287688380716:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:37.662456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289981688634:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.656786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.660260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789287616925546:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.657932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.661030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321041:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.664165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789287688380718:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52582
2025-11-28T14:52:37.662845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789289981688631:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.661961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789288941320957:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.660297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789287616925546:2132] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.656999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789287766158210:2107] [node 4] ICP22 created new session: [1:7577789287766158244:2107]
2025-11-28T14:52:37.662256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789288941320956:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789288941321035:2116] held: no
2025-11-28T14:52:37.662866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.662001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789287616925546:2132] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.664167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789289981688544:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.657989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789287766158211:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.662413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789288941321035:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:37.662645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789287616925553:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.664520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789286422262803:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.664199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289981688636:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63848
2025-11-28T14:52:37.662440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789288941321041:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.662668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789287616925553:2135] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.658011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287766158244:2107] [node 4] ICS09 handshake done sender: [1:7577789287766158219:2113] self: [1:7577789287766158215:0] peer: [4:7577789289981688629:0] socket: 676 qp: -1
2025-11-28T14:52:37.664559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789286422262803:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.664152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789287616925536:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.658036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.664522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789288941320956:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.658321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789287766158211:2109] [node 6] ICP22 created new session: [1:7577789287766158245:2109]
2025-11-28T14:52:37.664263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.664565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:37.658582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287766158244:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:37.664516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789287616925536:2107] [node 4] ICP22 created new session: [2:7577789287616925574:2107]
2025-11-28T14:52:37.658629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.664712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789288941320956:2048] [node 1] ICP22 created new session: [6:7577789288941321047:2048]
2025-11-28T14:52:37.666981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789289981688543:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789289981688630:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.658769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789287766158245:2109] [node 6] ICS09 handshake done sender: [1:7577789287766158225:2115] self: [1:7577789287766158224:0] peer: [6:7577789288941321037:0] socket: 680 qp: -1
2025-11-28T14:52:37.664700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789287616925574:2107] [node 4] ICS09 handshake done sender: [2:7577789287616925546:2132] self: [2:7577789287616925542:0] peer: [4:7577789289981688633:0] socket: 642 qp: -1
2025-11-28T14:52:37.667349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789289981688630:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.667337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789286422262803:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.664874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789288941321047:2048] [node 1] ICS09 handshake done sender: [6:7577789288941321035:2116] self: [6:7577789288941321037:0] peer: [1:7577789287766158224:0] socket: 682 qp: -1
2025-11-28T14:52:37.664856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789287616925553:2135] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:37.658980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789287766158245:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:37.667362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287688380718:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.667442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789289981688543:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.665325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789288941321047:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:37.664863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789287616925574:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:37.659354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.668174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:37.665070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.666522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:37.666294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789287616925537:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.668509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789287688380697:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.668547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789289981688543:2049] [node 2] ICP22 created new session: [4:7577789289981688640:2049]
2025-11-28T14:52:37.666802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.668313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789286422262788:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.667142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789288941321019:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.668239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321046:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.668543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789287688380697:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:37.666936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789287616925537:2118] [node 6] ICP22 created new session: [2:7577789287616925576:2118]
2025-11-28T14:52:37.668713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289981688640:2049] [node 2] ICS09 handshake done sender: [4:7577789289981688630:2118] self: [4:7577789289981688633:0] peer: [2:7577789287616925542:0] socket: 652 qp: -1
2025-11-28T14:52:37.668719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.668283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789288941320957:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789288941321039:2118] held: no
2025-11-28T14:52:37.667002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789287616925576:2118] [node 6] ICS09 handshake done sender: [2:7577789287616925553:2135] self: [2:7577789287616925552:0] peer: [6:7577789288941321045:0] socket: 657 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.668856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289981688640:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:37.668856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789286422262788:2107] [node 4] ICP22 created new session: [3:7577789286422262825:2107]
2025-11-28T14:52:37.668563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321049:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.667144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789287616925576:2118] [node 6] ICS10 traffic start
2025-11-28T14:52:37.668817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789288941321046:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.667285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:37.668993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789286422262825:2107] [node 4] ICS09 handshake done sender: [3:7577789286422262803:2117] self: [3:7577789286422262802:0] peer: [4:7577789289981688638:0] socket: 655 qp: -1
2025-11-28T14:52:37.668889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.668961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789288941321039:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:37.669003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789288941321018:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:37.670393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789287688380697:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:37.669564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789289981688544:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2554) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789289981688631:2119] held: no
2025-11-28T14:52:37.669019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789286422262825:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:37.670078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789289981688639:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63848
2025-11-28T14:52:37.669814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789288941320957:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.670511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789287688380683:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.669430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.670546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.670114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:37.670379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789289981688619:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:37.671063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789287688380683:2109] [node 6] ICP22 created new session: [5:7577789287688380721:2109]
2025-11-28T14:52:37.670393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789289981688619:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:37.670136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789288941320957:2049] [node 2] ICP22 created new session: [6:7577789288941321052:2049]
2025-11-28T14:52:37.671152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789287688380721:2109] [node 6] ICS09 handshake done sender: [5:7577789287688380697:2115] self: [5:7577789287688380694:0] peer: [6:7577789288941321050:0] socket: 651 qp: -1
2025-11-28T14:52:37.671120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789289981688631:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:37.671358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789287688380721:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:37.670399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789288941321052:2049] [node 2] ICS09 handshake done sender: [6:7577789288941321039:2118] self: [6:7577789288941321045:0] peer: [2:7577789287616925552:0] socket: 686 qp: -1
2025-11-28T14:52:37.671493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789289981688544:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.670424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789288941321052:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:37.671392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:37.670498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:37.671533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:37.672279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789289981688619:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:37.671477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789288941321019:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26317) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789288941321041:2120] held: no
2025-11-28T14:52:37.672267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321053:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.672346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789289981688544:2050] [node 3] ICP22 created new session: [4:7577789289981688643:2050]
2025-11-28T14:52:37.672920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789288941321041:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:37.672446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289981688643:2050] [node 3] ICS09 handshake done sender: [4:7577789289981688631:2119] self: [4:7577789289981688638:0] peer: [3:7577789286422262802:0] socket: 663 qp: -1
2025-11-28T14:52:37.672455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789289981688606:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:37.673050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789288941321019:2107] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:37.672556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289981688643:2050] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:37.673238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:37.672772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:37.673792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789288941321018:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26873) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789288941321046:2121] held: no
2025-11-28T14:52:37.672805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:37.673816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789288941321019:2107] [node 5] ICP22 created new session: [6:7577789288941321056:2107]
2025-11-28T14:52:37.672919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789289981688606:2109] [node 6] ICP22 created new session: [4:7577789289981688644:2109]
2025-11-28T14:52:37.672948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789289981688644:2109] [node 6] ICS09 handshake done sender: [4:7577789289981688619:2114] self: [4:7577789289981688618:0] peer: [6:7577789288941321051:0] socket: 691 qp: -1
2025-11-28T14:52:37.673971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789288941321056:2107] [node 5] ICS09 handshake done sender: [6:7577789288941321041:2120] self: [6:7577789288941321050:0] peer: [5:7577789287688380694:0] socket: 661 qp: -1
2025-11-28T14:52:37.674081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789288941321055:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56920
2025-11-28T14:52:37.672971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789289981688644:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:37.674104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789288941321056:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:37.672999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:37.674173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:37.674316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789288941321046:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:37.674406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789288941321018:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:37.674532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:37.674571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789288941321018:2106] [node 4] ICP22 created new session: [6:7577789288941321058:2106]
2025-11-28T14:52:37.674605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789288941321058:2106] [node 4] ICS09 handshake done sender: [6:7577789288941321046:2121] self: [6:7577789288941321051:0] peer: [4:7577789289981688618:0] socket: 692 qp: -1
2025-11-28T14:52:37.674626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789288941321058:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:37.674651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:37.720415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:37.720587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789287616925570:2048] [node 1] ICS07 socket disconnect 627 reason# EndOfStream
2025-11-28T14:52:37.720624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789287616925570:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.720692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.720732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789287616925570:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:37.721609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:37.721818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789286422262814:2048] [node 1] ICS07 socket disconnect 628 reason# EndOfStream
2025-11-28T14:52:37.721865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789286422262814:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.721975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.722027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789286422262814:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:37.722721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789287616925454:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:37.722750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789287616925570:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:37.723304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789286422262722:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.724541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789286422262814:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.724588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789286422262722:2048] [node 1] ICP30 unregister session Session# [3:7577789286422262814:2048] VirtualId# [3:7577789286422262805:0]
2025-11-28T14:52:37.724641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.724667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789286422262722:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.724681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.724760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:37.724872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789287616925454:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.724917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789287616925570:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.724942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789287616925454:2048] [node 1] ICP30 unregister session Session# [2:7577789287616925570:2048] VirtualId# [2:7577789287616925556:0]
2025-11-28T14:52:37.725074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789289981688634:2048] [node 1] ICS01 socket: 677 reason# EndOfStream
2025-11-28T14:52:37.724989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.725139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789289981688542:2048] [node 1] ICP30 unregister session Session# [4:7577789289981688634:2048] VirtualId# [4:7577789289981688629:0]
2025-11-28T14:52:37.725008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789287616925454:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.725176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789289981688634:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.725022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.725254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.725301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.725990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:37.726128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789287688380708:2048] [node 1] ICS07 socket disconnect 630 reason# EndOfStream
2025-11-28T14:52:37.726153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789287688380708:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.726207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.726239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789287688380708:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:37.726745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789287688380619:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.726790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789287688380708:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.726847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789287688380619:2048] [node 1] ICP30 unregister session Session# [5:7577789287688380708:2048] VirtualId# [5:7577789287688380701:0]
2025-11-28T14:52:37.726889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.726912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789287688380619:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.726920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27370) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.726941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:37.727070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789288941321047:2048] [node 1] ICS01 socket: 682 reason# EndOfStream
2025-11-28T14:52:37.727102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789288941320956:2048] [node 1] ICP30 unregister session Session# [6:7577789288941321047:2048] VirtualId# [6:7577789288941321037:0]
2025-11-28T14:52:37.727120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789288941321047:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.727186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:37.727217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:37.734052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789286422262722:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:37.735032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789287616925454:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:37.737100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789287688380619:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:37.796171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:37.796435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789286422262822:2049] [node 2] ICS01 socket: 662 reason# EndOfStream
2025-11-28T14:52:37.796481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789286422262723:2049] [node 2] ICP30 unregister session Session# [3:7577789286422262822:2049] VirtualId# [3:7577789286422262810:0]
2025-11-28T14:52:37.796519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789286422262822:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.796675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:37.796745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:37.798574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:37.798732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789289981688640:2049] [node 2] ICS01 socket: 652 reason# EndOfStream
2025-11-28T14:52:37.798763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789289981688543:2049] [node 2] ICP30 unregister session Session# [4:7577789289981688640:2049] VirtualId# [4:7577789289981688633:0]
2025-11-28T14:52:37.798784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789289981688640:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.798860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:37.798915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:37.799297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:37.799434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789287688380716:2049] [node 2] ICS01 socket: 656 reason# EndOfStream
2025-11-28T14:52:37.799460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789287688380620:2049] [node 2] ICP30 unregister session Session# [5:7577789287688380716:2049] VirtualId# [5:7577789287688380706:0]
2025-11-28T14:52:37.799478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789287688380716:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.799532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:37.799569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:37.800002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:37.800128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789288941321052:2049] [node 2] ICS01 socket: 686 reason# EndOfStream
2025-11-28T14:52:37.800155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789288941320957:2049] [node 2] ICP30 unregister session Session# [6:7577789288941321052:2049] VirtualId# [6:7577789288941321045:0]
2025-11-28T14:52:37.800176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789288941321052:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.800226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:37.800269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:37.849412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:37.849565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789289981688643:2050] [node 3] ICS01 socket: 663 reason# EndOfStream
2025-11-28T14:52:37.849603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789289981688544:2050] [node 3] ICP30 unregister session Session# [4:7577789289981688643:2050] VirtualId# [4:7577789289981688638:0]
2025-11-28T14:52:37.849631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789289981688643:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.849715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:37.849761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:37.850515Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:37.850682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789287688380711:2050] [node 3] ICS01 socket: 650 reason# EndOfStream
2025-11-28T14:52:37.850712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789287688380621:2050] [node 3] ICP30 unregister session Session# [5:7577789287688380711:2050] VirtualId# [5:7577789287688380703:0]
2025-11-28T14:52:37.850733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789287688380711:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.850794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:37.850835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:37.850965Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:37.851214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789288941321042:2050] [node 3] ICS07 socket disconnect 679 reason# EndOfStream
2025-11-28T14:52:37.851254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789288941321042:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.851324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:37.851368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789288941321042:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:37.851956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789288941320958:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2554) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:37.851990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789288941321042:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:37.852015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789288941320958:2050] [node 3] ICP30 unregister session Session# [6:7577789288941321042:2050] VirtualId# [6:7577789288941321036:0]
2025-11-28T14:52:37.852065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:37.852110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789288941320958:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.852128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:37.862035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789288941320958:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:37.912684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:37.912922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789287688380712:2108] [node 4] ICS01 socket: 638 reason# EndOfStream
2025-11-28T14:52:37.912968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789287688380682:2108] [node 4] ICP30 unregister session Session# [5:7577789287688380712:2108] VirtualId# [5:7577789287688380702:0]
2025-11-28T14:52:37.913021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789287688380712:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.913107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:37.913155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:37.913384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:37.913516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789288941321058:2106] [node 4] ICS01 socket: 692 reason# EndOfStream
2025-11-28T14:52:37.913569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789288941321018:2106] [node 4] ICP30 unregister session Session# [6:7577789288941321058:2106] VirtualId# [6:7577789288941321051:0]
2025-11-28T14:52:37.913590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789288941321058:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.913648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:37.913684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:37.946879Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:37.947059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789288941321056:2107] [node 5] ICS01 socket: 661 reason# EndOfStream
2025-11-28T14:52:37.947093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789288941321019:2107] [node 5] ICP30 unregister session Session# [6:7577789288941321056:2107] VirtualId# [6:7577789288941321050:0]
2025-11-28T14:52:37.947124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789288941321056:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:37.947198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:37.947244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:38.052379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789290950827801:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:38.052431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789290950827802:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:38.071998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789293948472788:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:38.072153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789293948472789:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:38.094503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789291972845971:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:38.094687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789291972845972:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:38.118186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789294078362092:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:38.118211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789294078362093:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:38.118230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789294078362094:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:38.139580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789292169910879:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:38.139585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789292169910878:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:38.139601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789292169910877:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:38.166301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789294771274318:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:38.166329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789294771274320:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:38.166584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789294771274319:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:38.250757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789293948472851:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:38.254440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789294771274380:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:38.255926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789290950827864:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:38.257499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789293948472852:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:38.256795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789291972846034:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:38.257530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789290950827865:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:38.257372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789291972846035:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:38.257566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789290950827866:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:38.257563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789294078362155:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:38.257618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789292169910940:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:38.257773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789292169910941:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:38.257804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789293948472853:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:38.258279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789294078362156:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:38.258343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789291972846036:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:38.258801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789294771274382:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:38.263014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789293948472788:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52498856ns) {Configure:104(52496282ns) {ProcessConfigured:135(209491ns) {StartInitialHandshake:157(207556ns) {DropHandshakes:470(1347ns) {DropIncomingHandshake:442(454ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:477(770ns) {SwitchToState:179(275ns)}}}}}
2025-11-28T14:52:38.263227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789293948472789:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(38985550ns) {Configure:104(38981791ns) {ProcessConfigured:135(194455ns) {StartInitialHandshake:157(192588ns) {DropHandshakes:470(1037ns) {DropIncomingHandshake:442(291ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:477(921ns) {SwitchToState:179(211ns)}}}}}
2025-11-28T14:52:38.264701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789291972845972:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40214267ns) {Configure:104(40210266ns) {ProcessConfigured:135(428359ns) {StartInitialHandshake:157(427112ns) {DropHandshakes:470(7797ns) {DropIncomingHandshake:442(682ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(753ns) PrepareNewSessionHandshake:477(952ns) {SwitchToState:179(507ns)}}}}}
2025-11-28T14:52:38.264718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789291972845971:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40330442ns) {Configure:104(40324192ns) {ProcessConfigured:135(423581ns) {StartInitialHandshake:157(421975ns) {DropHandshakes:470(821ns) {DropIncomingHandshake:442(279ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:477(854ns) {SwitchToState:179(33ns)}}}}}
2025-11-28T14:52:38.266708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789290950827802:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(57999601ns) {Configure:104(55858504ns) {ProcessConfigured:135(783102ns) {StartInitialHandshake:157(780828ns) {DropHandshakes:470(1127ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(4028ns) PrepareNewSessionHandshake:477(1677ns) {SwitchToState:179(762ns)}}}}}
2025-11-28T14:52:38.266368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789292169910879:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45395489ns) {Configure:104(45391232ns) {ProcessConfigured:135(1175542ns) {StartInitialHandshake:157(1173451ns) {DropHandshakes:470(271167ns) {DropIncomingHandshake:442(1409ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(1230ns) PrepareNewSessionHandshake:477(1430ns) {SwitchToState:179(515ns)}}}}}
2025-11-28T14:52:38.266614Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789294078362092:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21888551ns) {Configure:104(21880970ns) {ProcessConfigured:135(773476ns) {StartInitialHandshake:157(771734ns) {DropHandshakes:470(998ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:477(626ns) {SwitchToState:179(117ns)}}}}}
2025-11-28T14:52:38.267059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789290950827801:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55718108ns) {Configure:104(55711374ns) {ProcessConfigured:135(587093ns) {StartInitialHandshake:157(585592ns) {DropHandshakes:470(1056ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:477(640ns) {SwitchToState:179(138ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.266448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789292169910877:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33407811ns) {Configure:104(33400381ns) {ProcessConfigured:135(1003245ns) {StartInitialHandshake:157(1001228ns) {DropHandshakes:470(1031ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(213ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:477(693ns) {SwitchToState:179(346ns)}}}}}
2025-11-28T14:52:38.268756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789291972846047:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60974
2025-11-28T14:52:38.268508Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789292169910878:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46301042ns) {Configure:104(46296665ns) {ProcessConfigured:135(798975ns) {StartInitialHandshake:157(797356ns) {DropHandshakes:470(954ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(538ns) PrepareNewSessionHandshake:477(896ns) {SwitchToState:179(489ns)}}}}}
2025-11-28T14:52:38.269586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789291972846047:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.269404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789294771274318:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35703845ns) {Configure:104(35691329ns) {ProcessConfigured:135(1117294ns) {StartInitialHandshake:157(1116008ns) {DropHandshakes:470(1206ns) {DropIncomingHandshake:442(236ns) DropOutgoingHandshake:457(159ns)} GenerateSessionVirtualId:988(341ns) PrepareNewSessionHandshake:477(922ns) {SwitchToState:179(125ns)}}}}}
2025-11-28T14:52:38.269796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789291972845972:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.270055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789290950827876:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.270775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789293948472859:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.270798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827876:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.270818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789293948472859:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.271000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789294078362166:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58118
2025-11-28T14:52:38.273636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789294078362170:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58118
No RDMA context found for GID: 2025-11-28T14:52:38.273476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789293948472867:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54349
::ffff:127.1.0.0
2025-11-28T14:52:38.273682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789290950827882:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-28T14:52:38.274028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789294078362166:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.274277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789293948472859:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.274537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274396:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.274772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789294078362171:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58118
2025-11-28T14:52:38.275034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789292169910955:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12897
2025-11-28T14:52:38.278376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789291972846053:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60974
2025-11-28T14:52:38.276825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274397:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
2025-11-28T14:52:38.276824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789293948472868:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54349
2025-11-28T14:52:38.277185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789290950827881:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-28T14:52:38.277197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789294078362093:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.276792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789292169910956:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12897
2025-11-28T14:52:38.277250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789290950827883:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-28T14:52:38.278665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789291972846052:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60974
2025-11-28T14:52:38.277235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789293948472869:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54349
2025-11-28T14:52:38.278969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789294078362173:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58118
2025-11-28T14:52:38.279583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789292169910956:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.278372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789293948472789:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.279735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789294771274396:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.279603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789294078362171:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.280033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789292169910877:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.279775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789291972846053:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.279048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827883:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.278426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:38.280311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789294771274397:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.280589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789294078362173:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.280043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789291972846052:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.280776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789292169910955:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.279005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789293948472789:2049] [node 3] ICP22 created new session: [2:7577789293948472871:2049]
2025-11-28T14:52:38.280738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827881:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.281265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789294771274380:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.280596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789294078362170:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.281544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789292169910948:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.279211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789293948472861:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.281956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789294771274382:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.281326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827882:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.281375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789294078362092:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.282919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789291972846054:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60974
2025-11-28T14:52:38.281563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789292169910948:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.281572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827872:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.279229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789293948472861:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.281719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789294078362094:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.281578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827871:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.284320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789292169910948:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:38.279358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789293948472871:2049] [node 3] ICS09 handshake done sender: [2:7577789293948472859:2110] self: [2:7577789293948472856:0] peer: [3:7577789291972846051:0] socket: 630 qp: -1
2025-11-28T14:52:38.281589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789290950827872:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.283410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789291972846044:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.284286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274399:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
2025-11-28T14:52:38.281596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789290950827871:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.283405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789294078362165:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.279569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789293948472869:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.282950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827875:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.283427Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789294078362165:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.283424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789291972846044:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.285837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789294771274399:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.279587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789293948472871:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:38.282979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789290950827875:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.286338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789292169910940:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.285836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789294078362165:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:38.284333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789290950827872:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:38.285818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789291972846044:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:38.286711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:38.279802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789293948472868:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.286051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789294771274319:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.286745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789292169910940:2109] [node 6] ICP22 created new session: [5:7577789292169910960:2109]
2025-11-28T14:52:38.286402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789294078362178:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58118
2025-11-28T14:52:38.286105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789290950827801:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.286224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789291972846047:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:38.280261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:38.288707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789294078362156:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.287037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789292169910960:2109] [node 6] ICS09 handshake done sender: [5:7577789292169910948:2111] self: [5:7577789292169910945:0] peer: [6:7577789294771274398:0] socket: 657 qp: -1
2025-11-28T14:52:38.286278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789290950827871:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:38.286846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789291972845972:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.280407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789293948472788:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.290120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789294771274380:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24882) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789294771274396:2115] held: no
2025-11-28T14:52:38.290172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.286397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789290950827875:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:38.287611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789292169910960:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:38.281976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789293948472861:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:38.287140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:38.290501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789294078362156:2109] [node 6] ICP22 created new session: [4:7577789294078362182:2109]
2025-11-28T14:52:38.290148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274401:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
2025-11-28T14:52:38.288628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.286427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:38.288175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789291972846036:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.282001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789293948472867:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.288215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:38.294265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274403:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
2025-11-28T14:52:38.291006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789294078362166:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:38.288262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789291972845972:2049] [node 2] ICP22 created new session: [3:7577789291972846058:2049]
2025-11-28T14:52:38.286784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789290950827801:2048] [node 2] ICP22 created new session: [1:7577789290950827891:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.284772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789293948472851:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.296795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789294771274397:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:38.295890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789292169910877:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789292169910956:2116] held: no
2025-11-28T14:52:38.288668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789291972846036:2107] [node 4] ICP22 created new session: [3:7577789291972846060:2107]
2025-11-28T14:52:38.288201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789290950827891:2048] [node 2] ICS09 handshake done sender: [1:7577789290950827872:2111] self: [1:7577789290950827870:0] peer: [2:7577789293948472875:0] socket: 652 qp: -1
2025-11-28T14:52:38.291831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789294078362182:2109] [node 6] ICS09 handshake done sender: [4:7577789294078362165:2113] self: [4:7577789294078362160:0] peer: [6:7577789294771274400:0] socket: 661 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.297409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789294771274396:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:38.296727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789292169910963:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12897
2025-11-28T14:52:38.284805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.292143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789294078362182:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:38.290926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789291972846060:2107] [node 4] ICS09 handshake done sender: [3:7577789291972846044:2112] self: [3:7577789291972846042:0] peer: [4:7577789294078362177:0] socket: 640 qp: -1
2025-11-28T14:52:38.288657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789290950827891:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:38.292175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:38.299434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789292169910961:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12897
2025-11-28T14:52:38.285484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789293948472851:2107] [node 4] ICP22 created new session: [2:7577789293948472877:2107]
2025-11-28T14:52:38.298510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789294771274380:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.290080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789290950827864:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.290970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789291972846058:2049] [node 2] ICS09 handshake done sender: [3:7577789291972846047:2115] self: [3:7577789291972846051:0] peer: [2:7577789293948472856:0] socket: 633 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.285519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789293948472877:2107] [node 4] ICS09 handshake done sender: [2:7577789293948472861:2112] self: [2:7577789293948472855:0] peer: [4:7577789294078362174:0] socket: 631 qp: -1
2025-11-28T14:52:38.300251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789292169910963:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.299367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789294771274382:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.294303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789294078362093:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.291010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789291972846058:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:38.300472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789292169910965:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12897
2025-11-28T14:52:38.290093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789290950827865:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.300565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789292169910956:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:38.299385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:38.292122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789291972846060:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:38.285546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789293948472877:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:38.299743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:38.300581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789292169910878:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.294343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:38.290135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:38.292159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: 2025-11-28T14:52:38.299788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789294771274382:2108] [node 4] ICP22 created new session: [6:7577789294771274408:2108]
::ffff:127.1.0.02025-11-28T14:52:38.302378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789292169910877:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.293154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: 2025-11-28T14:52:38.290478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789290950827865:2107] [node 4] ICP22 created new session: [1:7577789290950827893:2107]
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.285588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:38.300429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789294771274380:2105] [node 5] ICP22 created new session: [6:7577789294771274407:2105]
2025-11-28T14:52:38.294705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789294078362093:2049] [node 2] ICP22 created new session: [4:7577789294078362185:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.302553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:38.300212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789291972846061:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60974
2025-11-28T14:52:38.290513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789290950827893:2107] [node 4] ICS09 handshake done sender: [1:7577789290950827875:2113] self: [1:7577789290950827868:0] peer: [4:7577789294078362176:0] socket: 654 qp: -1
2025-11-28T14:52:38.286694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789293948472862:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.302333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789294771274408:2108] [node 4] ICS09 handshake done sender: [6:7577789294771274397:2116] self: [6:7577789294771274400:0] peer: [4:7577789294078362160:0] socket: 670 qp: -1
2025-11-28T14:52:38.295162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789294078362094:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12014) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789294078362170:2116] held: no
2025-11-28T14:52:38.302501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789294771274407:2105] [node 5] ICS09 handshake done sender: [6:7577789294771274396:2115] self: [6:7577789294771274398:0] peer: [5:7577789292169910945:0] socket: 666 qp: -1
2025-11-28T14:52:38.303566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789291972846061:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.303501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789292169910965:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.302539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789294771274408:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:38.286718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789293948472862:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.290920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:38.304061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789291972845971:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.302996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789294771274407:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:38.303778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789292169910877:2048] [node 1] ICP22 created new session: [5:7577789292169910968:2048]
2025-11-28T14:52:38.305439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789291972846065:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60974
2025-11-28T14:52:38.302998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:38.291813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789290950827893:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:38.295909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789294078362185:2049] [node 2] ICS09 handshake done sender: [4:7577789294078362166:2114] self: [4:7577789294078362174:0] peer: [2:7577789293948472855:0] socket: 635 qp: -1
2025-11-28T14:52:38.304063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789292169910879:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.288631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789293948472879:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54349
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.292105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:38.304157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789292169910968:2048] [node 1] ICS09 handshake done sender: [5:7577789292169910956:2116] self: [5:7577789292169910957:0] peer: [1:7577789290950827869:0] socket: 668 qp: -1
2025-11-28T14:52:38.290999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789293948472788:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789293948472868:2116] held: no
2025-11-28T14:52:38.304359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789292169910968:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:38.303576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274404:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
2025-11-28T14:52:38.305508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:38.297385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789294078362185:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:38.291832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789293948472789:2049] [node 3] ICP18 handshake (actor [2:7577789293948472867:2115]) is too old
2025-11-28T14:52:38.303845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:38.305907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789292169910969:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12897
2025-11-28T14:52:38.292160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789290950827864:2108] [node 5] ICP22 created new session: [1:7577789290950827895:2108]
2025-11-28T14:52:38.293520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789293948472868:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:38.308752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789292169910969:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.297416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:38.305933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789294771274319:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19924) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789294771274399:2117] held: no
2025-11-28T14:52:38.307041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789291972846065:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.292326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:38.294771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789293948472862:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:38.298107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789294078362180:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58118
2025-11-28T14:52:38.308468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274406:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
2025-11-28T14:52:38.307237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789291972846045:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.294244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789290950827895:2108] [node 5] ICS09 handshake done sender: [1:7577789290950827871:2110] self: [1:7577789290950827869:0] peer: [5:7577789292169910957:0] socket: 653 qp: -1
2025-11-28T14:52:38.307265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789291972846045:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.308531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274409:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
2025-11-28T14:52:38.300217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789294078362092:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789294078362173:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.310079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789292169910941:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.295199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789293948472788:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.294455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789290950827895:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:38.301637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789294078362181:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58118
2025-11-28T14:52:38.298096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:38.310229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789291972846045:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:38.297299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:38.309873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789294771274399:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:38.310991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789292169910878:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19924) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789292169910963:2118] held: no
2025-11-28T14:52:38.299080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789293948472788:2048] [node 1] ICP22 created new session: [2:7577789293948472883:2048]
2025-11-28T14:52:38.304558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789290950827899:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51055
2025-11-28T14:52:38.311107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789292169910972:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12897
2025-11-28T14:52:38.301754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789294078362170:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:38.312223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789291972846035:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.299378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789293948472852:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.305685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827874:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.311114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789292169910974:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12897
2025-11-28T14:52:38.310185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789294771274409:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.310400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789294771274318:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:38.299442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789293948472883:2048] [node 1] ICS09 handshake done sender: [2:7577789293948472868:2116] self: [2:7577789293948472875:0] peer: [1:7577789290950827870:0] socket: 660 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.310497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789294771274319:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.313852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789292169910965:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:38.303572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789294078362094:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.305707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789290950827874:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.300451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:38.314492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:38.310730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789294771274406:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.303633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789294078362173:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:38.310335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789290950827874:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:38.310431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827899:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.314550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789291972846035:2108] [node 5] ICP22 created new session: [3:7577789291972846070:2108]
2025-11-28T14:52:38.300478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789293948472883:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:38.315395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789292169910879:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12014) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789292169910965:2119] held: no
2025-11-28T14:52:38.311995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:38.310580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789290950827802:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.304398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:38.314737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789291972846069:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60974
2025-11-28T14:52:38.315991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789292169910963:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:38.312193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789294771274320:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:38.300520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:38.314747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789291972846046:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.304598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789294078362094:2050] [node 3] ICP22 created new session: [4:7577789294078362188:2050]
2025-11-28T14:52:38.310817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789290950827877:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.316798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789292169910879:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.313739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789294771274319:2049] [node 2] ICP22 created new session: [6:7577789294771274413:2049]
2025-11-28T14:52:38.310846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789290950827877:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.314764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789291972846046:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.316810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789292169910878:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.313777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789294771274413:2049] [node 2] ICS09 handshake done sender: [6:7577789294771274399:2117] self: [6:7577789294771274402:0] peer: [2:7577789293948472858:0] socket: 675 qp: -1
2025-11-28T14:52:38.312076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:38.301655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789293948472852:2109] [node 6] ICP22 created new session: [2:7577789293948472885:2109]
2025-11-28T14:52:38.313814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789294771274413:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:38.317422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789292169910941:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1763) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789292169910969:2120] held: no
2025-11-28T14:52:38.312238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789290950827802:2049] [node 3] ICP22 created new session: [1:7577789290950827901:2049]
2025-11-28T14:52:38.305774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789294078362092:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.314809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789291972845971:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789291972846061:2119] held: no
2025-11-28T14:52:38.317843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: 2025-11-28T14:52:38.313941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:38.302339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789293948472885:2109] [node 6] ICS09 handshake done sender: [2:7577789293948472862:2113] self: [2:7577789293948472858:0] peer: [6:7577789294771274402:0] socket: 674 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:38.316067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789291972846070:2108] [node 5] ICS09 handshake done sender: [3:7577789291972846045:2113] self: [3:7577789291972846039:0] peer: [5:7577789292169910970:0] socket: 662 qp: -1
2025-11-28T14:52:38.316475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789291972846061:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:38.313806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789290950827901:2049] [node 3] ICS09 handshake done sender: [1:7577789290950827874:2112] self: [1:7577789290950827867:0] peer: [3:7577789291972846066:0] socket: 693 qp: -1
2025-11-28T14:52:38.317940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:38.317374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789294771274318:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789294771274409:2122] held: no
2025-11-28T14:52:38.314055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789290950827901:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:38.318578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274415:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
2025-11-28T14:52:38.317969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789292169910879:2050] [node 3] ICP22 created new session: [5:7577789292169910977:2050]
2025-11-28T14:52:38.314090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:38.302541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789293948472885:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:38.314108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789290950827877:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:38.317977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789292169910878:2049] [node 2] ICP22 created new session: [5:7577789292169910976:2049]
::ffff:127.1.0.02025-11-28T14:52:38.306732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789294078362188:2050] [node 3] ICS09 handshake done sender: [4:7577789294078362170:2116] self: [4:7577789294078362177:0] peer: [3:7577789291972846042:0] socket: 642 qp: -1
2025-11-28T14:52:38.314268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789290950827866:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.318598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789292169910976:2049] [node 2] ICS09 handshake done sender: [5:7577789292169910963:2118] self: [5:7577789292169910967:0] peer: [2:7577789293948472857:0] socket: 691 qp: -1
2025-11-28T14:52:38.304017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:38.314776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:38.307021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789294078362188:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:38.316486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789291972846070:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:38.318618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789292169910977:2050] [node 3] ICS09 handshake done sender: [5:7577789292169910965:2119] self: [5:7577789292169910970:0] peer: [3:7577789291972846039:0] socket: 694 qp: -1
2025-11-28T14:52:38.316085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789290950827866:2109] [node 6] ICP22 created new session: [1:7577789290950827905:2109]
2025-11-28T14:52:38.319240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789292169910977:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:38.304505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789293948472863:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.307055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:38.316518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:38.319554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789294771274417:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10067
2025-11-28T14:52:38.316243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789290950827905:2109] [node 6] ICS09 handshake done sender: [1:7577789290950827877:2115] self: [1:7577789290950827873:0] peer: [6:7577789294771274412:0] socket: 656 qp: -1
2025-11-28T14:52:38.319568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789294771274409:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:38.307512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:38.319315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789292169910976:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:38.304523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789293948472863:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.316818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789291972846046:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:38.304548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789293948472881:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54349
2025-11-28T14:52:38.319355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789292169910975:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12897
2025-11-28T14:52:38.320179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789294771274318:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.317481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789291972845971:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.316808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789290950827905:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:38.307560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789294078362092:2048] [node 1] ICP22 created new session: [4:7577789294078362190:2048]
2025-11-28T14:52:38.319359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:38.308754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789294078362190:2048] [node 1] ICS09 handshake done sender: [4:7577789294078362173:2118] self: [4:7577789294078362176:0] peer: [1:7577789290950827868:0] socket: 659 qp: -1
2025-11-28T14:52:38.320306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:38.319396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:38.305885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789293948472884:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54349
2025-11-28T14:52:38.316863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:38.317894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:38.309940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789294078362190:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:38.320463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789294771274318:2048] [node 1] ICP22 created new session: [6:7577789294771274418:2048]
2025-11-28T14:52:38.319788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789292169910969:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:38.318558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789291972845971:2048] [node 1] ICP22 created new session: [3:7577789291972846073:2048]
2025-11-28T14:52:38.310373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:38.307308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789293948472863:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:38.320204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789292169910941:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.310928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789294078362167:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.320768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789294771274406:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:38.320415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:38.310954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789294078362167:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:38.307316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789293948472881:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:38.318596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789291972846073:2048] [node 1] ICS09 handshake done sender: [3:7577789291972846061:2119] self: [3:7577789291972846066:0] peer: [1:7577789290950827867:0] socket: 637 qp: -1
2025-11-28T14:52:38.320608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789292169910941:2108] [node 4] ICP22 created new session: [5:7577789292169910980:2108]
2025-11-28T14:52:38.320932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789294771274320:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12014) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789294771274406:2121] held: no
2025-11-28T14:52:38.314323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789294078362167:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:38.314619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789294078362155:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.320842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789292169910980:2108] [node 4] ICS09 handshake done sender: [5:7577789292169910969:2120] self: [5:7577789292169910973:0] peer: [4:7577789294078362162:0] socket: 697 qp: -1
2025-11-28T14:52:38.314661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:38.320881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789292169910980:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:38.318622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789291972846034:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.309873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789293948472853:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:38.320935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789294771274418:2048] [node 1] ICS09 handshake done sender: [6:7577789294771274409:2122] self: [6:7577789294771274412:0] peer: [1:7577789290950827873:0] socket: 650 qp: -1
2025-11-28T14:52:38.314811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789294078362155:2108] [node 5] ICP22 created new session: [4:7577789294078362193:2108]
2025-11-28T14:52:38.320910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:38.315971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789294078362193:2108] [node 5] ICS09 handshake done sender: [4:7577789294078362167:2115] self: [4:7577789294078362162:0] peer: [5:7577789292169910973:0] socket: 665 qp: -1
2025-11-28T14:52:38.309923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:38.320958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789294771274418:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:38.318673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:38.316218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789294078362193:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:38.321110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:38.310265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789293948472853:2108] [node 5] ICP22 created new session: [2:7577789293948472890:2108]
2025-11-28T14:52:38.317416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:38.321439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789294771274320:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:38.310950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789293948472890:2108] [node 5] ICS09 handshake done sender: [2:7577789293948472863:2114] self: [2:7577789293948472857:0] peer: [5:7577789292169910967:0] socket: 688 qp: -1
2025-11-28T14:52:38.319215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789291972846073:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:38.321651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:38.311019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789293948472890:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:38.321687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789294771274320:2050] [node 3] ICP22 created new session: [6:7577789294771274420:2050]
2025-11-28T14:52:38.319252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789291972846034:2109] [node 6] ICP22 created new session: [3:7577789291972846074:2109]
2025-11-28T14:52:38.312163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:38.321731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789294771274420:2050] [node 3] ICS09 handshake done sender: [6:7577789294771274406:2121] self: [6:7577789294771274414:0] peer: [3:7577789291972846040:0] socket: 663 qp: -1
2025-11-28T14:52:38.319369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:38.321759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789294771274420:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:38.321791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:38.319410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789291972846074:2109] [node 6] ICS09 handshake done sender: [3:7577789291972846046:2114] self: [3:7577789291972846040:0] peer: [6:7577789294771274414:0] socket: 647 qp: -1
2025-11-28T14:52:38.319445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789291972846074:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:38.319612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:38.374072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:38.374248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789293948472883:2048] [node 1] ICS01 socket: 660 reason# EndOfStream
2025-11-28T14:52:38.374287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789293948472788:2048] [node 1] ICP30 unregister session Session# [2:7577789293948472883:2048] VirtualId# [2:7577789293948472875:0]
2025-11-28T14:52:38.374313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789293948472883:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.374398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:38.374446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:38.375075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:38.375304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789291972846073:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-11-28T14:52:38.375371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789291972845971:2048] [node 1] ICP30 unregister session Session# [3:7577789291972846073:2048] VirtualId# [3:7577789291972846066:0]
2025-11-28T14:52:38.375409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789291972846073:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.375483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:38.375542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:38.377634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:38.377784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789294078362190:2048] [node 1] ICS01 socket: 659 reason# EndOfStream
2025-11-28T14:52:38.377825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789294078362092:2048] [node 1] ICP30 unregister session Session# [4:7577789294078362190:2048] VirtualId# [4:7577789294078362176:0]
2025-11-28T14:52:38.377852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789294078362190:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.377932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:38.377981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:38.378269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:38.378396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789292169910968:2048] [node 1] ICS01 socket: 668 reason# EndOfStream
2025-11-28T14:52:38.378424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789292169910877:2048] [node 1] ICP30 unregister session Session# [5:7577789292169910968:2048] VirtualId# [5:7577789292169910957:0]
2025-11-28T14:52:38.378440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789292169910968:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.378496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:38.378532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:38.378870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:38.378995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789294771274418:2048] [node 1] ICS01 socket: 650 reason# EndOfStream
2025-11-28T14:52:38.379019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789294771274318:2048] [node 1] ICP30 unregister session Session# [6:7577789294771274418:2048] VirtualId# [6:7577789294771274412:0]
2025-11-28T14:52:38.379036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789294771274418:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.379081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:38.379116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:38.449434Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:38.449668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789291972846058:2049] [node 2] ICS07 socket disconnect 633 reason# EndOfStream
2025-11-28T14:52:38.449713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789291972846058:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.449802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:38.449874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789291972846058:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:38.450507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789291972845972:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19924) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:38.450555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789291972846058:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:38.450607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789291972845972:2049] [node 2] ICP30 unregister session Session# [3:7577789291972846058:2049] VirtualId# [3:7577789291972846051:0]
2025-11-28T14:52:38.450644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:38.450702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789291972845972:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:38.450717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:38.451545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:38.451702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789294078362185:2049] [node 2] ICS07 socket disconnect 635 reason# EndOfStream
2025-11-28T14:52:38.451730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789294078362185:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.451801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:38.451850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789294078362185:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:38.452222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:38.452337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789292169910976:2049] [node 2] ICS01 socket: 691 reason# EndOfStream
2025-11-28T14:52:38.452391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789292169910878:2049] [node 2] ICP30 unregister session Session# [5:7577789292169910976:2049] VirtualId# [5:7577789292169910967:0]
2025-11-28T14:52:38.452419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789294078362093:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19924) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:38.452411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789292169910976:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.452472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:38.452508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789294078362185:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:38.452504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:38.452565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789294078362093:2049] [node 2] ICP30 unregister session Session# [4:7577789294078362185:2049] VirtualId# [4:7577789294078362174:0]
2025-11-28T14:52:38.452643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:38.452682Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789294078362093:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:38.452703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:38.452960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:38.453096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789294771274413:2049] [node 2] ICS01 socket: 675 reason# EndOfStream
2025-11-28T14:52:38.453123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789294771274319:2049] [node 2] ICP30 unregister session Session# [6:7577789294771274413:2049] VirtualId# [6:7577789294771274402:0]
2025-11-28T14:52:38.453140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789294771274413:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.453190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:38.453220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:38.460028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789291972845972:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:38.462076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789294078362093:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:38.531747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:38.531920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789294078362188:2050] [node 3] ICS01 socket: 642 reason# EndOfStream
2025-11-28T14:52:38.531955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789294078362094:2050] [node 3] ICP30 unregister session Session# [4:7577789294078362188:2050] VirtualId# [4:7577789294078362177:0]
2025-11-28T14:52:38.531977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789294078362188:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.532038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:38.532077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:38.532715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:38.532837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789292169910977:2050] [node 3] ICS01 socket: 694 reason# EndOfStream
2025-11-28T14:52:38.532869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789292169910879:2050] [node 3] ICP30 unregister session Session# [5:7577789292169910977:2050] VirtualId# [5:7577789292169910970:0]
2025-11-28T14:52:38.532886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789292169910977:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.532933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:38.532963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:38.533572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:38.533676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789294771274420:2050] [node 3] ICS01 socket: 663 reason# EndOfStream
2025-11-28T14:52:38.533707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789294771274320:2050] [node 3] ICP30 unregister session Session# [6:7577789294771274420:2050] VirtualId# [6:7577789294771274414:0]
2025-11-28T14:52:38.533724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789294771274420:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.533760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:38.533788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:38.572722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:38.572875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789292169910980:2108] [node 4] ICS01 socket: 697 reason# EndOfStream
2025-11-28T14:52:38.572903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789292169910941:2108] [node 4] ICP30 unregister session Session# [5:7577789292169910980:2108] VirtualId# [5:7577789292169910973:0]
2025-11-28T14:52:38.572930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789292169910980:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.572992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:38.573022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:38.573728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:38.573885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789294771274408:2108] [node 4] ICS07 socket disconnect 670 reason# EndOfStream
2025-11-28T14:52:38.573910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789294771274408:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.573960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:38.574000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789294771274408:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:38.574736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789294771274382:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1763) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:38.574796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789294771274408:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:38.574841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789294771274382:2108] [node 4] ICP30 unregister session Session# [6:7577789294771274408:2108] VirtualId# [6:7577789294771274400:0]
2025-11-28T14:52:38.574880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:38.574914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789294771274382:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1763) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:38.574929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1763) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:38.585001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789294771274382:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:38.650363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:38.650512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789294771274407:2105] [node 5] ICS01 socket: 666 reason# EndOfStream
2025-11-28T14:52:38.650542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789294771274380:2105] [node 5] ICP30 unregister session Session# [6:7577789294771274407:2105] VirtualId# [6:7577789294771274398:0]
2025-11-28T14:52:38.650564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789294771274407:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:38.650636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:38.650707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-28T14:52:38.918531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789294536796434:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:38.918573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789294536796435:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:38.937845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789294721998097:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:38.937886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789294721998098:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:38.956466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789293806413749:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:38.956516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789293806413750:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:38.973093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789291547837415:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:38.973141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789291547837416:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:38.973165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789291547837417:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:38.990602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789292199693887:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:38.990640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789292199693888:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:38.990665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789292199693889:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:39.007263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789296783976671:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:39.007305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789296783976672:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:39.007330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789296783976673:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:39.090309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789296783976734:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:39.090290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789298101381108:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:39.090282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789299016965456:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:39.090692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789298831763793:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:39.091152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789295842804774:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:39.091204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789296494661246:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:39.091460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789298101381109:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:39.091430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789299016965457:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:39.091553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789295842804775:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:39.091777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789298831763794:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:39.092101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789296494661247:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:39.092118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789296783976735:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:39.092158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789299016965458:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:39.092192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789298101381110:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:39.092804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789298831763795:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.102568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789299016965472:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49158
2025-11-28T14:52:39.102628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789299016965473:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49158
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.105448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789295842804789:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15397
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.108524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789296494661258:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36935
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.108703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298101381121:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19567
2025-11-28T14:52:39.108866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298101381122:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19567
2025-11-28T14:52:39.109568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298101381122:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.110040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789299016965474:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49158
2025-11-28T14:52:39.110122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789299016965473:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.110224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298101381126:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19567
2025-11-28T14:52:39.110591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789293806413750:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.110573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789299016965475:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49158
2025-11-28T14:52:39.110646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789299016965472:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.110724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789295842804789:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.110822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789296494661258:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.111072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789299016965478:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49158
2025-11-28T14:52:39.111063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789298831763806:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46458
2025-11-28T14:52:39.111115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789295842804790:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15397
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.111186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789291547837416:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.111222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789298831763807:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46458
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.111341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789298831763808:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46458
2025-11-28T14:52:39.111766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789299016965464:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.112206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763806:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.111788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789299016965464:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.112342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763807:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.112411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298101381121:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.112455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789298831763809:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46458
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.112551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789295842804792:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15397
2025-11-28T14:52:39.112534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789299016965462:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.112560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789299016965462:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.112628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789298831763810:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46458
2025-11-28T14:52:39.112859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763808:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.113546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789299016965464:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.114151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789295842804793:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15397
2025-11-28T14:52:39.114437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789299016965456:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.114487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.114538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789299016965456:2107] [node 4] ICP22 created new session: [2:7577789299016965481:2107]
2025-11-28T14:52:39.114701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789299016965481:2107] [node 4] ICS09 handshake done sender: [2:7577789299016965464:2112] self: [2:7577789299016965463:0] peer: [4:7577789295842804791:0] socket: 44 qp: -1
2025-11-28T14:52:39.114911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789299016965481:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:39.114940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:39.115079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789295842804789:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.115298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789291547837416:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.115406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.115453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789291547837416:2049] [node 2] ICP22 created new session: [4:7577789295842804794:2049]
2025-11-28T14:52:39.115502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789295842804794:2049] [node 2] ICS09 handshake done sender: [4:7577789295842804789:2115] self: [4:7577789295842804791:0] peer: [2:7577789299016965463:0] socket: 47 qp: -1
2025-11-28T14:52:39.115534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789295842804794:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:39.115564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:39.116305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789299016965462:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.116751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298101381126:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.116900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298101381128:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19567
2025-11-28T14:52:39.117168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789299016965474:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.117288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298101381129:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19567
2025-11-28T14:52:39.117304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789294721998098:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.117330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:39.117359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789294721998098:2049] [node 3] ICP22 created new session: [2:7577789299016965483:2049]
2025-11-28T14:52:39.117393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789299016965483:2049] [node 3] ICS09 handshake done sender: [2:7577789299016965462:2111] self: [2:7577789299016965461:0] peer: [3:7577789298101381127:0] socket: 43 qp: -1
2025-11-28T14:52:39.117424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789299016965483:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:39.117444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:39.117907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298101381122:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.118248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789293806413750:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.118375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.118415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789293806413750:2049] [node 2] ICP22 created new session: [3:7577789298101381132:2049]
2025-11-28T14:52:39.118462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298101381132:2049] [node 2] ICS09 handshake done sender: [3:7577789298101381122:2116] self: [3:7577789298101381127:0] peer: [2:7577789299016965461:0] socket: 53 qp: -1
2025-11-28T14:52:39.118486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298101381132:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:39.118518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:39.118613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763809:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.118714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763810:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.119201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789299016965475:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.119585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789299016965478:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.119964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789296494661263:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36935
2025-11-28T14:52:39.120117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789295842804792:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.120227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789291547837415:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.120259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789296494661264:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36935
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.121092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976749:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
2025-11-28T14:52:39.121109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789294721998098:2049] [node 3] ICP18 handshake (actor [2:7577789299016965475:2118]) is too old
2025-11-28T14:52:39.121178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298101381128:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.121180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789294721998097:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.121469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763801:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.121501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789298831763801:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.121582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789296494661264:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.121670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789295842804790:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.121710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789296494661246:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.121738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789291547837417:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.121870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789295842804783:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.121887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789295842804783:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.122117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298101381116:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.122146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789298101381116:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.122132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763797:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.122161Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789298831763797:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.122955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789295842804783:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:39.123223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789293806413749:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.123376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789295842804774:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.123330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789298831763801:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:39.123428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:39.123422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789296494661266:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36935
2025-11-28T14:52:39.123469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789295842804774:2108] [node 5] ICP22 created new session: [4:7577789295842804801:2108]
2025-11-28T14:52:39.123501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789295842804801:2108] [node 5] ICS09 handshake done sender: [4:7577789295842804783:2113] self: [4:7577789295842804782:0] peer: [5:7577789296494661265:0] socket: 82 qp: -1
2025-11-28T14:52:39.123527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789295842804801:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:39.123571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.123720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298101381116:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:39.123907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789298831763797:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.124004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789298101381108:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.124083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.124145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789298101381108:2107] [node 4] ICP22 created new session: [3:7577789298101381137:2107]
2025-11-28T14:52:39.124198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298101381137:2107] [node 4] ICS09 handshake done sender: [3:7577789298101381116:2112] self: [3:7577789298101381115:0] peer: [4:7577789295842804798:0] socket: 63 qp: -1
2025-11-28T14:52:39.124220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298101381137:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:39.124203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789296494661264:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:39.124311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:39.124523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763799:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.124771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789296494661246:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.124560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789298831763799:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.124811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.124854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789296494661246:2108] [node 4] ICP22 created new session: [5:7577789296494661268:2108]
2025-11-28T14:52:39.124894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789296494661268:2108] [node 4] ICS09 handshake done sender: [5:7577789296494661264:2117] self: [5:7577789296494661265:0] peer: [4:7577789295842804782:0] socket: 83 qp: -1
2025-11-28T14:52:39.124920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789296494661268:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:39.124962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.125243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789296783976749:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.125268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789299016965489:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49158
2025-11-28T14:52:39.125350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789296783976672:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.125381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789298831763793:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.125471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.125518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789298831763793:2107] [node 4] ICP22 created new session: [1:7577789298831763821:2107]
2025-11-28T14:52:39.125625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789298831763821:2107] [node 4] ICS09 handshake done sender: [1:7577789298831763801:2112] self: [1:7577789298831763800:0] peer: [4:7577789295842804797:0] socket: 74 qp: -1
2025-11-28T14:52:39.125658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789298831763821:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.125742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:39.125881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789295842804800:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15397
2025-11-28T14:52:39.125957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789299016965468:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.125987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789299016965468:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.126417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789299016965478:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:39.126669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789294721998097:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.126701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.126788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789294721998097:2048] [node 1] ICP22 created new session: [2:7577789299016965491:2048]
2025-11-28T14:52:39.126815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789299016965491:2048] [node 1] ICS09 handshake done sender: [2:7577789299016965478:2119] self: [2:7577789299016965488:0] peer: [1:7577789298831763796:0] socket: 72 qp: -1
2025-11-28T14:52:39.126835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789299016965491:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.126856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.127153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789296494661267:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36935
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.127447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789295842804803:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15397
2025-11-28T14:52:39.127537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789294536796434:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.127564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789296494661263:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.127630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.127683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789294536796434:2048] [node 2] ICP22 created new session: [1:7577789298831763823:2048]
2025-11-28T14:52:39.127731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789298831763823:2048] [node 2] ICS09 handshake done sender: [1:7577789298831763797:2110] self: [1:7577789298831763796:0] peer: [2:7577789299016965488:0] socket: 71 qp: -1
2025-11-28T14:52:39.127783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789298831763823:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:39.127812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.127868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976751:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.127985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789296494661267:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.128040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976752:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.129468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789292199693888:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.129534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976753:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
2025-11-28T14:52:39.129562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789292199693889:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.129578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789295842804792:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:39.129796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789299016965468:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:39.129894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789291547837417:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28492) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789295842804790:2116] held: no
2025-11-28T14:52:39.129994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298101381118:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.130023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789298101381118:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.130351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789299016965466:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.130376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789299016965466:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.130587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789296783976752:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.130596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298101381118:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:39.130632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789291547837415:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.130676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.130725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789291547837415:2048] [node 1] ICP22 created new session: [4:7577789295842804805:2048]
2025-11-28T14:52:39.130661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789296783976749:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.130862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789295842804805:2048] [node 1] ICS09 handshake done sender: [4:7577789295842804792:2117] self: [4:7577789295842804797:0] peer: [1:7577789298831763800:0] socket: 75 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.131312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789298101381109:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.131220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789299016965458:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.131211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789295842804805:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.131352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:39.131390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:39.131312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:39.131375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789298101381109:2108] [node 5] ICP22 created new session: [3:7577789298101381141:2108]
2025-11-28T14:52:39.131339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789299016965458:2109] [node 6] ICP22 created new session: [2:7577789299016965493:2109]
2025-11-28T14:52:39.131370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976754:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
2025-11-28T14:52:39.131484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298101381141:2108] [node 5] ICS09 handshake done sender: [3:7577789298101381118:2113] self: [3:7577789298101381117:0] peer: [5:7577789296494661272:0] socket: 89 qp: -1
2025-11-28T14:52:39.131388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789299016965493:2109] [node 6] ICS09 handshake done sender: [2:7577789299016965468:2114] self: [2:7577789299016965467:0] peer: [6:7577789296783976750:0] socket: 84 qp: -1
2025-11-28T14:52:39.131527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298101381141:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:39.131408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789299016965493:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:39.131482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789296783976672:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.131478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:39.131509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.131549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.131539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789296783976672:2049] [node 2] ICP22 created new session: [6:7577789296783976755:2049]
2025-11-28T14:52:39.131562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789296783976755:2049] [node 2] ICS09 handshake done sender: [6:7577789296783976749:2115] self: [6:7577789296783976750:0] peer: [2:7577789299016965467:0] socket: 85 qp: -1
2025-11-28T14:52:39.131618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789296783976755:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:39.131638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.131977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789296494661273:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36935
2025-11-28T14:52:39.132497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789296494661267:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.132801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789292199693889:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.132846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:39.132889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789292199693889:2050] [node 3] ICP22 created new session: [5:7577789296494661274:2050]
2025-11-28T14:52:39.132925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789296494661274:2050] [node 3] ICS09 handshake done sender: [5:7577789296494661267:2119] self: [5:7577789296494661272:0] peer: [3:7577789298101381117:0] socket: 92 qp: -1
2025-11-28T14:52:39.132999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789296494661274:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:39.133026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:39.133326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789296783976673:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.134020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789295842804790:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.136270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298101381120:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.136313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789291547837417:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.136291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789298101381120:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.136385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:39.136788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789291547837417:2050] [node 3] ICP22 created new session: [4:7577789295842804807:2050]
2025-11-28T14:52:39.136814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789295842804807:2050] [node 3] ICS09 handshake done sender: [4:7577789295842804790:2116] self: [4:7577789295842804798:0] peer: [3:7577789298101381115:0] socket: 69 qp: -1
2025-11-28T14:52:39.137147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789295842804807:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:39.137213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.137357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298101381143:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19567
2025-11-28T14:52:39.137509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789299016965466:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:39.137524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789296783976754:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.137614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789296783976734:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.137807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789299016965457:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.137869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:39.137907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789299016965457:2108] [node 5] ICP22 created new session: [2:7577789299016965496:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.137937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789299016965496:2108] [node 5] ICS09 handshake done sender: [2:7577789299016965466:2113] self: [2:7577789299016965465:0] peer: [5:7577789296494661271:0] socket: 38 qp: -1
2025-11-28T14:52:39.137955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789299016965496:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:39.137928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789298831763799:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.137980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.137990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976760:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
2025-11-28T14:52:39.138103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789296494661257:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.138127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789296494661257:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.138812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976761:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
2025-11-28T14:52:39.138823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789296494661276:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36935
2025-11-28T14:52:39.138848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789294536796435:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.138935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298101381144:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19567
2025-11-28T14:52:39.138966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:39.139066Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298101381120:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.139076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789294536796435:2049] [node 3] ICP22 created new session: [1:7577789298831763828:2049]
2025-11-28T14:52:39.139141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789296494661277:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36935
2025-11-28T14:52:39.139156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789298831763828:2049] [node 3] ICS09 handshake done sender: [1:7577789298831763799:2111] self: [1:7577789298831763798:0] peer: [3:7577789298101381136:0] socket: 73 qp: -1
2025-11-28T14:52:39.139248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789298831763828:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:39.139299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:39.139292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789296494661263:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.139308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298101381128:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:39.139755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789296494661257:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:39.139713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298101381144:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.139851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789298101381110:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.139987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:39.140054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789298101381110:2109] [node 6] ICP22 created new session: [3:7577789298101381146:2109]
2025-11-28T14:52:39.140085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298101381146:2109] [node 6] ICS09 handshake done sender: [3:7577789298101381120:2114] self: [3:7577789298101381119:0] peer: [6:7577789296783976757:0] socket: 95 qp: -1
2025-11-28T14:52:39.140103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298101381146:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:39.140160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:39.140281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789292199693888:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.140311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.140272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789296783976752:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.140335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789292199693888:2049] [node 2] ICP22 created new session: [5:7577789296494661279:2049]
2025-11-28T14:52:39.140423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789296494661279:2049] [node 2] ICS09 handshake done sender: [5:7577789296494661263:2116] self: [5:7577789296494661271:0] peer: [2:7577789299016965465:0] socket: 45 qp: -1
2025-11-28T14:52:39.140447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789296494661279:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:39.140476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:39.140555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789296783976673:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.140596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:39.140618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789296783976673:2050] [node 3] ICP22 created new session: [6:7577789296783976763:2050]
2025-11-28T14:52:39.140640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789296783976763:2050] [node 3] ICS09 handshake done sender: [6:7577789296783976752:2117] self: [6:7577789296783976757:0] peer: [3:7577789298101381119:0] socket: 96 qp: -1
2025-11-28T14:52:39.140685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789296783976763:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:39.140705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:39.141049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789293806413749:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.141078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.141110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789293806413749:2048] [node 1] ICP22 created new session: [3:7577789298101381148:2048]
2025-11-28T14:52:39.141149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298101381148:2048] [node 1] ICS09 handshake done sender: [3:7577789298101381128:2118] self: [3:7577789298101381136:0] peer: [1:7577789298831763798:0] socket: 76 qp: -1
2025-11-28T14:52:39.141164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298101381148:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.141237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.142315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976762:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
2025-11-28T14:52:39.142786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789296494661247:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.142935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:39.143209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789296494661247:2109] [node 6] ICP22 created new session: [5:7577789296494661281:2109]
2025-11-28T14:52:39.143236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789296494661281:2109] [node 6] ICS09 handshake done sender: [5:7577789296494661257:2114] self: [5:7577789296494661256:0] peer: [6:7577789296783976759:0] socket: 39 qp: -1
2025-11-28T14:52:39.143260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789296494661281:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:39.143301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:39.143417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789296783976751:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.143731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789296783976754:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:39.144710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789296783976734:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.144739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:39.144768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789296783976734:2105] [node 5] ICP22 created new session: [6:7577789296783976765:2105]
2025-11-28T14:52:39.144804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789296783976765:2105] [node 5] ICS09 handshake done sender: [6:7577789296783976754:2119] self: [6:7577789296783976759:0] peer: [5:7577789296494661256:0] socket: 41 qp: -1
2025-11-28T14:52:39.144823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789296783976765:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:39.144841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.145109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789292199693888:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:39.145441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789298101381110:2109] [node 6] ICP18 handshake (actor [3:7577789298101381144:2121]) is too old
2025-11-28T14:52:39.146155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789296783976735:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.146931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789296494661277:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.147026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789292199693887:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.147194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763803:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.147229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789298831763803:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.147428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789295842804785:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.147449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789295842804785:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.147512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789296783976760:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.147729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789296783976671:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.147994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789298831763803:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.148290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789298831763794:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.148283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789295842804809:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15397
2025-11-28T14:52:39.148285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789296494661284:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36935
2025-11-28T14:52:39.148429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789295842804785:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:39.148468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:39.148515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789298831763794:2108] [node 5] ICP22 created new session: [1:7577789298831763831:2108]
2025-11-28T14:52:39.148614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789298831763831:2108] [node 5] ICS09 handshake done sender: [1:7577789298831763803:2113] self: [1:7577789298831763802:0] peer: [5:7577789296494661283:0] socket: 59 qp: -1
2025-11-28T14:52:39.148650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789296494661277:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:39.148658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789298831763831:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:39.149178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.149177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789295842804775:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.149215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789292199693887:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.149293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.149340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789292199693887:2048] [node 1] ICP22 created new session: [5:7577789296494661285:2048]
2025-11-28T14:52:39.149339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:39.149367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789296494661285:2048] [node 1] ICS09 handshake done sender: [5:7577789296494661277:2122] self: [5:7577789296494661283:0] peer: [1:7577789298831763802:0] socket: 102 qp: -1
2025-11-28T14:52:39.149394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789296494661285:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.149412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:39.149405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789295842804775:2109] [node 6] ICP22 created new session: [4:7577789295842804811:2109]
2025-11-28T14:52:39.149457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789295842804811:2109] [node 6] ICS09 handshake done sender: [4:7577789295842804785:2114] self: [4:7577789295842804784:0] peer: [6:7577789296783976768:0] socket: 52 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.149501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789295842804811:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:39.149540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976770:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
2025-11-28T14:52:39.149567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:39.149884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789296783976751:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:39.150042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789296783976735:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.150068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.150091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789296783976735:2107] [node 4] ICP22 created new session: [6:7577789296783976771:2107]
2025-11-28T14:52:39.150118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789296783976771:2107] [node 4] ICS09 handshake done sender: [6:7577789296783976751:2116] self: [6:7577789296783976768:0] peer: [4:7577789295842804784:0] socket: 94 qp: -1
2025-11-28T14:52:39.150137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789296783976771:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:39.150157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:39.151802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789298831763805:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.151835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789298831763805:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.152413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789295842804775:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:39.152958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789298831763805:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.153079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789296783976773:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4185
2025-11-28T14:52:39.153266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789298831763795:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.153678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789296783976760:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:39.153571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:39.153627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789298831763795:2109] [node 6] ICP22 created new session: [1:7577789298831763834:2109]
2025-11-28T14:52:39.153673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789298831763834:2109] [node 6] ICS09 handshake done sender: [1:7577789298831763805:2114] self: [1:7577789298831763804:0] peer: [6:7577789296783976769:0] socket: 101 qp: -1
2025-11-28T14:52:39.153702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789298831763834:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:39.153728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:39.154512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789296783976671:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.154564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.154615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789296783976671:2048] [node 1] ICP22 created new session: [6:7577789296783976774:2048]
2025-11-28T14:52:39.154658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789296783976774:2048] [node 1] ICS09 handshake done sender: [6:7577789296783976760:2120] self: [6:7577789296783976769:0] peer: [1:7577789298831763804:0] socket: 103 qp: -1
2025-11-28T14:52:39.154694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789296783976774:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.154729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:39.201228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:39.201389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789299016965491:2048] [node 1] ICS07 socket disconnect 72 reason# EndOfStream
2025-11-28T14:52:39.201417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789299016965491:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.201491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.201537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789299016965491:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.202535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:39.202663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789298101381148:2048] [node 1] ICS07 socket disconnect 76 reason# EndOfStream
2025-11-28T14:52:39.202682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789298101381148:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.202734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.202764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789298101381148:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.204759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789294721998097:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:39.204795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789299016965491:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.205436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789293806413749:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:39.205465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789298101381148:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.205554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:39.205702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789295842804805:2048] [node 1] ICS01 socket: 75 reason# EndOfStream
2025-11-28T14:52:39.205743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789291547837415:2048] [node 1] ICP30 unregister session Session# [4:7577789295842804805:2048] VirtualId# [4:7577789295842804797:0]
2025-11-28T14:52:39.205764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789295842804805:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.205815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.205756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789294721998097:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.205857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.205793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789299016965491:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.205813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789294721998097:2048] [node 1] ICP30 unregister session Session# [2:7577789299016965491:2048] VirtualId# [2:7577789299016965488:0]
2025-11-28T14:52:39.206026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789293806413749:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.206063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789298101381148:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.205869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.206087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789293806413749:2048] [node 1] ICP30 unregister session Session# [3:7577789298101381148:2048] VirtualId# [3:7577789298101381136:0]
2025-11-28T14:52:39.206191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.205888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789294721998097:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.206215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789293806413749:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.206223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.205898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.206866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:39.207007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789296494661285:2048] [node 1] ICS07 socket disconnect 102 reason# EndOfStream
2025-11-28T14:52:39.207032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789296494661285:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.207086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.207118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789296494661285:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.207715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789292199693887:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.207754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789296494661285:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.207787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789292199693887:2048] [node 1] ICP30 unregister session Session# [5:7577789296494661285:2048] VirtualId# [5:7577789296494661283:0]
2025-11-28T14:52:39.207824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.207859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789292199693887:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.207875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.207868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:39.208018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789296783976774:2048] [node 1] ICS07 socket disconnect 103 reason# EndOfStream
2025-11-28T14:52:39.208048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789296783976774:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.208101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.208152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789296783976774:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.208758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789296783976671:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.208795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789296783976774:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.208813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789296783976671:2048] [node 1] ICP30 unregister session Session# [6:7577789296783976774:2048] VirtualId# [6:7577789296783976769:0]
2025-11-28T14:52:39.208895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.208918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789296783976671:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.208926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.215746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789293806413749:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:39.217778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789292199693887:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:39.218830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789296783976671:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:39.228179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:39.228343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789298101381132:2049] [node 2] ICS01 socket: 53 reason# EndOfStream
2025-11-28T14:52:39.228390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789293806413750:2049] [node 2] ICP30 unregister session Session# [3:7577789298101381132:2049] VirtualId# [3:7577789298101381127:0]
2025-11-28T14:52:39.228412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789298101381132:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.228499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:39.228545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:39.232422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:39.232607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789295842804794:2049] [node 2] ICS07 socket disconnect 47 reason# EndOfStream
2025-11-28T14:52:39.232635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789295842804794:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.232721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:39.232756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789295842804794:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:39.233391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789291547837416:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1728) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.233437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789295842804794:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.233457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789291547837416:2049] [node 2] ICP30 unregister session Session# [4:7577789295842804794:2049] VirtualId# [4:7577789295842804791:0]
2025-11-28T14:52:39.233552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:39.233580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789291547837416:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.233589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.233594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:39.233825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789296494661279:2049] [node 2] ICS07 socket disconnect 45 reason# EndOfStream
2025-11-28T14:52:39.233866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789296494661279:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.233944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:39.233987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789296494661279:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:39.234518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:39.234647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789296783976755:2049] [node 2] ICS07 socket disconnect 85 reason# EndOfStream
2025-11-28T14:52:39.234671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789296783976755:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.234728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:39.234757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789296783976755:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:39.234780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789292199693888:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1728) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.234829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789296494661279:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.234870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789292199693888:2049] [node 2] ICP30 unregister session Session# [5:7577789296494661279:2049] VirtualId# [5:7577789296494661271:0]
2025-11-28T14:52:39.234976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:39.235011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789292199693888:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.235029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.235353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789296783976672:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1728) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.235386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789296783976755:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.235406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789296783976672:2049] [node 2] ICP30 unregister session Session# [6:7577789296783976755:2049] VirtualId# [6:7577789296783976750:0]
2025-11-28T14:52:39.235432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:39.235449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789296783976672:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.235459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.243390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789291547837416:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:39.244408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789292199693888:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:39.245430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789296783976672:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:39.274779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:39.274953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789295842804807:2050] [node 3] ICS01 socket: 69 reason# EndOfStream
2025-11-28T14:52:39.275002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789291547837417:2050] [node 3] ICP30 unregister session Session# [4:7577789295842804807:2050] VirtualId# [4:7577789295842804798:0]
2025-11-28T14:52:39.275025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789295842804807:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.275103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:39.275143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:39.275927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:39.276085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789296494661274:2050] [node 3] ICS07 socket disconnect 92 reason# EndOfStream
2025-11-28T14:52:39.276108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789296494661274:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.276172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:39.276206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789296494661274:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:39.277080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789292199693889:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28492) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.277115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789296494661274:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.277141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789292199693889:2050] [node 3] ICP30 unregister session Session# [5:7577789296494661274:2050] VirtualId# [5:7577789296494661272:0]
2025-11-28T14:52:39.277228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:39.277174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:39.277251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789292199693889:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28492) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.277260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28492) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.277321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789296783976763:2050] [node 3] ICS07 socket disconnect 96 reason# EndOfStream
2025-11-28T14:52:39.277346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789296783976763:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.277403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:39.277437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789296783976763:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:39.277989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789296783976673:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28492) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.278024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789296783976763:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.278042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789296783976673:2050] [node 3] ICP30 unregister session Session# [6:7577789296783976763:2050] VirtualId# [6:7577789296783976757:0]
2025-11-28T14:52:39.278126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:39.278151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789296783976673:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28492) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.278161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28492) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.287463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789292199693889:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:39.288464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789296783976673:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:39.296599Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:39.296789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789296494661268:2108] [node 4] ICS07 socket disconnect 83 reason# EndOfStream
2025-11-28T14:52:39.296814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789296494661268:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.296875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:39.296906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789296494661268:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:39.297475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789296494661246:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9532) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.297515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789296494661268:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.297536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789296494661246:2108] [node 4] ICP30 unregister session Session# [5:7577789296494661268:2108] VirtualId# [5:7577789296494661265:0]
2025-11-28T14:52:39.297631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:39.297659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789296494661246:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9532) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.297669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9532) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.297674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:39.297826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789296783976771:2107] [node 4] ICS07 socket disconnect 94 reason# EndOfStream
2025-11-28T14:52:39.297869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789296783976771:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.297936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:39.297969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789296783976771:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:39.298458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789296783976735:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9532) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.298497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789296783976771:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.298524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789296783976735:2107] [node 4] ICP30 unregister session Session# [6:7577789296783976771:2107] VirtualId# [6:7577789296783976768:0]
2025-11-28T14:52:39.298605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:39.298628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789296783976735:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9532) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.298638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9532) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.307917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789296494661246:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:39.308973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789296783976735:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:39.346477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:39.346712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789296783976765:2105] [node 5] ICS01 socket: 41 reason# EndOfStream
2025-11-28T14:52:39.346762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789296783976734:2105] [node 5] ICP30 unregister session Session# [6:7577789296783976765:2105] VirtualId# [6:7577789296783976759:0]
2025-11-28T14:52:39.346800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789296783976765:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.346886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:39.346948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:39.429682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789296520720411:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:39.429720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789296520720412:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:39.453555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789298576573994:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:39.453584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789298576573995:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:39.476109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789298681445988:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:39.476163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789298681445989:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:39.499202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789298388813064:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:39.499228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789298388813065:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:39.499237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789298388813066:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:39.515963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789295874391722:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:39.516006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789295874391724:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:39.516075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789295874391723:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:39.539658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789297645507844:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:39.539695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789297645507846:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:39.539718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789297645507845:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:39.621352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789296520720475:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:39.626040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789297645507905:2105] [node 4] ICP01 ready to work
2025-11-28T14:52:39.625897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789298388813127:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:39.626100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789298681446052:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:39.626503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789298576574057:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:39.626388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789296520720476:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:39.626444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789295874391785:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:39.626205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789297645507906:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:39.626402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789298388813128:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:39.626640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789298681446054:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:39.626649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789296520720477:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:39.627097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789298576574058:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:39.627095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789298681446053:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:39.627230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789298576574059:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:39.627285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789295874391786:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.641157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789296520720486:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24937
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.641460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789298388813142:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63227
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.641622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298681446063:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:52:39.642532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298681446066:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.643378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720486:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.643398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298681446063:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.643762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789296520720489:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24937
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.644531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789296520720490:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24937
2025-11-28T14:52:39.644972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298681446070:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.645556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789296520720494:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24937
2025-11-28T14:52:39.645628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789298576574068:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.647368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789298576574069:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.647523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789295874391800:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11026
2025-11-28T14:52:39.647517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789298576574070:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-11-28T14:52:39.647492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789298388813143:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63227
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.647663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789298576574073:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-11-28T14:52:39.647750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789298576574074:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-11-28T14:52:39.647836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789298576574068:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.647931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789298576574069:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.647952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789298388813144:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63227
2025-11-28T14:52:39.647954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789298576574070:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.647951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789298388813142:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.649308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789298576574073:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.649390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298681446066:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.649488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298681446072:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:52:39.649599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720489:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.649862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298681446070:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.649932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789298681445988:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.650120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720490:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.650141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789295874391800:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.650381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720494:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.650460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789296520720496:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24937
2025-11-28T14:52:39.650462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789295874391722:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.650563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720480:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.650589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789296520720480:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.650744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789298388813143:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.650886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789298388813066:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.651076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298681446060:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.651111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789298681446060:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.651237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720483:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.651270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789296520720483:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.651899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298681446060:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.652358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789298388813147:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63227
2025-11-28T14:52:39.652444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789298681446052:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.652658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.652707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789298681446052:2107] [node 4] ICP22 created new session: [3:7577789298681446076:2107]
2025-11-28T14:52:39.653136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298681446076:2107] [node 4] ICS09 handshake done sender: [3:7577789298681446060:2113] self: [3:7577789298681446059:0] peer: [4:7577789298388813146:0] socket: 67 qp: -1
2025-11-28T14:52:39.653182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298681446076:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:39.653165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789296520720480:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.653335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:39.653418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789296520720483:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:39.653386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789298388813143:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.653878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789298388813066:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.654056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789296520720412:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.654114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:39.654238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:39.654223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789298388813066:2050] [node 3] ICP22 created new session: [4:7577789298388813148:2050]
2025-11-28T14:52:39.654288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789296520720412:2049] [node 3] ICP22 created new session: [1:7577789296520720502:2049]
2025-11-28T14:52:39.654280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789298388813148:2050] [node 3] ICS09 handshake done sender: [4:7577789298388813143:2116] self: [4:7577789298388813146:0] peer: [3:7577789298681446059:0] socket: 70 qp: -1
2025-11-28T14:52:39.654276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789295874391802:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11026
2025-11-28T14:52:39.654332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789298388813148:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:39.654375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789296520720502:2049] [node 3] ICS09 handshake done sender: [1:7577789296520720480:2111] self: [1:7577789296520720478:0] peer: [3:7577789298681446074:0] socket: 64 qp: -1
2025-11-28T14:52:39.654403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:39.654406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789296520720475:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.654414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789296520720502:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:39.654451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:39.654552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789295874391800:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:39.654453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.654545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789296520720475:2108] [node 5] ICP22 created new session: [1:7577789296520720504:2108]
2025-11-28T14:52:39.654595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789296520720504:2108] [node 5] ICS09 handshake done sender: [1:7577789296520720483:2113] self: [1:7577789296520720482:0] peer: [5:7577789295874391801:0] socket: 71 qp: -1
2025-11-28T14:52:39.655202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789295874391722:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.654630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789296520720504:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:39.655298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298681446078:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:52:39.655033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.655321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.655374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789295874391722:2048] [node 1] ICP22 created new session: [5:7577789295874391803:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.655427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789295874391803:2048] [node 1] ICS09 handshake done sender: [5:7577789295874391800:2115] self: [5:7577789295874391801:0] peer: [1:7577789296520720482:0] socket: 74 qp: -1
2025-11-28T14:52:39.655451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789295874391803:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.656486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507922:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
2025-11-28T14:52:39.655561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:39.656877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298681446070:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:39.657320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789298388813066:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:39.657529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298681446072:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.657561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789298681445988:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:39.657596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.657646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789298681445988:2048] [node 1] ICP22 created new session: [3:7577789298681446080:2048]
2025-11-28T14:52:39.657689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298681446080:2048] [node 1] ICS09 handshake done sender: [3:7577789298681446070:2118] self: [3:7577789298681446074:0] peer: [1:7577789296520720478:0] socket: 68 qp: -1
2025-11-28T14:52:39.657670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789295874391805:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11026
2025-11-28T14:52:39.657859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298681446080:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.657900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:39.659004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789298576574074:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.659177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789298576573994:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.659447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720496:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.659513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789295874391805:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.659634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789295874391785:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.659636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720481:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.659662Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789296520720481:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.660815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789298388813136:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.660842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789298388813136:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.661897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789298388813144:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.661992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789298388813151:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63227
2025-11-28T14:52:39.661972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507923:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
2025-11-28T14:52:39.662070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789298388813065:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.662172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789298388813136:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:39.662298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789298576574065:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.662351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789298388813127:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.663065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789297645507922:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.662534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789295874391808:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11026
2025-11-28T14:52:39.662332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789298576574065:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.662396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:39.663227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789297645507846:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.662429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789298388813127:2108] [node 5] ICP22 created new session: [4:7577789298388813154:2108]
2025-11-28T14:52:39.663232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789298388813154:2108] [node 5] ICS09 handshake done sender: [4:7577789298388813136:2113] self: [4:7577789298388813134:0] peer: [5:7577789295874391807:0] socket: 52 qp: -1
2025-11-28T14:52:39.663289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789298388813154:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:39.663344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789295874391805:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:39.663329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.663522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789296520720481:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.663671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789298576574065:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:39.663723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789295874391785:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.663774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.663823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789295874391785:2108] [node 4] ICP22 created new session: [5:7577789295874391809:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.663881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789295874391809:2108] [node 4] ICS09 handshake done sender: [5:7577789295874391805:2117] self: [5:7577789295874391807:0] peer: [4:7577789298388813134:0] socket: 55 qp: -1
2025-11-28T14:52:39.664083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789295874391809:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:39.664119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:39.664112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789298576574084:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-11-28T14:52:39.664126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789298681445989:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.664437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298681446062:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.664634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789298576574057:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.664466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789298681446062:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.665303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.665795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298681446082:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:52:39.665670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789298576574057:2107] [node 4] ICP22 created new session: [2:7577789298576574085:2107]
2025-11-28T14:52:39.666120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298681446062:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:39.665766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789298576574085:2107] [node 4] ICS09 handshake done sender: [2:7577789298576574065:2112] self: [2:7577789298576574064:0] peer: [4:7577789298388813153:0] socket: 80 qp: -1
2025-11-28T14:52:39.665804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789298576574085:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:39.666062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:39.666764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789298576574074:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.666700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789296520720411:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.666824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.666902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298681446082:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.667553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507925:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
2025-11-28T14:52:39.666913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789296520720411:2048] [node 2] ICP22 created new session: [1:7577789296520720510:2048]
2025-11-28T14:52:39.667688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789296520720510:2048] [node 2] ICS09 handshake done sender: [1:7577789296520720481:2112] self: [1:7577789296520720479:0] peer: [2:7577789298576574082:0] socket: 69 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.667730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789296520720510:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:39.667544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789298681446054:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.667787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:39.667722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:39.667770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789298576574062:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.667793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507926:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
2025-11-28T14:52:39.667784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789298681446054:2110] [node 6] ICP22 created new session: [3:7577789298681446087:2110]
2025-11-28T14:52:39.667834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298681446087:2110] [node 6] ICS09 handshake done sender: [3:7577789298681446062:2114] self: [3:7577789298681446061:0] peer: [6:7577789297645507924:0] socket: 53 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.667804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789298576574062:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.667846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789297645507922:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.667878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298681446087:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:39.669785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:39.669267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789298388813156:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63227
2025-11-28T14:52:39.669771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789295874391811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11026
2025-11-28T14:52:39.670573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789298388813065:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789298388813144:2117] held: no
2025-11-28T14:52:39.670589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789297645507846:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789297645507922:2115] held: no
2025-11-28T14:52:39.670743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789298576573994:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.670810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.670847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789295874391811:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.670850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789298576573994:2048] [node 1] ICP22 created new session: [2:7577789298576574089:2048]
2025-11-28T14:52:39.671084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789298388813151:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.671063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789298576574089:2048] [node 1] ICS09 handshake done sender: [2:7577789298576574074:2119] self: [2:7577789298576574082:0] peer: [1:7577789296520720479:0] socket: 72 qp: -1
2025-11-28T14:52:39.671098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789298576574089:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.671144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:39.671237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789297645507923:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.671455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789297645507846:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.671526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:39.671562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789297645507846:2050] [node 3] ICP22 created new session: [6:7577789297645507928:2050]
2025-11-28T14:52:39.671605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789297645507928:2050] [node 3] ICS09 handshake done sender: [6:7577789297645507922:2115] self: [6:7577789297645507924:0] peer: [3:7577789298681446061:0] socket: 90 qp: -1
2025-11-28T14:52:39.671627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789297645507928:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:39.671676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.672011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507927:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
2025-11-28T14:52:39.672287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789297645507844:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.672619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720488:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.672652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789296520720488:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.673301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789295874391724:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.673347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789298388813144:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.674201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789296520720488:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:39.674343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789298681446065:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.674364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789298681446065:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.674733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789296520720477:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.674784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.674991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789297645507927:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.674830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789296520720477:2109] [node 6] ICP22 created new session: [1:7577789296520720513:2109]
2025-11-28T14:52:39.674836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789298576574062:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.675011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789296520720513:2109] [node 6] ICS09 handshake done sender: [1:7577789296520720488:2116] self: [1:7577789296520720487:0] peer: [6:7577789297645507931:0] socket: 92 qp: -1
2025-11-28T14:52:39.675046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789295874391814:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11026
2025-11-28T14:52:39.675042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789296520720513:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:39.675082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.675322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789298388813065:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.675422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789298576573994:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:39.675576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298681446065:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:39.675354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.675376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789298388813065:2049] [node 2] ICP22 created new session: [4:7577789298388813159:2049]
2025-11-28T14:52:39.675714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789298681446089:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34102
2025-11-28T14:52:39.675597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789298388813159:2049] [node 2] ICS09 handshake done sender: [4:7577789298388813144:2117] self: [4:7577789298388813153:0] peer: [2:7577789298576574064:0] socket: 81 qp: -1
2025-11-28T14:52:39.675623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789298388813159:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:39.675724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:39.675971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789298576573995:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.676057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789297645507906:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.675968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789298388813064:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.676198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789298681446053:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.676496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:39.676469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789297645507844:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789297645507923:2116] held: no
2025-11-28T14:52:39.676517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:39.676531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789298681446053:2108] [node 5] ICP22 created new session: [3:7577789298681446091:2108]
2025-11-28T14:52:39.676666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298681446091:2108] [node 5] ICS09 handshake done sender: [3:7577789298681446065:2116] self: [3:7577789298681446064:0] peer: [5:7577789295874391813:0] socket: 100 qp: -1
2025-11-28T14:52:39.676665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789298576573995:2049] [node 3] ICP22 created new session: [2:7577789298576574091:2049]
2025-11-28T14:52:39.676666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789295874391815:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11026
2025-11-28T14:52:39.676693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298681446091:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:39.676715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789298576574091:2049] [node 3] ICS09 handshake done sender: [2:7577789298576574062:2110] self: [2:7577789298576574061:0] peer: [3:7577789298681446083:0] socket: 77 qp: -1
2025-11-28T14:52:39.676720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.676756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789298576574091:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:39.676799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.676947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789295874391811:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:39.677300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507932:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
2025-11-28T14:52:39.677880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789298681446072:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.677772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789297645507926:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.677571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789295874391724:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.677703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.677806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789295874391724:2050] [node 3] ICP22 created new session: [5:7577789295874391816:2050]
2025-11-28T14:52:39.678506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789297645507923:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:39.678605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789298388813158:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63227
2025-11-28T14:52:39.678672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789297645507905:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.677844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789295874391816:2050] [node 3] ICS09 handshake done sender: [5:7577789295874391811:2119] self: [5:7577789295874391813:0] peer: [3:7577789298681446064:0] socket: 102 qp: -1
2025-11-28T14:52:39.677992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789295874391816:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:39.679054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789298681445989:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.679076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789297645507844:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.678023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:39.679461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789298388813141:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.679492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.679080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789295874391799:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.679481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789298388813141:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.679106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789295874391799:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.679520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.679740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789297645507844:2048] [node 1] ICP22 created new session: [6:7577789297645507936:2048]
2025-11-28T14:52:39.679542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789298681445989:2049] [node 2] ICP22 created new session: [3:7577789298681446093:2049]
2025-11-28T14:52:39.679789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789297645507936:2048] [node 1] ICS09 handshake done sender: [6:7577789297645507923:2116] self: [6:7577789297645507931:0] peer: [1:7577789296520720487:0] socket: 93 qp: -1
2025-11-28T14:52:39.679820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789298681446093:2049] [node 2] ICS09 handshake done sender: [3:7577789298681446072:2119] self: [3:7577789298681446083:0] peer: [2:7577789298576574061:0] socket: 83 qp: -1
2025-11-28T14:52:39.679804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789297645507936:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.680526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:39.679898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789298681446093:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:39.680555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:39.680612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789298388813158:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.681097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789295874391724:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:39.681160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789298388813141:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.681240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789296520720485:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.681326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789298388813128:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.681253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789295874391818:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11026
2025-11-28T14:52:39.681279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789296520720485:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.681391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789295874391799:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:39.681361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:39.682190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789298388813128:2109] [node 6] ICP22 created new session: [4:7577789298388813164:2109]
2025-11-28T14:52:39.682847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789295874391786:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.683027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:39.682401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789298388813164:2109] [node 6] ICS09 handshake done sender: [4:7577789298388813141:2114] self: [4:7577789298388813140:0] peer: [6:7577789297645507935:0] socket: 61 qp: -1
2025-11-28T14:52:39.683071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789295874391786:2109] [node 6] ICP22 created new session: [5:7577789295874391820:2109]
2025-11-28T14:52:39.683128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789295874391820:2109] [node 6] ICS09 handshake done sender: [5:7577789295874391799:2114] self: [5:7577789295874391798:0] peer: [6:7577789297645507933:0] socket: 103 qp: -1
2025-11-28T14:52:39.682995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789298388813164:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:39.683354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789295874391820:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:39.683017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:39.683387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:39.683490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789296520720485:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:39.683695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789296520720476:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:39.683961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.684190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789296520720476:2107] [node 4] ICP22 created new session: [1:7577789296520720516:2107]
2025-11-28T14:52:39.684275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789298388813166:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63227
2025-11-28T14:52:39.684421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789296520720516:2107] [node 4] ICS09 handshake done sender: [1:7577789296520720485:2114] self: [1:7577789296520720484:0] peer: [4:7577789298388813161:0] socket: 58 qp: -1
2025-11-28T14:52:39.684471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789296520720516:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:39.684514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:39.686071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789297645507905:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789297645507926:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.686330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507938:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.686605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789298388813151:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:39.686994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507939:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
2025-11-28T14:52:39.687874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789295874391818:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.687679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789298388813064:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789298388813151:2119] held: no
2025-11-28T14:52:39.688087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789297645507926:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:39.688211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789297645507905:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.688313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:39.688357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789297645507905:2105] [node 4] ICP22 created new session: [6:7577789297645507941:2105]
2025-11-28T14:52:39.688438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789297645507941:2105] [node 4] ICS09 handshake done sender: [6:7577789297645507926:2118] self: [6:7577789297645507935:0] peer: [4:7577789298388813140:0] socket: 101 qp: -1
2025-11-28T14:52:39.688473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789297645507941:2105] [node 4] ICS10 traffic start
2025-11-28T14:52:39.688486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789297645507927:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:39.688451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789298388813064:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.688507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:39.688508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:39.688542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789298388813064:2048] [node 1] ICP22 created new session: [4:7577789298388813167:2048]
2025-11-28T14:52:39.688381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789295874391786:2109] [node 6] ICP18 handshake (actor [5:7577789295874391818:2122]) is too old
2025-11-28T14:52:39.688584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789298388813167:2048] [node 1] ICS09 handshake done sender: [4:7577789298388813151:2119] self: [4:7577789298388813161:0] peer: [1:7577789296520720484:0] socket: 73 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.688652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789298388813167:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:39.688774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789295874391814:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.688679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:39.689057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507940:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
2025-11-28T14:52:39.689016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789295874391723:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.689233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789297645507906:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.689297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:39.689259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789298576574067:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.689345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789297645507906:2106] [node 5] ICP22 created new session: [6:7577789297645507943:2106]
2025-11-28T14:52:39.689306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789298576574067:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.689471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789297645507943:2106] [node 5] ICS09 handshake done sender: [6:7577789297645507927:2119] self: [6:7577789297645507933:0] peer: [5:7577789295874391798:0] socket: 49 qp: -1
2025-11-28T14:52:39.689508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789297645507943:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:39.689554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.690206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789298576574067:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.691157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789298576574058:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.691256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:39.691309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789298576574058:2108] [node 5] ICP22 created new session: [2:7577789298576574095:2108]
2025-11-28T14:52:39.691243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789295874391823:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11026
2025-11-28T14:52:39.691367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789298576574095:2108] [node 5] ICS09 handshake done sender: [2:7577789298576574067:2113] self: [2:7577789298576574066:0] peer: [5:7577789295874391822:0] socket: 78 qp: -1
2025-11-28T14:52:39.691392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789298576574095:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:39.691416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:39.691437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789297645507906:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:39.691439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789295874391814:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.691482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789297645507940:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.691680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789297645507845:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:39.691737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789295874391723:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.691794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.691904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789295874391723:2049] [node 2] ICP22 created new session: [5:7577789295874391824:2049]
2025-11-28T14:52:39.691939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789295874391824:2049] [node 2] ICS09 handshake done sender: [5:7577789295874391814:2120] self: [5:7577789295874391822:0] peer: [2:7577789298576574066:0] socket: 79 qp: -1
2025-11-28T14:52:39.691973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789295874391824:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:39.692402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789298576574072:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:39.692006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:39.692433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789298576574072:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:39.693451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789295874391723:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:39.693640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789298576574072:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:39.693633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789297645507946:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41976
2025-11-28T14:52:39.693940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789298576574059:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:39.694002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:39.694061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789298576574059:2109] [node 6] ICP22 created new session: [2:7577789298576574098:2109]
2025-11-28T14:52:39.694106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789298576574098:2109] [node 6] ICS09 handshake done sender: [2:7577789298576574072:2117] self: [2:7577789298576574071:0] peer: [6:7577789297645507945:0] socket: 44 qp: -1
2025-11-28T14:52:39.694120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789297645507940:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:39.694143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789298576574098:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:39.694182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:39.694272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789297645507845:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:39.694320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:39.694442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789297645507845:2049] [node 2] ICP22 created new session: [6:7577789297645507947:2049]
2025-11-28T14:52:39.694511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789297645507947:2049] [node 2] ICS09 handshake done sender: [6:7577789297645507940:2123] self: [6:7577789297645507945:0] peer: [2:7577789298576574071:0] socket: 48 qp: -1
2025-11-28T14:52:39.694577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789297645507947:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:39.694626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:39.735959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:39.736210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789298576574089:2048] [node 1] ICS07 socket disconnect 72 reason# EndOfStream
2025-11-28T14:52:39.736281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789298576574089:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.736461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.736524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789298576574089:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.736992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:39.737246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789298681446080:2048] [node 1] ICS07 socket disconnect 68 reason# EndOfStream
2025-11-28T14:52:39.737288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789298681446080:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.737376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.737437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789298681446080:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.738180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789298681445988:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.738226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789298681446080:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.738452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789298576573994:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:39.738411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789298681445988:2048] [node 1] ICP30 unregister session Session# [3:7577789298681446080:2048] VirtualId# [3:7577789298681446074:0]
2025-11-28T14:52:39.738503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789298576574089:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.738787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.738820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789298681445988:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.738833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.739560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:39.739713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789298388813167:2048] [node 1] ICS01 socket: 73 reason# EndOfStream
2025-11-28T14:52:39.739764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789298388813064:2048] [node 1] ICP30 unregister session Session# [4:7577789298388813167:2048] VirtualId# [4:7577789298388813161:0]
2025-11-28T14:52:39.739785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789298388813167:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.739844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.739889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.740022Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789298576573994:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.740063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789298576574089:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.740125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789298576573994:2048] [node 1] ICP30 unregister session Session# [2:7577789298576574089:2048] VirtualId# [2:7577789298576574082:0]
2025-11-28T14:52:39.740229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.740256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789298576573994:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.740264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.740320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:39.740476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789295874391803:2048] [node 1] ICS07 socket disconnect 74 reason# EndOfStream
2025-11-28T14:52:39.740531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789295874391803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.740607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.740642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789295874391803:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:39.741181Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:39.741183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789295874391722:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.741210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789295874391803:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.741453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789297645507936:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-11-28T14:52:39.741245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789295874391722:2048] [node 1] ICP30 unregister session Session# [5:7577789295874391803:2048] VirtualId# [5:7577789295874391801:0]
2025-11-28T14:52:39.741414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.741522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789297645507844:2048] [node 1] ICP30 unregister session Session# [6:7577789297645507936:2048] VirtualId# [6:7577789297645507931:0]
2025-11-28T14:52:39.741431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789295874391722:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.741568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789297645507936:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.741653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:39.741439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26977) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.741712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:39.748246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789298681445988:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:39.751296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789295874391722:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:39.789083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:39.789314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789298681446093:2049] [node 2] ICS07 socket disconnect 83 reason# EndOfStream
2025-11-28T14:52:39.789358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789298681446093:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.789477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:39.789542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789298681446093:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:39.790212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789298681445989:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.790293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789298681446093:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.790318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789298681445989:2049] [node 2] ICP30 unregister session Session# [3:7577789298681446093:2049] VirtualId# [3:7577789298681446083:0]
2025-11-28T14:52:39.790530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:39.790570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789298681445989:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.790580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.791136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:39.791281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789298388813159:2049] [node 2] ICS01 socket: 81 reason# EndOfStream
2025-11-28T14:52:39.791310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789298388813065:2049] [node 2] ICP30 unregister session Session# [4:7577789298388813159:2049] VirtualId# [4:7577789298388813153:0]
2025-11-28T14:52:39.791334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789298388813159:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.791425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:39.791468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:39.791793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:39.791925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789295874391824:2049] [node 2] ICS07 socket disconnect 79 reason# EndOfStream
2025-11-28T14:52:39.791948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789295874391824:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.792020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:39.792068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789295874391824:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:39.792447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:39.792643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789295874391723:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.792665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789297645507947:2049] [node 2] ICS07 socket disconnect 48 reason# EndOfStream
2025-11-28T14:52:39.792707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789297645507947:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.792780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:39.792844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789297645507947:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:39.792718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789295874391824:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.792908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789295874391723:2049] [node 2] ICP30 unregister session Session# [5:7577789295874391824:2049] VirtualId# [5:7577789295874391822:0]
2025-11-28T14:52:39.792991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:39.793036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789295874391723:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.793045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.793502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789297645507845:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.793547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789297645507947:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.793596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789297645507845:2049] [node 2] ICP30 unregister session Session# [6:7577789297645507947:2049] VirtualId# [6:7577789297645507945:0]
2025-11-28T14:52:39.793633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:39.793659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789297645507845:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.793672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.802530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789295874391723:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:39.803533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789297645507845:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:39.829310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:39.829477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789298388813148:2050] [node 3] ICS07 socket disconnect 70 reason# EndOfStream
2025-11-28T14:52:39.829507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789298388813148:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.829590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:39.829639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789298388813148:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:39.830030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:39.830159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789295874391816:2050] [node 3] ICS07 socket disconnect 102 reason# EndOfStream
2025-11-28T14:52:39.830183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789295874391816:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.830250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:39.830282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789298388813066:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.830365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789298388813148:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.830283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789295874391816:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:39.830397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789298388813066:2050] [node 3] ICP30 unregister session Session# [4:7577789298388813148:2050] VirtualId# [4:7577789298388813146:0]
2025-11-28T14:52:39.830625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:39.830645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789298388813066:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.830653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.830938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:39.831103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789297645507928:2050] [node 3] ICS01 socket: 90 reason# EndOfStream
2025-11-28T14:52:39.831148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789297645507846:2050] [node 3] ICP30 unregister session Session# [6:7577789297645507928:2050] VirtualId# [6:7577789297645507924:0]
2025-11-28T14:52:39.831178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789297645507928:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.831245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:39.831119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789295874391724:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.831284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:39.831161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789295874391816:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.831181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789295874391724:2050] [node 3] ICP30 unregister session Session# [5:7577789295874391816:2050] VirtualId# [5:7577789295874391813:0]
2025-11-28T14:52:39.831352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:39.831375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789295874391724:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.831383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.840450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789298388813066:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:39.841417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789295874391724:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:39.888586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:39.888788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789295874391809:2108] [node 4] ICS07 socket disconnect 55 reason# EndOfStream
2025-11-28T14:52:39.888814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789295874391809:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.888900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:39.888946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789295874391809:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:39.889633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789295874391785:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64502) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.889693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:39.889852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789297645507941:2105] [node 4] ICS01 socket: 101 reason# EndOfStream
2025-11-28T14:52:39.889671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789295874391809:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.889885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789297645507905:2105] [node 4] ICP30 unregister session Session# [6:7577789297645507941:2105] VirtualId# [6:7577789297645507935:0]
2025-11-28T14:52:39.889912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789297645507941:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.889695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789295874391785:2108] [node 4] ICP30 unregister session Session# [5:7577789295874391809:2108] VirtualId# [5:7577789295874391807:0]
2025-11-28T14:52:39.889783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:39.889980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:39.889807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789295874391785:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64502) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.890020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:39.889816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64502) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.928581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:39.928828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789297645507943:2106] [node 5] ICS07 socket disconnect 49 reason# EndOfStream
2025-11-28T14:52:39.928871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789297645507943:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:39.928978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:39.929031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789297645507943:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:39.929716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789297645507906:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4651) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:39.929830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789297645507943:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:39.929874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789297645507906:2106] [node 5] ICP30 unregister session Session# [6:7577789297645507943:2106] VirtualId# [6:7577789297645507933:0]
2025-11-28T14:52:39.929964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:39.929994Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789297645507906:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4651) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:39.930004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4651) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:39.981251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789295596382507:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:39.981482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789295596382508:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:39.997279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789297618829481:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:39.997324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789297618829482:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:40.013781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789300281853550:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:40.013808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789300281853551:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:40.030731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789300134790793:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:40.030783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789300134790795:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:40.030803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789300134790794:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:40.053345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789302329364761:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:40.053388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789302329364762:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:40.053626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789302329364763:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:40.076243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789300240231045:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:40.076338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789300240231047:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:40.076356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789300240231046:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:40.149720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789300240231106:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:40.154149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789299891349867:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:40.154834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789299891349866:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:40.154860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789300240231108:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:40.154713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789300134790857:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:40.155172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789302329364824:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:40.154883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789300281853613:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:40.156421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789302329364825:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:40.154942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789300134790856:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:40.155363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789301913796841:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:40.156790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789299891349868:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:40.156671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789301913796842:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:40.156936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789300281853614:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:40.157034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789300281853615:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:40.157868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789301913796843:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.168655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789302329364832:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.169340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789302329364832:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.169562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789302329364840:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.170950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789302329364841:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-28T14:52:40.171004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789300281853625:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51308
2025-11-28T14:52:40.170996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789301913796850:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29781
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.174129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789299891349875:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.174650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789302329364840:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.174836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789301913796850:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.174975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789301913796852:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29781
2025-11-28T14:52:40.174983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789302329364824:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.175030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789301913796854:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29781
2025-11-28T14:52:40.175473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789300134790863:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.175353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789301913796855:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29781
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.175511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789300134790863:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.175569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789301913796854:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.176315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349875:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.177315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789301913796852:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.177158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789299891349876:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-28T14:52:40.177552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789300134790863:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:40.177367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789300281853625:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.177916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789302329364844:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-28T14:52:40.177317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789299891349877:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.178023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789300281853630:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51308
2025-11-28T14:52:40.177366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789299891349878:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-28T14:52:40.178392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349876:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:40.178679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789302329364841:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:40.178682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789300134790856:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.178679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349877:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.179713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789302329364763:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.178737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:40.179137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789299891349886:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6496
2025-11-28T14:52:40.180175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789300281853619:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.178780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789300134790856:2108] [node 5] ICP22 created new session: [4:7577789300134790872:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.180377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789302329364845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-28T14:52:40.179190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349878:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.179455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789300134790872:2108] [node 5] ICS09 handshake done sender: [4:7577789300134790863:2112] self: [4:7577789300134790859:0] peer: [5:7577789302329364843:0] socket: 60 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.180209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789300281853619:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.181184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789301913796863:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29781
2025-11-28T14:52:40.181707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789302329364846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-28T14:52:40.179757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789300134790872:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:40.181551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349886:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.181756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789300281853619:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:40.181446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789301913796855:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.182021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789302329364844:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.179809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:40.181849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789300281853631:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51308
2025-11-28T14:52:40.182154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789302329364762:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.182142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789300281853630:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.182246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789300281853551:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.182460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789302329364840:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.183026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789301913796848:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.182280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789300281853633:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51308
2025-11-28T14:52:40.183853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789302329364848:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-28T14:52:40.182782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789300281853613:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.183893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789302329364824:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.183051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789301913796848:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.183927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:40.182837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:40.183052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789301913796849:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.184965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789302329364824:2108] [node 4] ICP22 created new session: [5:7577789302329364850:2108]
2025-11-28T14:52:40.183064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789300281853633:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.183082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789301913796849:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.183898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789300281853613:2108] [node 5] ICP22 created new session: [3:7577789300281853637:2108]
2025-11-28T14:52:40.185249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789302329364850:2108] [node 4] ICS09 handshake done sender: [5:7577789302329364840:2116] self: [5:7577789302329364843:0] peer: [4:7577789300134790859:0] socket: 61 qp: -1
2025-11-28T14:52:40.185598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789301913796849:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:40.185137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789300281853637:2108] [node 5] ICS09 handshake done sender: [3:7577789300281853619:2110] self: [3:7577789300281853617:0] peer: [5:7577789302329364847:0] socket: 63 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.185273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789302329364850:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:40.185613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789301913796848:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.185310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.185179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789300281853637:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:40.186103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789301913796863:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.185435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:40.186318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789297618829481:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.186463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231123:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
2025-11-28T14:52:40.186406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789301913796841:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.186440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:40.186621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789300281853631:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.186475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789301913796841:2108] [node 5] ICP22 created new session: [2:7577789301913796869:2108]
2025-11-28T14:52:40.186680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789302329364841:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.186611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789297618829482:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.187054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789300281853639:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51308
2025-11-28T14:52:40.187127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789300281853550:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.187554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789300240231123:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.186619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789301913796869:2108] [node 5] ICS09 handshake done sender: [2:7577789301913796849:2113] self: [2:7577789301913796845:0] peer: [5:7577789302329364849:0] socket: 79 qp: -1
2025-11-28T14:52:40.187629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789302329364763:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27848) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789302329364841:2117] held: no
2025-11-28T14:52:40.187615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349873:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.186641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789301913796869:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:40.187662Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789299891349873:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.187902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789300240231047:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.186657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:40.188141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789300281853630:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:40.186673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:40.188228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349874:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.187012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789297618829482:2049] [node 3] ICP22 created new session: [2:7577789301913796870:2049]
2025-11-28T14:52:40.188233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789300281853627:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.188269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789299891349874:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.187092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789301913796870:2049] [node 3] ICS09 handshake done sender: [2:7577789301913796848:2112] self: [2:7577789301913796847:0] peer: [3:7577789300281853635:0] socket: 76 qp: -1
2025-11-28T14:52:40.188259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789300281853627:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.187119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789301913796870:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:40.187249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.189995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789299891349874:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.190012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789299891349873:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:40.189878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789300281853551:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.189919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:40.190853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789295596382508:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.191106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:40.189953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789300281853551:2049] [node 2] ICP22 created new session: [3:7577789300281853643:2049]
2025-11-28T14:52:40.190869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789302329364763:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.191144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789295596382508:2049] [node 3] ICP22 created new session: [1:7577789299891349894:2049]
2025-11-28T14:52:40.190877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789302329364845:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:40.191231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: 2025-11-28T14:52:40.190160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789300281853643:2049] [node 2] ICS09 handshake done sender: [3:7577789300281853630:2116] self: [3:7577789300281853635:0] peer: [2:7577789301913796847:0] socket: 77 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:40.191185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789299891349894:2049] [node 3] ICS09 handshake done sender: [1:7577789299891349874:2112] self: [1:7577789299891349872:0] peer: [3:7577789300281853641:0] socket: 87 qp: -1
2025-11-28T14:52:40.191384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789302329364763:2050] [node 3] ICP22 created new session: [5:7577789302329364853:2050]
2025-11-28T14:52:40.190204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789300281853643:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:40.191908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789302329364853:2050] [node 3] ICS09 handshake done sender: [5:7577789302329364841:2117] self: [5:7577789302329364847:0] peer: [3:7577789300281853617:0] socket: 66 qp: -1
2025-11-28T14:52:40.192256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789302329364853:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:40.191201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789295596382507:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.190308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.192399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789301913796875:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29781
2025-11-28T14:52:40.192284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.192355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789300134790875:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43249
2025-11-28T14:52:40.191362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789299891349894:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:40.191832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.191852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:40.192647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231125:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
2025-11-28T14:52:40.191880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789295596382507:2048] [node 2] ICP22 created new session: [1:7577789299891349896:2048]
2025-11-28T14:52:40.192325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789299891349896:2048] [node 2] ICS09 handshake done sender: [1:7577789299891349873:2111] self: [1:7577789299891349869:0] peer: [2:7577789301913796868:0] socket: 84 qp: -1
2025-11-28T14:52:40.192385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789299891349896:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:40.192608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.193220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789302329364762:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789302329364844:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.193305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789300281853627:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:40.193395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789302329364852:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.193738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231126:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
2025-11-28T14:52:40.195278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789301913796863:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:40.195426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789300281853615:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.195441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789300240231123:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.196403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789297618829481:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24601) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789301913796863:2120] held: no
2025-11-28T14:52:40.195460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.195613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789300281853615:2109] [node 6] ICP22 created new session: [3:7577789300281853647:2109]
2025-11-28T14:52:40.196834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789300134790875:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.196796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789300240231047:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.196148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789300281853551:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:40.197047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:40.197182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789300240231047:2050] [node 3] ICP22 created new session: [6:7577789300240231129:2050]
2025-11-28T14:52:40.196356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789300281853647:2109] [node 6] ICS09 handshake done sender: [3:7577789300281853627:2114] self: [3:7577789300281853626:0] peer: [6:7577789300240231124:0] socket: 96 qp: -1
2025-11-28T14:52:40.197087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789300134790876:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43249
2025-11-28T14:52:40.197213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789300240231129:2050] [node 3] ICS09 handshake done sender: [6:7577789300240231123:2115] self: [6:7577789300240231124:0] peer: [3:7577789300281853626:0] socket: 97 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.197585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789302329364761:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.196460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789300281853550:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24601) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789300281853631:2117] held: no
No RDMA context found for GID: 2025-11-28T14:52:40.197458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789300240231129:2050] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:40.197201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789300134790794:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.197550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789301913796860:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.197875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789302329364844:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:40.197532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.196483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789300281853647:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:40.197805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789300240231125:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.197578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789301913796860:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.196627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.198089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231128:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
2025-11-28T14:52:40.198184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231127:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
2025-11-28T14:52:40.198794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789300240231046:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.198920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789297618829481:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.199033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789302329364762:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.198965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:40.199273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:40.199013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789297618829481:2048] [node 1] ICP22 created new session: [2:7577789301913796877:2048]
2025-11-28T14:52:40.199300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789301913796877:2048] [node 1] ICS09 handshake done sender: [2:7577789301913796863:2120] self: [2:7577789301913796868:0] peer: [1:7577789299891349869:0] socket: 86 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.199625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789300240231127:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.199399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789302329364762:2049] [node 2] ICP22 created new session: [5:7577789302329364857:2049]
2025-11-28T14:52:40.199329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789301913796877:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.199945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789302329364857:2049] [node 2] ICS09 handshake done sender: [5:7577789302329364844:2118] self: [5:7577789302329364849:0] peer: [2:7577789301913796845:0] socket: 80 qp: -1
2025-11-28T14:52:40.199973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789302329364857:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:40.200788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789300240231045:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.200761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.200928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349871:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.201127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789300134790878:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43249
No RDMA context found for GID: 2025-11-28T14:52:40.199357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:40.200965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789299891349871:2110] [node 5] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.199374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789301913796853:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.201920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349881:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.202206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789300134790880:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43249
2025-11-28T14:52:40.202716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789300240231128:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.201939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789299891349881:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.199390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789301913796853:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.202802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789300281853645:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51308
2025-11-28T14:52:40.201887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789301913796860:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.202799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789300134790879:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43249
2025-11-28T14:52:40.202828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789299891349871:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:40.203154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789300240231106:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.202964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789300134790876:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.203334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789299891349881:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:40.203645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789300281853631:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:40.203426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789299891349867:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.203264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789300134790793:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.204782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789300281853550:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.204757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789301913796842:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.203839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:40.204827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:40.203439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789300134790875:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.204153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789302329364835:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.204009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789299891349867:2108] [node 5] ICP22 created new session: [1:7577789299891349902:2108]
2025-11-28T14:52:40.204878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789300281853550:2048] [node 1] ICP22 created new session: [3:7577789300281853649:2048]
2025-11-28T14:52:40.204802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:40.204156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789299891349902:2108] [node 5] ICS09 handshake done sender: [1:7577789299891349871:2110] self: [1:7577789299891349870:0] peer: [5:7577789302329364856:0] socket: 82 qp: -1
2025-11-28T14:52:40.203980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789300134790878:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.204832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789301913796842:2107] [node 4] ICP22 created new session: [2:7577789301913796880:2107]
2025-11-28T14:52:40.204896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789299891349902:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:40.204178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789302329364835:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.204142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789300134790794:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.205224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789302329364859:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-28T14:52:40.204866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789301913796880:2107] [node 4] ICS09 handshake done sender: [2:7577789301913796860:2119] self: [2:7577789301913796858:0] peer: [4:7577789300134790877:0] socket: 103 qp: -1
2025-11-28T14:52:40.204926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:40.205373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789300281853649:2048] [node 1] ICS09 handshake done sender: [3:7577789300281853631:2117] self: [3:7577789300281853641:0] peer: [1:7577789299891349872:0] socket: 90 qp: -1
2025-11-28T14:52:40.204902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:40.205578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231135:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
2025-11-28T14:52:40.205187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789299891349882:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.205052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789300134790794:2049] [node 2] ICP22 created new session: [4:7577789300134790882:2049]
2025-11-28T14:52:40.205215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789299891349882:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.205239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789300134790882:2049] [node 2] ICS09 handshake done sender: [4:7577789300134790875:2115] self: [4:7577789300134790877:0] peer: [2:7577789301913796858:0] socket: 104 qp: -1
2025-11-28T14:52:40.204888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789301913796880:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:40.205690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789300281853649:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.205505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789299891349868:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.205551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789300134790882:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:40.206376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:40.205693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.207327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789299891349868:2109] [node 6] ICP22 created new session: [1:7577789299891349904:2109]
2025-11-28T14:52:40.206383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.206401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:40.207789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789302329364761:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24601) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789302329364845:2119] held: no
2025-11-28T14:52:40.207792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789299891349904:2109] [node 6] ICS09 handshake done sender: [1:7577789299891349881:2117] self: [1:7577789299891349880:0] peer: [6:7577789300240231133:0] socket: 81 qp: -1
2025-11-28T14:52:40.208438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789300134790880:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.208197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789299891349904:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:40.208317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789300240231127:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:40.208311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789301913796853:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:40.208246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.209093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789300134790795:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.210801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789300281853629:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.210828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789300281853629:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.210760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789302329364835:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:40.211744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789300240231045:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.212046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789300281853629:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.211655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789299891349882:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.211947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.212268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789302329364845:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:40.212008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789300240231045:2048] [node 1] ICP22 created new session: [6:7577789300240231139:2048]
2025-11-28T14:52:40.212632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789300281853614:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.212748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789302329364825:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.212680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-28T14:52:40.212619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789301913796843:2109] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.214270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789300281853614:2107] [node 4] ICP22 created new session: [3:7577789300281853652:2107]
2025-11-28T14:52:40.212682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:40.212442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789300240231139:2048] [node 1] ICS09 handshake done sender: [6:7577789300240231127:2118] self: [6:7577789300240231133:0] peer: [1:7577789299891349880:0] socket: 105 qp: -1
2025-11-28T14:52:40.214174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789300134790887:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43249
2025-11-28T14:52:40.214341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789300281853652:2107] [node 4] ICS09 handshake done sender: [3:7577789300281853629:2115] self: [3:7577789300281853628:0] peer: [4:7577789300134790886:0] socket: 111 qp: -1
2025-11-28T14:52:40.214228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:40.214506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789300281853652:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:40.214201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789301913796843:2109] [node 6] ICP22 created new session: [2:7577789301913796882:2109]
2025-11-28T14:52:40.212600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789300240231139:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.214534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789302329364825:2109] [node 6] ICP22 created new session: [5:7577789302329364861:2109]
2025-11-28T14:52:40.214980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789300134790888:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43249
2025-11-28T14:52:40.214996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789302329364861:2109] [node 6] ICS09 handshake done sender: [5:7577789302329364835:2114] self: [5:7577789302329364833:0] peer: [6:7577789300240231134:0] socket: 106 qp: -1
2025-11-28T14:52:40.214500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789301913796882:2109] [node 6] ICS09 handshake done sender: [2:7577789301913796853:2116] self: [2:7577789301913796851:0] peer: [6:7577789300240231132:0] socket: 71 qp: -1
2025-11-28T14:52:40.214336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:40.215404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.215046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789302329364861:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:40.214661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789301913796882:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:40.215193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789300134790876:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:40.215427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789299891349866:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.215089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.214659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231136:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
2025-11-28T14:52:40.214826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.215495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:40.215550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789302329364761:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.216175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789299891349866:2107] [node 4] ICP22 created new session: [1:7577789299891349907:2107]
2025-11-28T14:52:40.216215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:40.216458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789300134790793:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.216637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789299891349907:2107] [node 4] ICS09 handshake done sender: [1:7577789299891349882:2118] self: [1:7577789299891349879:0] peer: [4:7577789300134790881:0] socket: 69 qp: -1
2025-11-28T14:52:40.216672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789302329364761:2048] [node 1] ICP22 created new session: [5:7577789302329364863:2048]
2025-11-28T14:52:40.216686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789299891349907:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:40.216865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789302329364863:2048] [node 1] ICS09 handshake done sender: [5:7577789302329364845:2119] self: [5:7577789302329364856:0] peer: [1:7577789299891349870:0] socket: 85 qp: -1
2025-11-28T14:52:40.217029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789300134790880:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.216727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.217316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:40.217174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789302329364863:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.217383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789300134790793:2048] [node 1] ICP22 created new session: [4:7577789300134790889:2048]
2025-11-28T14:52:40.217323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:40.217630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789300134790889:2048] [node 1] ICS09 handshake done sender: [4:7577789300134790876:2116] self: [4:7577789300134790881:0] peer: [1:7577789299891349879:0] socket: 107 qp: -1
2025-11-28T14:52:40.217670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789300134790889:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.217703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:40.219382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789300240231046:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789300240231125:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.222256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231137:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.222888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789300134790795:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.223289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:40.223268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231138:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
2025-11-28T14:52:40.223529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789300240231125:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:40.223411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789300134790795:2050] [node 3] ICP22 created new session: [4:7577789300134790891:2050]
2025-11-28T14:52:40.223548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789300134790891:2050] [node 3] ICS09 handshake done sender: [4:7577789300134790880:2119] self: [4:7577789300134790886:0] peer: [3:7577789300281853628:0] socket: 113 qp: -1
2025-11-28T14:52:40.223588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789300134790891:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:40.223629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.223802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789300240231106:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23429) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789300240231128:2119] held: no
2025-11-28T14:52:40.224355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789300240231045:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:40.224583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789300240231046:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.224656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:40.224694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789300240231046:2049] [node 2] ICP22 created new session: [6:7577789300240231141:2049]
2025-11-28T14:52:40.224740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789300240231141:2049] [node 2] ICS09 handshake done sender: [6:7577789300240231125:2116] self: [6:7577789300240231132:0] peer: [2:7577789301913796851:0] socket: 102 qp: -1
2025-11-28T14:52:40.224777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789300240231141:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:40.224852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.224932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789300134790795:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:40.225071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789300240231128:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:40.225960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789300240231106:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.226029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:40.226067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789300240231106:2106] [node 5] ICP22 created new session: [6:7577789300240231143:2106]
2025-11-28T14:52:40.226109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789300240231143:2106] [node 5] ICS09 handshake done sender: [6:7577789300240231128:2119] self: [6:7577789300240231134:0] peer: [5:7577789302329364833:0] socket: 108 qp: -1
2025-11-28T14:52:40.226135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789300240231143:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:40.226168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:40.226339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789300240231138:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.226502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789300240231108:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.226801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789300134790867:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.226835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789300134790867:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.228231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789300134790867:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.228396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789300240231146:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48233
2025-11-28T14:52:40.228458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789300134790857:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.228533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:40.228608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789300134790857:2109] [node 6] ICP22 created new session: [4:7577789300134790894:2109]
2025-11-28T14:52:40.228664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789300134790894:2109] [node 6] ICS09 handshake done sender: [4:7577789300134790867:2114] self: [4:7577789300134790866:0] peer: [6:7577789300240231145:0] socket: 70 qp: -1
2025-11-28T14:52:40.228691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789300134790894:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:40.228730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.228676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789300240231138:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.228805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789300240231108:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.228841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:40.228995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789300240231108:2108] [node 4] ICP22 created new session: [6:7577789300240231147:2108]
2025-11-28T14:52:40.229022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789300240231147:2108] [node 4] ICS09 handshake done sender: [6:7577789300240231138:2123] self: [6:7577789300240231145:0] peer: [4:7577789300134790866:0] socket: 54 qp: -1
2025-11-28T14:52:40.229160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789300240231147:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:40.229185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.229820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789300240231108:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:40.280960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:40.281138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789301913796877:2048] [node 1] ICS01 socket: 86 reason# EndOfStream
2025-11-28T14:52:40.281179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789297618829481:2048] [node 1] ICP30 unregister session Session# [2:7577789301913796877:2048] VirtualId# [2:7577789301913796868:0]
2025-11-28T14:52:40.281202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789301913796877:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.281287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.281358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.307882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:40.308040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789300281853649:2048] [node 1] ICS01 socket: 90 reason# EndOfStream
2025-11-28T14:52:40.308077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789300281853550:2048] [node 1] ICP30 unregister session Session# [3:7577789300281853649:2048] VirtualId# [3:7577789300281853641:0]
2025-11-28T14:52:40.308100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789300281853649:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.308170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.308214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.310343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:40.310490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789300134790889:2048] [node 1] ICS07 socket disconnect 107 reason# EndOfStream
2025-11-28T14:52:40.310513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789300134790889:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.310570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.310601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789300134790889:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:40.311302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789300134790793:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24601) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:40.311340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789300134790889:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:40.311313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:40.311363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789300134790793:2048] [node 1] ICP30 unregister session Session# [4:7577789300134790889:2048] VirtualId# [4:7577789300134790881:0]
2025-11-28T14:52:40.311528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.311558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789300134790793:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24601) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.311566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24601) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.311602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789302329364863:2048] [node 1] ICS01 socket: 85 reason# EndOfStream
2025-11-28T14:52:40.311653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789302329364761:2048] [node 1] ICP30 unregister session Session# [5:7577789302329364863:2048] VirtualId# [5:7577789302329364856:0]
2025-11-28T14:52:40.311711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789302329364863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.311824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.311890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.312384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:40.312556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789300240231139:2048] [node 1] ICS07 socket disconnect 105 reason# EndOfStream
2025-11-28T14:52:40.312610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789300240231139:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.312691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.312728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789300240231139:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:40.313480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789300240231045:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24601) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:40.313517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789300240231139:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:40.313538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789300240231045:2048] [node 1] ICP30 unregister session Session# [6:7577789300240231139:2048] VirtualId# [6:7577789300240231133:0]
2025-11-28T14:52:40.313675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.313703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789300240231045:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24601) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.313712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24601) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.321676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789300134790793:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:40.323737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789300240231045:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:40.370751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:40.370941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789300281853643:2049] [node 2] ICS07 socket disconnect 77 reason# EndOfStream
2025-11-28T14:52:40.370965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789300281853643:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.371034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:40.371070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789300281853643:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:40.373089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:40.373233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789300281853551:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21876) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:40.373249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789300134790882:2049] [node 2] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-28T14:52:40.373273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789300281853643:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:40.373271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789300134790882:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.373528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:40.373562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789300134790882:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:40.374436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789300281853551:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21876) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:40.374482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:40.374525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789300281853643:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:40.374466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789300134790794:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21876) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:40.374688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789300134790882:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:40.374769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789300134790794:2049] [node 2] ICP30 unregister session Session# [4:7577789300134790882:2049] VirtualId# [4:7577789300134790877:0]
2025-11-28T14:52:40.374703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789302329364857:2049] [node 2] ICS01 socket: 80 reason# EndOfStream
2025-11-28T14:52:40.374719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789300281853551:2049] [node 2] ICP30 unregister session Session# [3:7577789300281853643:2049] VirtualId# [3:7577789300281853635:0]
2025-11-28T14:52:40.374882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789302329364762:2049] [node 2] ICP30 unregister session Session# [5:7577789302329364857:2049] VirtualId# [5:7577789302329364849:0]
2025-11-28T14:52:40.374911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:40.374939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789302329364857:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.374942Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789300134790794:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.374943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:40.374951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.374992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789300281853551:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.375005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.375022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:40.375070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:40.375920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:40.376117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789300240231141:2049] [node 2] ICS01 socket: 102 reason# EndOfStream
2025-11-28T14:52:40.376160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789300240231046:2049] [node 2] ICP30 unregister session Session# [6:7577789300240231141:2049] VirtualId# [6:7577789300240231132:0]
2025-11-28T14:52:40.376191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789300240231141:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.376260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:40.376312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:40.385194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789300134790794:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:40.385219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789300281853551:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:40.437721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:40.437883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789300134790891:2050] [node 3] ICS07 socket disconnect 113 reason# EndOfStream
2025-11-28T14:52:40.437924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789300134790891:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.438000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:40.438049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789300134790891:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:40.438610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:40.438680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789300134790795:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27848) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:40.438725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789300134790891:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:40.438749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789300134790795:2050] [node 3] ICP30 unregister session Session# [4:7577789300134790891:2050] VirtualId# [4:7577789300134790886:0]
2025-11-28T14:52:40.438816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789302329364853:2050] [node 3] ICS01 socket: 66 reason# EndOfStream
2025-11-28T14:52:40.438970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:40.438997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789302329364763:2050] [node 3] ICP30 unregister session Session# [5:7577789302329364853:2050] VirtualId# [5:7577789302329364847:0]
2025-11-28T14:52:40.438998Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789300134790795:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27848) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.439008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27848) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.439029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789302329364853:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.439107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:40.439156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:40.439286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:40.439463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789300240231129:2050] [node 3] ICS07 socket disconnect 97 reason# EndOfStream
2025-11-28T14:52:40.439492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789300240231129:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.439571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:40.439637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789300240231129:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:40.440515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789300240231047:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27848) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:40.440551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789300240231129:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:40.440580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789300240231047:2050] [node 3] ICP30 unregister session Session# [6:7577789300240231129:2050] VirtualId# [6:7577789300240231124:0]
2025-11-28T14:52:40.440695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:40.440734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789300240231047:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27848) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.440747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27848) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.450701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789300240231047:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:40.458335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:40.458549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789302329364850:2108] [node 4] ICS07 socket disconnect 61 reason# EndOfStream
2025-11-28T14:52:40.458581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789302329364850:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.458674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:40.458723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789302329364850:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:40.459071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:40.459199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789300240231147:2108] [node 4] ICS07 socket disconnect 54 reason# EndOfStream
2025-11-28T14:52:40.459218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789300240231147:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.459266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:40.459293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789300240231147:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:40.459428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789302329364824:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61864) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:40.459512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789302329364850:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:40.459546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789302329364824:2108] [node 4] ICP30 unregister session Session# [5:7577789302329364850:2108] VirtualId# [5:7577789302329364843:0]
2025-11-28T14:52:40.459634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:40.459687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789302329364824:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.459702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.460133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789300240231108:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61864) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:40.460163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789300240231147:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:40.460188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789300240231108:2108] [node 4] ICP30 unregister session Session# [6:7577789300240231147:2108] VirtualId# [6:7577789300240231145:0]
2025-11-28T14:52:40.460330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:40.460361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789300240231108:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.460389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:40.469151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789302329364824:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:40.470122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789300240231108:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:40.506560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:40.506719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789300240231143:2106] [node 5] ICS01 socket: 108 reason# EndOfStream
2025-11-28T14:52:40.506748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789300240231106:2106] [node 5] ICP30 unregister session Session# [6:7577789300240231143:2106] VirtualId# [6:7577789300240231134:0]
2025-11-28T14:52:40.506775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789300240231143:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.506856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:40.506903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:40.577097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789302581796344:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:40.577109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789302581796345:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:40.599892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789302441799580:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:40.600525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789302441799581:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:40.623520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789302076234600:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:40.623539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789302076234601:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:40.645367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789301617151591:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:40.645408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789301617151592:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:40.645431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789301617151593:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:40.669724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789300655743461:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:40.669768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789300655743463:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:40.669823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789300655743462:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:40.695836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789301455117454:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:40.695871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789301455117455:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:40.695912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789301455117456:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:40.772108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789302581796407:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:40.782531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789301455117513:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:40.785194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789301455117515:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:40.784518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789302581796408:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:40.785187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789302441799644:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:40.785157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789302076234664:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:40.785622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789302441799643:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:40.785691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789300655743524:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:40.785514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789302076234663:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:40.786083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789302441799645:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:40.786084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789301617151654:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:40.785554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789302076234665:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:40.786487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789301617151655:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:40.786741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789302581796409:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:40.787059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789300655743525:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.804746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789301617151667:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43823
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.805942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789300655743537:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39196
2025-11-28T14:52:40.806674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117530:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.806962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789302441799657:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42356
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.807238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117532:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:40.807478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789300655743537:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.807586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117531:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
2025-11-28T14:52:40.809007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789302581796422:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50271
2025-11-28T14:52:40.809256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789300655743525:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-28T14:52:40.809271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789302581796421:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50271
2025-11-28T14:52:40.808746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789301455117530:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.810227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789302441799659:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42356
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.810499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789300655743527:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.812269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789302076234678:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52751
2025-11-28T14:52:40.810345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789301617151661:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.809765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789302581796424:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50271
2025-11-28T14:52:40.809316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789301455117454:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.810332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789302441799658:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42356
2025-11-28T14:52:40.810532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789300655743527:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.810373Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789301617151661:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.812586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789302076234679:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52751
2025-11-28T14:52:40.810321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796422:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.809795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789301455117531:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.812617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789302441799660:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42356
2025-11-28T14:52:40.814091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789300655743527:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:40.812589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789302076234677:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52751
2025-11-28T14:52:40.810443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789301455117513:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.810338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796414:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.812682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789302441799661:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42356
2025-11-28T14:52:40.814249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789301617151667:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.810360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789302581796414:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.812612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789302076234680:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52751
2025-11-28T14:52:40.812670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789301455117532:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.813814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789302076234680:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.812834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789302441799659:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.810370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789302581796423:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50271
2025-11-28T14:52:40.816138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789301617151593:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.813791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789302441799661:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.813103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789301455117455:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.816435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789300655743524:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.810493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796424:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.814059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789302076234678:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.816392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789301617151661:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:40.813796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789302441799658:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:40.812677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796423:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.816629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:40.814270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789302076234677:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.814081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789302441799657:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.817355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789300655743524:2109] [node 6] ICP22 created new session: [5:7577789300655743543:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.814517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796421:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.814517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789302076234679:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.814275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789302441799651:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.819280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789301617151654:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.819054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117538:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
2025-11-28T14:52:40.815284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789302581796414:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:40.818849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789300655743543:2109] [node 6] ICS09 handshake done sender: [5:7577789300655743527:2110] self: [5:7577789300655743526:0] peer: [6:7577789301455117536:0] socket: 62 qp: -1
2025-11-28T14:52:40.814297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789302441799651:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.814881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789302076234600:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.819368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117539:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.815623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796413:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.818897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789300655743543:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.820585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:40.814849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789302441799660:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.820282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
2025-11-28T14:52:40.814881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789302076234601:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.819029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.815644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789302581796413:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.820853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789301617151654:2108] [node 5] ICP22 created new session: [4:7577789301617151673:2108]
2025-11-28T14:52:40.816426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789302076234671:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.815550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789302441799648:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.823368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789301455117454:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24516) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789301455117530:2114] held: no
2025-11-28T14:52:40.820262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789300655743542:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39196
2025-11-28T14:52:40.816443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789302076234671:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.818075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796415:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.815575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789302441799648:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.821307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789301617151673:2108] [node 5] ICS09 handshake done sender: [4:7577789301617151661:2112] self: [4:7577789301617151659:0] peer: [5:7577789300655743540:0] socket: 63 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.815575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789302441799580:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.824318Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789301455117455:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789301455117532:2116] held: no
2025-11-28T14:52:40.821420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789300655743537:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.818101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789302581796415:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.816392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789302441799651:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:40.819743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789302076234671:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.821834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789300655743525:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.827011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789301455117531:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:40.821416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789301617151673:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:40.819058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789302441799648:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.818122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789302581796408:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.822824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789302076234687:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52751
2025-11-28T14:52:40.823310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.818797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:40.819160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789302441799643:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.822881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.824254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789300655743525:2108] [node 4] ICP22 created new session: [5:7577789300655743546:2108]
2025-11-28T14:52:40.823261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789302076234690:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52751
2025-11-28T14:52:40.819389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:40.827803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789301455117532:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.827083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789301617151674:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43823
2025-11-28T14:52:40.818834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789302581796413:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.819729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789302441799643:2109] [node 6] ICP22 created new session: [2:7577789302441799671:2109]
2025-11-28T14:52:40.825991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789300655743546:2108] [node 4] ICS09 handshake done sender: [5:7577789300655743537:2114] self: [5:7577789300655743540:0] peer: [4:7577789301617151659:0] socket: 76 qp: -1
2025-11-28T14:52:40.818837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789302581796408:2109] [node 6] ICP22 created new session: [1:7577789302581796433:2109]
2025-11-28T14:52:40.824561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789302076234663:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.819991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789302441799581:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.829268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789301455117513:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.828438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789301617151676:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43823
2025-11-28T14:52:40.819391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789302581796433:2109] [node 6] ICS09 handshake done sender: [1:7577789302581796414:2111] self: [1:7577789302581796410:0] peer: [6:7577789301455117535:0] socket: 60 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.826028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789300655743546:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:40.825323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789302076234600:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24516) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789302076234679:2116] held: no
2025-11-28T14:52:40.820025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:40.828926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789301617151667:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.829318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:40.829724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789301617151676:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.820636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789302441799581:2049] [node 3] ICP22 created new session: [2:7577789302441799672:2049]
2025-11-28T14:52:40.826060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.819715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789302581796433:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.825576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:40.829773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789301617151593:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.821099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789302441799671:2109] [node 6] ICS09 handshake done sender: [2:7577789302441799651:2111] self: [2:7577789302441799646:0] peer: [6:7577789301455117537:0] socket: 67 qp: -1
2025-11-28T14:52:40.829368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789301455117513:2105] [node 5] ICP22 created new session: [6:7577789301455117541:2105]
2025-11-28T14:52:40.819746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.830224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789300655743548:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39196
2025-11-28T14:52:40.821278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789302441799672:2049] [node 3] ICS09 handshake done sender: [2:7577789302441799648:2110] self: [2:7577789302441799647:0] peer: [3:7577789302076234685:0] socket: 64 qp: -1
2025-11-28T14:52:40.826259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789302076234663:2107] [node 4] ICP22 created new session: [3:7577789302076234691:2107]
2025-11-28T14:52:40.820280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789302581796345:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.829701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789301455117541:2105] [node 5] ICS09 handshake done sender: [6:7577789301455117531:2115] self: [6:7577789301455117536:0] peer: [5:7577789300655743526:0] socket: 83 qp: -1
2025-11-28T14:52:40.821311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789302441799671:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:40.827050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789302076234691:2107] [node 4] ICS09 handshake done sender: [3:7577789302076234671:2111] self: [3:7577789302076234667:0] peer: [4:7577789301617151670:0] socket: 55 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.830063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789301455117541:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:40.830408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:40.831358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789301617151593:2050] [node 3] ICP22 created new session: [4:7577789301617151678:2050]
2025-11-28T14:52:40.821263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:40.821352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.830225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:40.821401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789302581796345:2049] [node 3] ICP22 created new session: [1:7577789302581796436:2049]
2025-11-28T14:52:40.827087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789302076234691:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:40.821580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789302441799672:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:40.830230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789301455117455:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.821742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789302581796436:2049] [node 3] ICS09 handshake done sender: [1:7577789302581796413:2110] self: [1:7577789302581796411:0] peer: [3:7577789302076234684:0] socket: 65 qp: -1
2025-11-28T14:52:40.831662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789301617151677:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43823
2025-11-28T14:52:40.821772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.831732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789301617151678:2050] [node 3] ICS09 handshake done sender: [4:7577789301617151667:2114] self: [4:7577789301617151670:0] peer: [3:7577789302076234667:0] socket: 69 qp: -1
2025-11-28T14:52:40.832884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789300655743549:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39196
2025-11-28T14:52:40.827122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789302076234688:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52751
2025-11-28T14:52:40.822841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789302581796436:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:40.832187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789301617151678:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:40.833479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789300655743548:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.831519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:40.829257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789302441799676:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42356
2025-11-28T14:52:40.827501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789302076234601:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789302076234678:2115] held: no
2025-11-28T14:52:40.831575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789301455117455:2049] [node 2] ICP22 created new session: [6:7577789301455117544:2049]
2025-11-28T14:52:40.824252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.827746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.831361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789302441799660:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:40.832214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789301617151592:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.834079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789300655743462:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.832298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.831638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789301455117544:2049] [node 2] ICS09 handshake done sender: [6:7577789301455117532:2116] self: [6:7577789301455117537:0] peer: [2:7577789302441799646:0] socket: 88 qp: -1
2025-11-28T14:52:40.829663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789302076234690:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.832853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789302441799580:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24516) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789302441799660:2117] held: no
2025-11-28T14:52:40.826532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789302581796415:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:40.831640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789301455117530:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:40.832241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789302076234679:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.836053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789302441799652:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.831677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789301455117544:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:40.827747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789302581796344:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.832919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789302076234678:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:40.837070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789301617151677:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.835057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789302076234600:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.836090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789302441799652:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.835229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789302076234601:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.831708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.827815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:40.835320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:40.837241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789302441799580:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.835353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789302076234600:2048] [node 1] ICP22 created new session: [3:7577789302076234696:2048]
2025-11-28T14:52:40.838178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789300655743551:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39196
2025-11-28T14:52:40.828059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789302581796344:2048] [node 2] ICP22 created new session: [1:7577789302581796441:2048]
2025-11-28T14:52:40.838482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789301617151681:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43823
2025-11-28T14:52:40.832819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789301455117454:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.837294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:40.835555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789302076234696:2048] [node 1] ICS09 handshake done sender: [3:7577789302076234679:2116] self: [3:7577789302076234684:0] peer: [1:7577789302581796411:0] socket: 90 qp: -1
2025-11-28T14:52:40.829088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789302581796441:2048] [node 2] ICS09 handshake done sender: [1:7577789302581796415:2112] self: [1:7577789302581796412:0] peer: [2:7577789302441799668:0] socket: 66 qp: -1
2025-11-28T14:52:40.838911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789301617151591:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.835606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789302076234696:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.837391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789302441799663:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.833401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:40.829331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789302581796441:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:40.837426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789302441799663:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.835611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: 2025-11-28T14:52:40.837522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789302441799580:2048] [node 1] ICP22 created new session: [2:7577789302441799678:2048]
::ffff:127.1.0.0
2025-11-28T14:52:40.835701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: 2025-11-28T14:52:40.838519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789302441799678:2048] [node 1] ICS09 handshake done sender: [2:7577789302441799660:2117] self: [2:7577789302441799668:0] peer: [1:7577789302581796412:0] socket: 93 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:40.836025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789302076234601:2049] [node 2] ICP22 created new session: [3:7577789302076234697:2049]
2025-11-28T14:52:40.833459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789301455117454:2048] [node 1] ICP22 created new session: [6:7577789301455117547:2048]
2025-11-28T14:52:40.829506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789302581796437:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50271
2025-11-28T14:52:40.836887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789302076234697:2049] [node 2] ICS09 handshake done sender: [3:7577789302076234678:2115] self: [3:7577789302076234685:0] peer: [2:7577789302441799647:0] socket: 86 qp: -1
2025-11-28T14:52:40.829520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.839002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789302441799678:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.837135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789302076234697:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:40.837171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.839515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789300655743524:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18356335ns) {HandleHandshakeStatus:329(18354512ns) {DropIncomingHandshake:442(825ns) DropOutgoingHandshake:457(1166ns) SwitchToState:179(263ns) ProcessPendingSessionEvents:495(13917027ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13267061ns max# 1276922ns}}}
2025-11-28T14:52:40.832850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796437:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.839038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:40.833494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789301455117547:2048] [node 1] ICS09 handshake done sender: [6:7577789301455117530:2114] self: [6:7577789301455117535:0] peer: [1:7577789302581796410:0] socket: 75 qp: -1
2025-11-28T14:52:40.833529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789301455117547:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.839683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789300655743551:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.833562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:40.839382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117550:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
2025-11-28T14:52:40.839759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796417:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.839580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117549:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
2025-11-28T14:52:40.839781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789301617151681:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.839793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789302581796417:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.839914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789302441799652:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.839829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789300655743463:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.840000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789302441799663:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:40.840745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789300655743549:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.841491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789302076234672:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.841213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789302441799643:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18026626ns) {HandleHandshakeStatus:329(18023514ns) {DropIncomingHandshake:442(11800ns) DropOutgoingHandshake:457(387ns) SwitchToState:179(204ns) ProcessPendingSessionEvents:495(11568330ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10935465ns max# 2171043ns}}}
2025-11-28T14:52:40.841534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789302076234672:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.841118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789300655743461:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.841422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789302441799645:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.841749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789302581796417:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.842460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789302581796426:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.841476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789302441799644:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.842489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789302581796426:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.841507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:40.843217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789302581796407:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.843617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789302581796426:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:40.841601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.844622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789300655743553:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39196
2025-11-28T14:52:40.841918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789302441799644:2107] [node 4] ICP22 created new session: [2:7577789302441799683:2107]
2025-11-28T14:52:40.844928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789301455117550:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.844680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:40.845300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789301455117549:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.842332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789302441799645:2108] [node 5] ICP22 created new session: [2:7577789302441799682:2108]
2025-11-28T14:52:40.845243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789301617151654:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(20097048ns) {HandleHandshakeStatus:329(20095400ns) {DropIncomingHandshake:442(202ns) DropOutgoingHandshake:457(202ns) SwitchToState:179(132ns) ProcessPendingSessionEvents:495(13388699ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12988271ns max# 818843ns}}}
2025-11-28T14:52:40.845236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789300655743462:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789300655743548:2117] held: no
2025-11-28T14:52:40.845625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789302076234672:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:40.844928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789302581796407:2107] [node 4] ICP22 created new session: [1:7577789302581796446:2107]
2025-11-28T14:52:40.845291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789301617151684:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43823
2025-11-28T14:52:40.843006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789302441799682:2108] [node 5] ICS09 handshake done sender: [2:7577789302441799663:2119] self: [2:7577789302441799662:0] peer: [5:7577789300655743550:0] socket: 115 qp: -1
2025-11-28T14:52:40.846183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789301455117515:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.845263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789302581796409:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.843144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789302441799682:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:40.846489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789301455117456:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:40.843235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789302441799683:2107] [node 4] ICS09 handshake done sender: [2:7577789302441799652:2112] self: [2:7577789302441799649:0] peer: [4:7577789301617151679:0] socket: 74 qp: -1
2025-11-28T14:52:40.846912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789302076234664:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.843273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789302441799683:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:40.847237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:40.847282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789301617151591:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24516) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789301617151677:2118] held: no
2025-11-28T14:52:40.843317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.845421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:40.845443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789302581796446:2107] [node 4] ICS09 handshake done sender: [1:7577789302581796417:2113] self: [1:7577789302581796416:0] peer: [4:7577789301617151683:0] socket: 71 qp: -1
2025-11-28T14:52:40.843428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:40.847687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789302076234682:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.847705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789302076234682:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.845821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789302581796446:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:40.847891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789302076234664:2108] [node 5] ICP22 created new session: [3:7577789302076234701:2108]
2025-11-28T14:52:40.845831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789302581796409:2108] [node 5] ICP22 created new session: [1:7577789302581796447:2108]
2025-11-28T14:52:40.848152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789302076234701:2108] [node 5] ICS09 handshake done sender: [3:7577789302076234672:2112] self: [3:7577789302076234666:0] peer: [5:7577789300655743552:0] socket: 116 qp: -1
2025-11-28T14:52:40.848173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789302076234701:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:40.845851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.848323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:40.846185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789302581796447:2108] [node 5] ICS09 handshake done sender: [1:7577789302581796426:2118] self: [1:7577789302581796425:0] peer: [5:7577789300655743554:0] socket: 56 qp: -1
2025-11-28T14:52:40.850481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789300655743461:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24516) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789300655743549:2118] held: no
2025-11-28T14:52:40.846710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789302581796447:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:40.851047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789300655743548:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:40.846949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:40.851077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789301617151669:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:40.851102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789301617151669:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:40.852083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789302076234682:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.853207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789302076234665:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.853120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789301617151676:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:40.853636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789300655743556:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39196
No RDMA context found for GID: No RDMA context found for GID: 2025-11-28T14:52:40.853700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:40.853174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789301617151592:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789301617151676:2117] held: no
2025-11-28T14:52:40.853638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789300655743555:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39196
2025-11-28T14:52:40.854203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789300655743462:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.854251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:40.854333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117553:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
2025-11-28T14:52:40.854361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789300655743462:2049] [node 2] ICP22 created new session: [5:7577789300655743557:2049]
2025-11-28T14:52:40.853835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789302076234665:2109] [node 6] ICP22 created new session: [3:7577789302076234704:2109]
2025-11-28T14:52:40.854346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789301617151685:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43823
2025-11-28T14:52:40.854172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789302076234704:2109] [node 6] ICS09 handshake done sender: [3:7577789302076234682:2118] self: [3:7577789302076234681:0] peer: [6:7577789301455117552:0] socket: 118 qp: -1
2025-11-28T14:52:40.854396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789300655743557:2049] [node 2] ICS09 handshake done sender: [5:7577789300655743548:2117] self: [5:7577789300655743550:0] peer: [2:7577789302441799662:0] socket: 54 qp: -1
2025-11-28T14:52:40.854368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789302076234704:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:40.854431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789300655743557:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:40.854416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.854447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789300655743463:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4046) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789300655743551:2119] held: no
2025-11-28T14:52:40.854922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789301455117456:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4046) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789301455117549:2121] held: no
2025-11-28T14:52:40.854944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789301617151592:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.854835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.855264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:40.855378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789301617151592:2049] [node 2] ICP22 created new session: [4:7577789301617151688:2049]
2025-11-28T14:52:40.855410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789300655743549:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:40.855428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789301455117549:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.855415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789301617151688:2049] [node 2] ICS09 handshake done sender: [4:7577789301617151676:2117] self: [4:7577789301617151679:0] peer: [2:7577789302441799649:0] socket: 78 qp: -1
2025-11-28T14:52:40.855586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789301617151688:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:40.856482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:40.857375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789301617151677:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:40.857451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789301617151669:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:40.857485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789301455117456:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.857502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789300655743461:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.857538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:40.857559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:40.857576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789301455117456:2050] [node 3] ICP22 created new session: [6:7577789301455117555:2050]
2025-11-28T14:52:40.857603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789300655743461:2048] [node 1] ICP22 created new session: [5:7577789300655743559:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:40.857876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789301617151591:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.857905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789300655743559:2048] [node 1] ICS09 handshake done sender: [5:7577789300655743549:2118] self: [5:7577789300655743554:0] peer: [1:7577789302581796425:0] socket: 70 qp: -1
2025-11-28T14:52:40.857705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789301455117555:2050] [node 3] ICS09 handshake done sender: [6:7577789301455117549:2121] self: [6:7577789301455117552:0] peer: [3:7577789302076234681:0] socket: 68 qp: -1
2025-11-28T14:52:40.857908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789300655743551:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:40.857784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789301455117555:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:40.857901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789301617151655:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:40.857956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789300655743559:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.857988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:40.857928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:40.857921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.857993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789301455117554:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25447
2025-11-28T14:52:40.857943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:40.858576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789300655743463:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.858284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789301455117550:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:40.857966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789301617151591:2048] [node 1] ICP22 created new session: [4:7577789301617151690:2048]
2025-11-28T14:52:40.857972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789301617151655:2109] [node 6] ICP22 created new session: [4:7577789301617151691:2109]
2025-11-28T14:52:40.858606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:40.858711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789301455117515:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:40.858921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789300655743463:2050] [node 3] ICP22 created new session: [5:7577789300655743561:2050]
2025-11-28T14:52:40.858139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789301617151691:2109] [node 6] ICS09 handshake done sender: [4:7577789301617151669:2115] self: [4:7577789301617151668:0] peer: [6:7577789301455117551:0] socket: 59 qp: -1
2025-11-28T14:52:40.859202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:40.859133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789300655743561:2050] [node 3] ICS09 handshake done sender: [5:7577789300655743551:2119] self: [5:7577789300655743552:0] peer: [3:7577789302076234666:0] socket: 117 qp: -1
2025-11-28T14:52:40.858158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789301617151690:2048] [node 1] ICS09 handshake done sender: [4:7577789301617151677:2118] self: [4:7577789301617151683:0] peer: [1:7577789302581796416:0] socket: 114 qp: -1
2025-11-28T14:52:40.859261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789301455117515:2106] [node 4] ICP22 created new session: [6:7577789301455117557:2106]
2025-11-28T14:52:40.859153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789300655743561:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:40.858172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789301617151691:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:40.859374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789301455117557:2106] [node 4] ICS09 handshake done sender: [6:7577789301455117550:2122] self: [6:7577789301455117551:0] peer: [4:7577789301617151668:0] socket: 119 qp: -1
2025-11-28T14:52:40.859342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:40.859835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789301455117557:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:40.858191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:40.858224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789301617151690:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:40.859880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:40.858249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:40.860872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789301455117515:2106] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:40.904522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:40.904776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789302441799678:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-11-28T14:52:40.904826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789302441799580:2048] [node 1] ICP30 unregister session Session# [2:7577789302441799678:2048] VirtualId# [2:7577789302441799668:0]
2025-11-28T14:52:40.904867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789302441799678:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.904938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.904989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.905666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:40.905820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789302076234696:2048] [node 1] ICS01 socket: 90 reason# EndOfStream
2025-11-28T14:52:40.905856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789302076234600:2048] [node 1] ICP30 unregister session Session# [3:7577789302076234696:2048] VirtualId# [3:7577789302076234684:0]
2025-11-28T14:52:40.905882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789302076234696:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.905970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.906018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.907703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:40.907953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789301617151690:2048] [node 1] ICS01 socket: 114 reason# EndOfStream
2025-11-28T14:52:40.907999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789301617151591:2048] [node 1] ICP30 unregister session Session# [4:7577789301617151690:2048] VirtualId# [4:7577789301617151683:0]
2025-11-28T14:52:40.908031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789301617151690:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.908137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.908205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.908324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:40.908564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789300655743559:2048] [node 1] ICS01 socket: 70 reason# EndOfStream
2025-11-28T14:52:40.908631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789300655743461:2048] [node 1] ICP30 unregister session Session# [5:7577789300655743559:2048] VirtualId# [5:7577789300655743554:0]
2025-11-28T14:52:40.908662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789300655743559:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.908730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.908778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.909119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:40.909262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789301455117547:2048] [node 1] ICS01 socket: 75 reason# EndOfStream
2025-11-28T14:52:40.909290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789301455117454:2048] [node 1] ICP30 unregister session Session# [6:7577789301455117547:2048] VirtualId# [6:7577789301455117535:0]
2025-11-28T14:52:40.909313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789301455117547:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.909367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:40.909406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:40.955706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:40.955947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789302076234697:2049] [node 2] ICS01 socket: 86 reason# EndOfStream
2025-11-28T14:52:40.955994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789302076234601:2049] [node 2] ICP30 unregister session Session# [3:7577789302076234697:2049] VirtualId# [3:7577789302076234685:0]
2025-11-28T14:52:40.956032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789302076234697:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.956157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:40.956212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:40.957995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:40.958151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789301617151688:2049] [node 2] ICS01 socket: 78 reason# EndOfStream
2025-11-28T14:52:40.958182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789301617151592:2049] [node 2] ICP30 unregister session Session# [4:7577789301617151688:2049] VirtualId# [4:7577789301617151679:0]
2025-11-28T14:52:40.958206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789301617151688:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.958269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:40.958303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:40.958821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:40.959020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789300655743557:2049] [node 2] ICS01 socket: 54 reason# EndOfStream
2025-11-28T14:52:40.959070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789300655743462:2049] [node 2] ICP30 unregister session Session# [5:7577789300655743557:2049] VirtualId# [5:7577789300655743550:0]
2025-11-28T14:52:40.959116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789300655743557:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.959183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:40.959229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:40.959512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:40.959647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789301455117544:2049] [node 2] ICS01 socket: 88 reason# EndOfStream
2025-11-28T14:52:40.959679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789301455117455:2049] [node 2] ICP30 unregister session Session# [6:7577789301455117544:2049] VirtualId# [6:7577789301455117537:0]
2025-11-28T14:52:40.959699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789301455117544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:40.959790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:40.959819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:41.025164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:41.025339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789301617151678:2050] [node 3] ICS07 socket disconnect 69 reason# EndOfStream
2025-11-28T14:52:41.025362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789301617151678:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.025433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:41.025493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789301617151678:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:41.026061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789301617151593:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4046) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:41.026027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:41.026146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789301617151678:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:41.026168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789301617151593:2050] [node 3] ICP30 unregister session Session# [4:7577789301617151678:2050] VirtualId# [4:7577789301617151670:0]
2025-11-28T14:52:41.026247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789300655743561:2050] [node 3] ICS01 socket: 117 reason# EndOfStream
2025-11-28T14:52:41.026269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:41.026295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789301617151593:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4046) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.026299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789300655743463:2050] [node 3] ICP30 unregister session Session# [5:7577789300655743561:2050] VirtualId# [5:7577789300655743552:0]
2025-11-28T14:52:41.026302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4046) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.026367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789300655743561:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.026444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:41.026496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:41.026897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:41.027036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789301455117555:2050] [node 3] ICS01 socket: 68 reason# EndOfStream
2025-11-28T14:52:41.027072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789301455117456:2050] [node 3] ICP30 unregister session Session# [6:7577789301455117555:2050] VirtualId# [6:7577789301455117552:0]
2025-11-28T14:52:41.027098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789301455117555:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.027189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:41.027235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:41.036443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789301617151593:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:41.085008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:41.085184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789300655743546:2108] [node 4] ICS01 socket: 76 reason# EndOfStream
2025-11-28T14:52:41.085224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789300655743525:2108] [node 4] ICP30 unregister session Session# [5:7577789300655743546:2108] VirtualId# [5:7577789300655743540:0]
2025-11-28T14:52:41.085249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789300655743546:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.085327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:41.085389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:41.085882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:41.086030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789301455117557:2106] [node 4] ICS07 socket disconnect 119 reason# EndOfStream
2025-11-28T14:52:41.086056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789301455117557:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.086143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:41.086185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789301455117557:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:41.086861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789301455117515:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12203) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:41.086902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789301455117557:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:41.086986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789301455117515:2106] [node 4] ICP30 unregister session Session# [6:7577789301455117557:2106] VirtualId# [6:7577789301455117551:0]
2025-11-28T14:52:41.087101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:41.087132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789301455117515:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.087141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.096863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789301455117515:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:41.124118Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:41.124393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789301455117541:2105] [node 5] ICS07 socket disconnect 83 reason# EndOfStream
2025-11-28T14:52:41.124440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789301455117541:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.124562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:41.124647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789301455117541:2105] [node 5] ICS15 start handshake
2025-11-28T14:52:41.125439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789301455117513:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7321) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:41.125546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789301455117541:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:41.125604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789301455117513:2105] [node 5] ICP30 unregister session Session# [6:7577789301455117541:2105] VirtualId# [6:7577789301455117536:0]
2025-11-28T14:52:41.125726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:41.125767Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789301455117513:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7321) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.125781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7321) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.135782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789301455117513:2105] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:41.209302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789305729121196:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:41.209502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789305729121197:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:41.232280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789303876620380:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.232314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789303876620381:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:41.255174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789305203743906:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.255428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789305203743907:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:41.280497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789306823197232:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.280583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789306823197233:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:41.280771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789306823197234:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:41.300943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789306769167966:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.301041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789306769167968:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:41.301114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789306769167967:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:41.324900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789304058160905:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.324933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789304058160906:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:41.324982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789304058160907:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:41.406524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789305729121259:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:41.413183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789304058160966:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:41.414778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789306769168030:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:41.413809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789303876620444:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:41.414853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789305203743969:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:41.414171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789306823197295:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:41.414871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789305203743971:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:41.414947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789306769168031:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:41.414174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789303876620446:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:41.414963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789305203743970:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:41.414941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789305729121261:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:41.414735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789303876620445:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:41.414955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789305729121260:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:41.415858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789306823197296:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:41.415865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789304058160969:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.427772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789306823197307:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.429021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789306823197311:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14388
2025-11-28T14:52:41.429402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789303876620457:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42065
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.429988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789305729121275:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36913
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.430610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058160980:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
2025-11-28T14:52:41.432543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789306823197308:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.430657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789303876620461:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42065
2025-11-28T14:52:41.433211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789306769168045:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28436
2025-11-28T14:52:41.432523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789303876620462:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42065
2025-11-28T14:52:41.432936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789305729121276:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36913
2025-11-28T14:52:41.430628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058160981:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
2025-11-28T14:52:41.433164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789306823197307:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.433214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789305729121277:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36913
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:41.430631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058160984:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.434291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789306823197232:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.434305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789303876620457:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.433724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789304058160980:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.437169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789305203743985:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56629
2025-11-28T14:52:41.437163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121266:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.438668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789303876620380:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.438610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789304058160905:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.438653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789306769168045:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.437205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789305203743986:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56629
2025-11-28T14:52:41.439461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789303876620463:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42065
2025-11-28T14:52:41.437173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121269:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.441029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789303876620462:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.439467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789306769168046:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28436
2025-11-28T14:52:41.440610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789304058160981:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.437202Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789305729121266:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.441612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789303876620463:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.442321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789305203743986:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.437202Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789305729121269:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.441773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789303876620455:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.440634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789304058160984:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.442426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789305203743985:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.439476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789305729121278:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36913
2025-11-28T14:52:41.440532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789306769167967:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.442251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789304058160907:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.440493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789305729121269:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:41.441800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789303876620455:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.440527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121277:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.442592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789304058160906:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.441133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789306769168046:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.442636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789303876620454:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.442626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789305203743980:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.442639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789303876620461:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.440528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121267:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.442636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789306823197302:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.442617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789306769168030:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.442658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789305203743980:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.442671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789306823197302:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.440546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789305729121267:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.442658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789303876620454:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.444944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789305203743980:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:41.444208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789303876620455:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:41.441050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121275:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.444596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789306823197314:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14388
2025-11-28T14:52:41.444614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789306823197311:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.445719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789303876620454:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:41.442871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789305729121266:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:41.445474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058160987:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
2025-11-28T14:52:41.444248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789305729121267:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:41.444686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789306823197308:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.445975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789303876620445:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.446024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:41.445228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789306823197234:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.444375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121278:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.448069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789306769167967:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20900) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789306769168045:2116] held: no
2025-11-28T14:52:41.446764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789305203743973:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.449014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789304058160980:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.446511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789306823197302:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.444377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121276:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.446803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789305203743973:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.446165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789303876620445:2109] [node 5] ICP22 created new session: [2:7577789303876620471:2109]
2025-11-28T14:52:41.449325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789306769168049:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28436
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.446748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789306823197307:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:41.444416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789305729121259:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.446539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789303876620471:2109] [node 5] ICS09 handshake done sender: [2:7577789303876620455:2114] self: [2:7577789303876620450:0] peer: [5:7577789306769168047:0] socket: 67 qp: -1
2025-11-28T14:52:41.452048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789306769168050:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28436
2025-11-28T14:52:41.451844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789304058160905:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12688) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789304058160980:2115] held: no
2025-11-28T14:52:41.448011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789305203743969:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.445334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.447950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789303876620471:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:41.448915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789306823197232:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12688) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789306823197307:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.452302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058160990:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
2025-11-28T14:52:41.449325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:41.445519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789305729121259:2107] [node 4] ICP22 created new session: [1:7577789305729121284:2107]
2025-11-28T14:52:41.449355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789306823197295:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.448024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789303876620444:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.450777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789305203743969:2109] [node 6] ICP22 created new session: [3:7577789305203743990:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.448066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:41.456845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789306769168046:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:41.454308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789304058160906:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20900) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789304058160981:2116] held: no
2025-11-28T14:52:41.451170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:41.445519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789305729121196:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.451903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789305203743990:2109] [node 6] ICS09 handshake done sender: [3:7577789305203743980:2113] self: [3:7577789305203743976:0] peer: [6:7577789304058160988:0] socket: 74 qp: -1
2025-11-28T14:52:41.448703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:41.457071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789306769168051:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28436
2025-11-28T14:52:41.452397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789306823197295:2108] [node 5] ICP22 created new session: [4:7577789306823197318:2108]
2025-11-28T14:52:41.456440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058160991:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.445583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:41.451973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789305203743973:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:41.458099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789306769168030:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.445663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789305729121260:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.448960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789303876620444:2110] [node 6] ICP22 created new session: [2:7577789303876620473:2110]
2025-11-28T14:52:41.451818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789303876620473:2110] [node 6] ICS09 handshake done sender: [2:7577789303876620454:2113] self: [2:7577789303876620451:0] peer: [6:7577789304058160989:0] socket: 68 qp: -1
2025-11-28T14:52:41.458363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789306769168045:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:41.453485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789306823197318:2108] [node 5] ICS09 handshake done sender: [4:7577789306823197302:2111] self: [4:7577789306823197298:0] peer: [5:7577789306769168048:0] socket: 84 qp: -1
2025-11-28T14:52:41.460039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789304058160905:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.452143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789305203743990:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:41.445695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789305729121284:2107] [node 4] ICS09 handshake done sender: [1:7577789305729121269:2112] self: [1:7577789305729121265:0] peer: [4:7577789306823197313:0] socket: 61 qp: -1
2025-11-28T14:52:41.458459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789306769168051:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.451843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789303876620473:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:41.453539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789306823197318:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: No RDMA context found for GID: 2025-11-28T14:52:41.445704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789305729121196:2048] [node 2] ICP22 created new session: [1:7577789305729121287:2048]
2025-11-28T14:52:41.454017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:41.458561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789306769167966:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.452080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:41.460084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:41.454293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:41.454381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789305203743970:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.453938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789303876620469:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42065
2025-11-28T14:52:41.460018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.445771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.460545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789304058160905:2048] [node 1] ICP22 created new session: [6:7577789304058160995:2048]
2025-11-28T14:52:41.458979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789306823197321:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14388
2025-11-28T14:52:41.455979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:41.460314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789306769167967:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.460824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789304058160984:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:41.445790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789305729121284:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:41.460074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789306823197234:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13789) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789306823197308:2116] held: no
2025-11-28T14:52:41.453991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789303876620380:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12688) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789303876620457:2116] held: no
2025-11-28T14:52:41.456451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789305203743970:2107] [node 4] ICP22 created new session: [3:7577789305203743995:2107]
2025-11-28T14:52:41.461251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789306823197232:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.460825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789304058160995:2048] [node 1] ICS09 handshake done sender: [6:7577789304058160980:2115] self: [6:7577789304058160986:0] peer: [1:7577789305729121262:0] socket: 66 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.457719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789305203743995:2107] [node 4] ICS09 handshake done sender: [3:7577789305203743973:2110] self: [3:7577789305203743972:0] peer: [4:7577789306823197317:0] socket: 69 qp: -1
2025-11-28T14:52:41.460346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:41.445991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.461136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789303876620457:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:41.458023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789305203743995:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:41.462983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789303876620477:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42065
2025-11-28T14:52:41.464143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:41.460520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789306769168030:2108] [node 4] ICP22 created new session: [5:7577789306769168055:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.460805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789306769167967:2049] [node 2] ICP22 created new session: [5:7577789306769168056:2049]
2025-11-28T14:52:41.445995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789305729121260:2109] [node 6] ICP22 created new session: [1:7577789305729121288:2109]
2025-11-28T14:52:41.461119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789304058160995:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:41.461175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789306769168055:2108] [node 4] ICS09 handshake done sender: [5:7577789306769168046:2117] self: [5:7577789306769168048:0] peer: [4:7577789306823197298:0] socket: 98 qp: -1
2025-11-28T14:52:41.458938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789305203743993:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56629
2025-11-28T14:52:41.446176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789305729121287:2048] [node 2] ICS09 handshake done sender: [1:7577789305729121267:2111] self: [1:7577789305729121263:0] peer: [2:7577789303876620464:0] socket: 60 qp: -1
2025-11-28T14:52:41.463690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789303876620380:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.461219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789306769168056:2049] [node 2] ICS09 handshake done sender: [5:7577789306769168045:2116] self: [5:7577789306769168047:0] peer: [2:7577789303876620450:0] socket: 90 qp: -1
2025-11-28T14:52:41.446298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789305729121287:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:41.461221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789306769168055:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:41.459408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:41.465001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789306823197232:2048] [node 1] ICP22 created new session: [4:7577789306823197325:2048]
2025-11-28T14:52:41.446318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789305729121288:2109] [node 6] ICS09 handshake done sender: [1:7577789305729121266:2110] self: [1:7577789305729121262:0] peer: [6:7577789304058160986:0] socket: 62 qp: -1
2025-11-28T14:52:41.463930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:41.446479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:41.462993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789306769168056:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:41.463319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:41.461217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:41.463592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789306769168053:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28436
2025-11-28T14:52:41.466072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789306823197325:2048] [node 1] ICS09 handshake done sender: [4:7577789306823197307:2115] self: [4:7577789306823197313:0] peer: [1:7577789305729121265:0] socket: 63 qp: -1
2025-11-28T14:52:41.446493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789305729121288:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:41.461182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789305203743993:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.461225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789304058160907:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.463653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:41.446739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:41.466262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789306823197325:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:41.463699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789305203743906:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.464134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789303876620380:2048] [node 1] ICP22 created new session: [2:7577789303876620478:2048]
2025-11-28T14:52:41.461274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789304058160981:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:41.460771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789305729121294:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36913
2025-11-28T14:52:41.465059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789306769168058:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28436
2025-11-28T14:52:41.463051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:41.465386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789303876620478:2048] [node 1] ICS09 handshake done sender: [2:7577789303876620457:2116] self: [2:7577789303876620464:0] peer: [1:7577789305729121263:0] socket: 64 qp: -1
2025-11-28T14:52:41.467134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:41.466293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789306769168053:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.461131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121271:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.464089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789305203743997:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56629
2025-11-28T14:52:41.463339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058160992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
2025-11-28T14:52:41.467105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789306769167968:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.466060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789303876620477:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.467387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789306823197308:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:41.467592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789306769168061:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28436
2025-11-28T14:52:41.461152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789305729121271:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.466127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789303876620478:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:41.463375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789304058160907:2050] [node 3] ICP22 created new session: [6:7577789304058160997:2050]
2025-11-28T14:52:41.463556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121294:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.465393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789305203743999:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56629
2025-11-28T14:52:41.466486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:41.469718Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789306769168051:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:41.463688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789304058160997:2050] [node 3] ICS09 handshake done sender: [6:7577789304058160984:2117] self: [6:7577789304058160988:0] peer: [3:7577789305203743976:0] socket: 77 qp: -1
2025-11-28T14:52:41.463997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789305729121271:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:41.467858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789303876620456:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.470275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789306769167966:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.464099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789304058160997:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:41.466416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789305203743997:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.466438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789305729121270:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.467587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789306823197323:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14388
2025-11-28T14:52:41.465396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789304058160906:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.470643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789306769168058:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.467880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789303876620456:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.466465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789305729121270:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.467669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789305729121261:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.467079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789305203743907:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.470574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789303876620456:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:41.466288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:41.470728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:41.467835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:41.467828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789306823197324:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14388
2025-11-28T14:52:41.470985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789306769167966:2048] [node 1] ICP22 created new session: [5:7577789306769168063:2048]
2025-11-28T14:52:41.467605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789305203743981:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.471604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789303876620381:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.466423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:41.468748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789305729121261:2108] [node 5] ICP22 created new session: [1:7577789305729121297:2108]
2025-11-28T14:52:41.472355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789306769168063:2048] [node 1] ICS09 handshake done sender: [5:7577789306769168051:2120] self: [5:7577789306769168054:0] peer: [1:7577789305729121268:0] socket: 100 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.466483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789304058160906:2049] [node 2] ICP22 created new session: [6:7577789304058160998:2049]
2025-11-28T14:52:41.471855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:41.467624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789305203743981:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.469743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789305729121297:2108] [node 5] ICS09 handshake done sender: [1:7577789305729121271:2114] self: [1:7577789305729121268:0] peer: [5:7577789306769168054:0] socket: 81 qp: -1
2025-11-28T14:52:41.472407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789303876620381:2049] [node 3] ICP22 created new session: [2:7577789303876620482:2049]
2025-11-28T14:52:41.467027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789304058160998:2049] [node 2] ICS09 handshake done sender: [6:7577789304058160981:2116] self: [6:7577789304058160989:0] peer: [2:7577789303876620451:0] socket: 72 qp: -1
2025-11-28T14:52:41.468805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789306823197234:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.472559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789306769168063:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:41.469877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789305729121297:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:41.472414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789303876620453:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.467047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789304058160998:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:41.470217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789305203743981:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:41.472445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789303876620453:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.472612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:41.469672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789306823197323:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.470284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:41.471021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789305203743971:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.467617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:41.473483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789303876620482:2049] [node 3] ICS09 handshake done sender: [2:7577789303876620456:2115] self: [2:7577789303876620447:0] peer: [3:7577789305203744001:0] socket: 122 qp: -1
2025-11-28T14:52:41.472017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789305729121270:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:41.469753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:41.469792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789306823197234:2050] [node 3] ICP22 created new session: [4:7577789306823197327:2050]
2025-11-28T14:52:41.474659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789306769167968:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13789) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789306769168053:2121] held: no
2025-11-28T14:52:41.469822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789304058160992:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.469837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789306823197327:2050] [node 3] ICS09 handshake done sender: [4:7577789306823197308:2116] self: [4:7577789306823197317:0] peer: [3:7577789305203743972:0] socket: 75 qp: -1
2025-11-28T14:52:41.473519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789303876620482:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:41.470736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058161000:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
2025-11-28T14:52:41.473541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:41.469869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789306823197327:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.475098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789306769168064:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28436
2025-11-28T14:52:41.473562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:41.474813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789305729121197:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.470982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789304058160969:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.469895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:41.474288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789305203743971:2108] [node 5] ICP22 created new session: [3:7577789305203744005:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.475042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:41.474327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789304058161000:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.470226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789306823197233:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.474349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789305203744005:2108] [node 5] ICS09 handshake done sender: [3:7577789305203743981:2114] self: [3:7577789305203743978:0] peer: [5:7577789306769168062:0] socket: 92 qp: -1
2025-11-28T14:52:41.475173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789304058160966:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:41.474950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789305203744005:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:41.472395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789306823197306:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.475293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789305729121197:2049] [node 3] ICP22 created new session: [1:7577789305729121300:2049]
2025-11-28T14:52:41.474387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789303876620444:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(21703552ns) {HandleHandshakeStatus:329(21701396ns) {DropIncomingHandshake:442(13966ns) DropOutgoingHandshake:457(276ns) SwitchToState:179(144ns) ProcessPendingSessionEvents:495(15728725ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15652620ns max# 2132167ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.481914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789304058160969:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13368) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789304058160992:2121] held: no
2025-11-28T14:52:41.475207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:41.472420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789306823197306:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.475595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789303876620453:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:41.481898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789305203744003:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56629
2025-11-28T14:52:41.484211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058161004:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
2025-11-28T14:52:41.474653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789306823197306:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:41.475459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789305729121300:2049] [node 3] ICS09 handshake done sender: [1:7577789305729121270:2113] self: [1:7577789305729121264:0] peer: [3:7577789305203744000:0] socket: 71 qp: -1
2025-11-28T14:52:41.484238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789306769168041:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:41.476009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789303876620445:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(23753946ns) {HandleHandshakeStatus:329(23683094ns) {DropIncomingHandshake:442(10421ns) DropOutgoingHandshake:457(356ns) SwitchToState:179(289ns) ProcessPendingSessionEvents:495(18921211ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17475959ns max# 1785319ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.475685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789305729121300:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:41.484297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789306769168041:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:41.475528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789306823197296:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.482836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789303876620446:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.485696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789306769168053:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:41.486972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789306769167968:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.485033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:41.487319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:41.486410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789303876620446:2107] [node 4] ICP22 created new session: [2:7577789303876620485:2107]
2025-11-28T14:52:41.476292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:41.475864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:41.487368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789306769168041:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:41.486589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789303876620485:2107] [node 4] ICS09 handshake done sender: [2:7577789303876620453:2112] self: [2:7577789303876620449:0] peer: [4:7577789306823197328:0] socket: 94 qp: -1
2025-11-28T14:52:41.485146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789305203743969:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(22938655ns) {HandleHandshakeStatus:329(22936499ns) {DropIncomingHandshake:442(910ns) DropOutgoingHandshake:457(665ns) SwitchToState:179(125ns) ProcessPendingSessionEvents:495(13721808ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12642482ns max# 1357489ns}}}
2025-11-28T14:52:41.486794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789303876620485:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:41.476230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789306823197296:2109] [node 6] ICP22 created new session: [4:7577789306823197331:2109]
2025-11-28T14:52:41.487505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789306769167968:2050] [node 3] ICP22 created new session: [5:7577789306769168068:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:41.481948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789305729121260:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(22100598ns) {HandleHandshakeStatus:329(22099483ns) {DropIncomingHandshake:442(5813ns) DropOutgoingHandshake:457(253ns) SwitchToState:179(121ns) ProcessPendingSessionEvents:495(15766833ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15685285ns max# 1357660ns}}}
2025-11-28T14:52:41.487920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789306769168031:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:41.485613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789305203743907:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20900) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789305203743997:2118] held: no
2025-11-28T14:52:41.487371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:41.480744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789306823197331:2109] [node 6] ICS09 handshake done sender: [4:7577789306823197306:2114] self: [4:7577789306823197303:0] peer: [6:7577789304058161002:0] socket: 80 qp: -1
2025-11-28T14:52:41.481727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789306823197331:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:41.488212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789306769168068:2050] [node 3] ICS09 handshake done sender: [5:7577789306769168053:2121] self: [5:7577789306769168062:0] peer: [3:7577789305203743978:0] socket: 120 qp: -1
2025-11-28T14:52:41.482823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789305729121259:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(25262401ns) {HandleHandshakeStatus:329(25260468ns) {DropIncomingHandshake:442(13418ns) DropOutgoingHandshake:457(460ns) SwitchToState:179(555ns) ProcessPendingSessionEvents:495(20943879ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20194714ns max# 1742859ns}}}
2025-11-28T14:52:41.488691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789304058160992:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:41.488399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789306769168068:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:41.486490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789305203743970:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17226597ns) {HandleHandshakeStatus:329(17224965ns) {DropIncomingHandshake:442(8374ns) DropOutgoingHandshake:457(171ns) SwitchToState:179(478ns) ProcessPendingSessionEvents:495(9937210ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9881631ns max# 1785441ns}}}
2025-11-28T14:52:41.481776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:41.487014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789305203744004:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56629
2025-11-28T14:52:41.483912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789305729121196:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(24158233ns) {HandleHandshakeStatus:329(24156545ns) {DropIncomingHandshake:442(9969ns) DropOutgoingHandshake:457(320ns) SwitchToState:179(128ns) ProcessPendingSessionEvents:495(20342651ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18937910ns max# 1870805ns}}}
2025-11-28T14:52:41.485362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789306823197233:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20900) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789306823197323:2120] held: no
2025-11-28T14:52:41.488436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:41.485664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789306823197332:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14388
2025-11-28T14:52:41.489101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789304058161005:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28494
2025-11-28T14:52:41.489004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:41.489054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789306769168031:2109] [node 6] ICP22 created new session: [5:7577789306769168070:2109]
2025-11-28T14:52:41.489924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789304058160969:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.489222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789306769168070:2109] [node 6] ICS09 handshake done sender: [5:7577789306769168041:2115] self: [5:7577789306769168037:0] peer: [6:7577789304058161003:0] socket: 127 qp: -1
2025-11-28T14:52:41.487384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789306823197295:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(23475413ns) {HandleHandshakeStatus:329(23473326ns) {DropIncomingHandshake:442(635ns) DropOutgoingHandshake:457(341ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(15644871ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15555545ns max# 2078007ns}}}
2025-11-28T14:52:41.489902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789306823197323:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:41.489250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789306769168070:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:41.490563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:41.489281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:41.489283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789306769168030:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(16472217ns) {HandleHandshakeStatus:329(16470456ns) {DropIncomingHandshake:442(1169ns) DropOutgoingHandshake:457(8305ns) SwitchToState:179(301ns) ProcessPendingSessionEvents:495(9896801ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9836313ns max# 2460142ns}}}
2025-11-28T14:52:41.490527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789305203743906:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12688) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789305203743993:2117] held: no
2025-11-28T14:52:41.490562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789305203743969:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:41.490756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789304058160969:2107] [node 4] ICP22 created new session: [6:7577789304058161006:2107]
2025-11-28T14:52:41.490633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789306823197233:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.491048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789304058161006:2107] [node 4] ICS09 handshake done sender: [6:7577789304058160992:2121] self: [6:7577789304058161002:0] peer: [4:7577789306823197303:0] socket: 85 qp: -1
2025-11-28T14:52:41.490969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:41.491070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789304058161006:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:41.491145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789306823197233:2049] [node 2] ICP22 created new session: [4:7577789306823197334:2049]
2025-11-28T14:52:41.491629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:41.491667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789306823197334:2049] [node 2] ICS09 handshake done sender: [4:7577789306823197323:2120] self: [4:7577789306823197328:0] peer: [2:7577789303876620449:0] socket: 119 qp: -1
2025-11-28T14:52:41.491899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789305729121261:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18775000ns) {HandleHandshakeStatus:329(18674391ns) {DropIncomingHandshake:442(9645ns) DropOutgoingHandshake:457(216ns) SwitchToState:179(265ns) ProcessPendingSessionEvents:495(14692208ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14616561ns max# 4625377ns}}}
2025-11-28T14:52:41.491708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789306823197334:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:41.491779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:41.493003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789305203743997:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:41.493186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789304058160966:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5231) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789304058161000:2122] held: no
2025-11-28T14:52:41.493749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789304058161000:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:41.493866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789305203743971:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19553784ns) {HandleHandshakeStatus:329(19551367ns) {DropIncomingHandshake:442(11791ns) DropOutgoingHandshake:457(199ns) SwitchToState:179(128ns) ProcessPendingSessionEvents:495(9974389ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9705703ns max# 1044527ns}}}
2025-11-28T14:52:41.493910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789305203743907:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.494193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789305203743993:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:41.494175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789303876620381:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17459223ns) {HandleHandshakeStatus:329(17456358ns) {DropIncomingHandshake:442(10365ns) DropOutgoingHandshake:457(221ns) SwitchToState:179(187ns) ProcessPendingSessionEvents:495(14721060ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14511024ns max# 4329821ns}}}
2025-11-28T14:52:41.494224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:41.494366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789305203743907:2049] [node 2] ICP22 created new session: [3:7577789305203744007:2049]
2025-11-28T14:52:41.494398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789305203744007:2049] [node 2] ICS09 handshake done sender: [3:7577789305203743997:2118] self: [3:7577789305203744001:0] peer: [2:7577789303876620447:0] socket: 124 qp: -1
2025-11-28T14:52:41.494514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789305203744007:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:41.494591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789304058160966:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.494540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:41.494647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789305203743906:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:41.494713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:41.494746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789304058160966:2105] [node 5] ICP22 created new session: [6:7577789304058161008:2105]
2025-11-28T14:52:41.494776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789304058161008:2105] [node 5] ICS09 handshake done sender: [6:7577789304058161000:2122] self: [6:7577789304058161003:0] peer: [5:7577789306769168037:0] socket: 130 qp: -1
2025-11-28T14:52:41.494799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789304058161008:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:41.494811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:41.494786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789306769167966:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19733924ns) {HandleHandshakeStatus:329(19731991ns) {DropIncomingHandshake:442(522ns) DropOutgoingHandshake:457(3308ns) SwitchToState:179(94ns) ProcessPendingSessionEvents:495(15491365ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15414495ns max# 4794106ns}}}
2025-11-28T14:52:41.494851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:41.494878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789305203743906:2048] [node 1] ICP22 created new session: [3:7577789305203744009:2048]
2025-11-28T14:52:41.494892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789306769167966:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:41.494931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789305203744009:2048] [node 1] ICS09 handshake done sender: [3:7577789305203743993:2117] self: [3:7577789305203744000:0] peer: [1:7577789305729121264:0] socket: 86 qp: -1
2025-11-28T14:52:41.494975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789305203744009:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:41.495042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:41.495087Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789306823197296:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16272724ns) {HandleHandshakeStatus:329(16270391ns) {DropIncomingHandshake:442(17691ns) DropOutgoingHandshake:457(175ns) SwitchToState:179(186ns) ProcessPendingSessionEvents:495(7886467ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7801365ns max# 734732ns}}}
2025-11-28T14:52:41.534468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:41.534636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789303876620478:2048] [node 1] ICS01 socket: 64 reason# EndOfStream
2025-11-28T14:52:41.534691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789303876620380:2048] [node 1] ICP30 unregister session Session# [2:7577789303876620478:2048] VirtualId# [2:7577789303876620464:0]
2025-11-28T14:52:41.534724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789303876620478:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.534808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:41.534853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:41.535317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:41.535466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789305203744009:2048] [node 1] ICS01 socket: 86 reason# EndOfStream
2025-11-28T14:52:41.535502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789305203743906:2048] [node 1] ICP30 unregister session Session# [3:7577789305203744009:2048] VirtualId# [3:7577789305203744000:0]
2025-11-28T14:52:41.535520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789305203744009:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.535590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:41.535641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:41.537212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:41.537468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789306823197325:2048] [node 1] ICS01 socket: 63 reason# EndOfStream
2025-11-28T14:52:41.537514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789306823197232:2048] [node 1] ICP30 unregister session Session# [4:7577789306823197325:2048] VirtualId# [4:7577789306823197313:0]
2025-11-28T14:52:41.537570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789306823197325:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.537646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:41.537698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:41.537870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:41.538002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789306769168063:2048] [node 1] ICS07 socket disconnect 100 reason# EndOfStream
2025-11-28T14:52:41.538027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789306769168063:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.538086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:41.538134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789306769168063:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:41.538512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:41.538639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789304058160995:2048] [node 1] ICS01 socket: 66 reason# EndOfStream
2025-11-28T14:52:41.538668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789304058160905:2048] [node 1] ICP30 unregister session Session# [6:7577789304058160995:2048] VirtualId# [6:7577789304058160986:0]
2025-11-28T14:52:41.538686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789304058160995:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.538716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789306769167966:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12688) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:41.538739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:41.538753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789306769168063:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:41.538838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:41.538898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789306769167966:2048] [node 1] ICP30 unregister session Session# [5:7577789306769168063:2048] VirtualId# [5:7577789306769168054:0]
2025-11-28T14:52:41.538983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:41.539005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789306769167966:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12688) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.539014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12688) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.548494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789306769167966:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:41.593468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:41.593624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789305203744007:2049] [node 2] ICS01 socket: 124 reason# EndOfStream
2025-11-28T14:52:41.593659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789305203743907:2049] [node 2] ICP30 unregister session Session# [3:7577789305203744007:2049] VirtualId# [3:7577789305203744001:0]
2025-11-28T14:52:41.593688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789305203744007:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.593777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:41.593824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:41.595906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:41.596121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789306823197334:2049] [node 2] ICS01 socket: 119 reason# EndOfStream
2025-11-28T14:52:41.596204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789306823197233:2049] [node 2] ICP30 unregister session Session# [4:7577789306823197334:2049] VirtualId# [4:7577789306823197328:0]
2025-11-28T14:52:41.596252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789306823197334:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.596335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:41.596399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:41.596705Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:41.596844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789306769168056:2049] [node 2] ICS01 socket: 90 reason# EndOfStream
2025-11-28T14:52:41.596873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789306769167967:2049] [node 2] ICP30 unregister session Session# [5:7577789306769168056:2049] VirtualId# [5:7577789306769168047:0]
2025-11-28T14:52:41.596893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789306769168056:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.596964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:41.597006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:41.597434Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:41.597610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789304058160998:2049] [node 2] ICS01 socket: 72 reason# EndOfStream
2025-11-28T14:52:41.597652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789304058160906:2049] [node 2] ICP30 unregister session Session# [6:7577789304058160998:2049] VirtualId# [6:7577789304058160989:0]
2025-11-28T14:52:41.597693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789304058160998:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.597764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:41.597807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:41.675220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:41.675388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789306823197327:2050] [node 3] ICS01 socket: 75 reason# EndOfStream
2025-11-28T14:52:41.675415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789306823197234:2050] [node 3] ICP30 unregister session Session# [4:7577789306823197327:2050] VirtualId# [4:7577789306823197317:0]
2025-11-28T14:52:41.675442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789306823197327:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.675511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:41.675546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:41.676292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:41.676499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789306769168068:2050] [node 3] ICS01 socket: 120 reason# EndOfStream
2025-11-28T14:52:41.676531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789306769167968:2050] [node 3] ICP30 unregister session Session# [5:7577789306769168068:2050] VirtualId# [5:7577789306769168062:0]
2025-11-28T14:52:41.676551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789306769168068:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.676625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:41.676656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:41.677310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:41.677449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789304058160997:2050] [node 3] ICS07 socket disconnect 77 reason# EndOfStream
2025-11-28T14:52:41.677473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789304058160997:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.677534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:41.677567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789304058160997:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:41.678136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789304058160907:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13789) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:41.678177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789304058160997:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:41.678209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789304058160907:2050] [node 3] ICP30 unregister session Session# [6:7577789304058160997:2050] VirtualId# [6:7577789304058160988:0]
2025-11-28T14:52:41.678304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:41.678327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789304058160907:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13789) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.678335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13789) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.687684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789304058160907:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:41.724561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:41.724767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789306769168055:2108] [node 4] ICS07 socket disconnect 98 reason# EndOfStream
2025-11-28T14:52:41.724799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789306769168055:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.724877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:41.724938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789306769168055:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:41.725604Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:41.725657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789306769168030:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13368) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:41.725697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789306769168055:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:41.725755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789306769168030:2108] [node 4] ICP30 unregister session Session# [5:7577789306769168055:2108] VirtualId# [5:7577789306769168048:0]
2025-11-28T14:52:41.725793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789304058161006:2107] [node 4] ICS01 socket: 85 reason# EndOfStream
2025-11-28T14:52:41.725835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789304058160969:2107] [node 4] ICP30 unregister session Session# [6:7577789304058161006:2107] VirtualId# [6:7577789304058161002:0]
2025-11-28T14:52:41.725889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789304058161006:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.725883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:41.725909Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789306769168030:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13368) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.725919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13368) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:41.725997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:41.726054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:41.735839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789306769168030:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:41.772962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:41.773207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789304058161008:2105] [node 5] ICS01 socket: 130 reason# EndOfStream
2025-11-28T14:52:41.773249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789304058160966:2105] [node 5] ICP30 unregister session Session# [6:7577789304058161008:2105] VirtualId# [6:7577789304058161003:0]
2025-11-28T14:52:41.773289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789304058161008:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:41.773356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:41.773404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:41.860750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789306435040967:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:41.860796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789306435040968:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:41.884665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789304502446327:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.884702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789304502446328:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:41.905042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789304082160885:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.905078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789304082160886:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:41.925277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789304286259172:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.925318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789304286259173:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:41.925348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789304286259174:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:41.946598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789305845690008:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.946631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789305845690009:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:41.946656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789305845690010:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:41.966040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789307220334249:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:41.966072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789307220334250:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:41.966101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789307220334251:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:42.047816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789311515301606:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:42.047938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789310730008326:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:42.048017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789308377128245:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:42.047964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789308797413686:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:42.049319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789308581226531:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:42.049460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789308797413687:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:42.049521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789310730008327:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:42.049519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789308377128246:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:42.049557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789310140657367:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:42.049857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789308797413688:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:42.049848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789308581226532:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:42.049992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789311515301609:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:42.050087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789308377128247:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:42.050202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789310730008328:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:42.050347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789310140657368:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.059334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789308581226546:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11617
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.060223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308797413699:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51773
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.061031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308797413703:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51773
2025-11-28T14:52:42.061132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308797413704:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51773
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.061496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789310730008339:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65095
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.061660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789310730008340:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65095
2025-11-28T14:52:42.061717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789310730008341:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65095
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.062053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789310730008342:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65095
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.064720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789310140657379:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.065764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789308377128261:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49405
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.066255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789308377128262:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49405
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.066416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789308581226547:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11617
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.066537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789308377128261:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:42.066554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789308581226546:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:42.066584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789308377128263:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49405
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.066628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789308581226548:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11617
2025-11-28T14:52:42.066645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789308377128264:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49405
2025-11-28T14:52:42.066731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789304286259174:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.066878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789308377128253:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.066895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789308377128253:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.067927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308797413705:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51773
2025-11-28T14:52:42.067958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789308377128253:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.068024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789310140657379:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.068287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789308377128245:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.068271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789308581226550:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11617
2025-11-28T14:52:42.068352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.068496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789308377128245:2107] [node 4] ICP22 created new session: [3:7577789308377128267:2107]
2025-11-28T14:52:42.068503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308797413699:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.068557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789308377128267:2107] [node 4] ICS09 handshake done sender: [3:7577789308377128253:2113] self: [3:7577789308377128252:0] peer: [4:7577789308581226549:0] socket: 68 qp: -1
2025-11-28T14:52:42.068724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789308377128267:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:42.068772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.069004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008339:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.069012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308797413703:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.069150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008340:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.069159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789308581226547:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.069213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008341:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.069218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789308581226546:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:42.069400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308797413704:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.069416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789304286259174:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.069505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.069536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789304286259174:2050] [node 3] ICP22 created new session: [4:7577789308581226551:2050]
2025-11-28T14:52:42.069487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008342:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.069568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789308581226551:2050] [node 3] ICS09 handshake done sender: [4:7577789308581226546:2115] self: [4:7577789308581226549:0] peer: [3:7577789308377128252:0] socket: 69 qp: -1
2025-11-28T14:52:42.069602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789308581226551:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.069581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308797413705:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.069631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.069684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308797413706:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51773
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.069905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789310730008346:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65095
2025-11-28T14:52:42.069899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789304502446327:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.070399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789308377128262:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.070423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008330:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.070442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789310730008330:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.070483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789308377128263:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.070781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789304082160886:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.071090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789310730008330:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.071640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308797413711:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51773
2025-11-28T14:52:42.071649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789304082160885:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.072166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789306435040967:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.072259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:42.072291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789306435040967:2048] [node 2] ICP22 created new session: [1:7577789310730008352:2048]
2025-11-28T14:52:42.072320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789310730008352:2048] [node 2] ICS09 handshake done sender: [1:7577789310730008330:2110] self: [1:7577789310730008329:0] peer: [2:7577789308797413710:0] socket: 92 qp: -1
2025-11-28T14:52:42.072399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789310730008352:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:42.072456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789308581226548:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.072422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.072659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308797413692:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.072722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789304286259173:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.072688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789308797413692:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.072816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789310140657383:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-28T14:52:42.072981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789304286259172:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.073307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308797413706:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.073425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308797413694:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.073449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789308797413694:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.073445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789308377128264:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.073625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789310140657385:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-28T14:52:42.073725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789310140657383:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.073721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008346:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.073841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789305845690010:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.074039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789308377128255:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.074056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789308377128255:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.074401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308797413705:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.074564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789308377128255:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:42.074637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008332:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.074658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789310730008332:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.074826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789310140657385:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.075266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789308377128246:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.075322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.075347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789308377128246:2108] [node 5] ICP22 created new session: [3:7577789308377128274:2108]
2025-11-28T14:52:42.075398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008334:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.075417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789310730008334:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.075378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789308377128274:2108] [node 5] ICS09 handshake done sender: [3:7577789308377128255:2114] self: [3:7577789308377128254:0] peer: [5:7577789310140657386:0] socket: 88 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.075425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789308377128274:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.075471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:42.076457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789308581226556:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11617
2025-11-28T14:52:42.076517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789310140657387:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-28T14:52:42.076773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789304502446327:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.076881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.076976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789304502446327:2048] [node 1] ICP22 created new session: [2:7577789308797413714:2048]
2025-11-28T14:52:42.077026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308797413714:2048] [node 1] ICS09 handshake done sender: [2:7577789308797413705:2118] self: [2:7577789308797413710:0] peer: [1:7577789310730008329:0] socket: 94 qp: -1
2025-11-28T14:52:42.077063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308797413714:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:42.077099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:42.077120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789310140657367:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.077157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789310730008332:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:42.077209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789308581226557:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11617
2025-11-28T14:52:42.077366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789310140657383:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:42.077387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789308581226540:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.077405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789308581226540:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.077469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789310730008334:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.078166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789305845690010:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.078226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.078266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789305845690010:2050] [node 3] ICP22 created new session: [5:7577789310140657390:2050]
2025-11-28T14:52:42.078329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789310140657390:2050] [node 3] ICS09 handshake done sender: [5:7577789310140657383:2116] self: [5:7577789310140657386:0] peer: [3:7577789308377128254:0] socket: 112 qp: -1
2025-11-28T14:52:42.078362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789310140657390:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:42.078329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789306435040968:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.078357Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789308581226548:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.078387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.078370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.078396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789306435040968:2049] [node 3] ICP22 created new session: [1:7577789310730008358:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.078421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789310730008358:2049] [node 3] ICS09 handshake done sender: [1:7577789310730008332:2111] self: [1:7577789310730008331:0] peer: [3:7577789308377128271:0] socket: 93 qp: -1
2025-11-28T14:52:42.078469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789308377128276:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49405
2025-11-28T14:52:42.078437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789310730008358:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:42.078450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789304502446327:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:42.078653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789308377128277:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49405
2025-11-28T14:52:42.078464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.078648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789304286259172:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.078694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.078730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789304286259172:2048] [node 1] ICP22 created new session: [4:7577789308581226559:2048]
2025-11-28T14:52:42.078757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789308581226559:2048] [node 1] ICS09 handshake done sender: [4:7577789308581226548:2117] self: [4:7577789308581226555:0] peer: [1:7577789310730008333:0] socket: 102 qp: -1
2025-11-28T14:52:42.078783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789308581226559:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.078799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:42.078925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301623:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
2025-11-28T14:52:42.079183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789308377128263:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.079374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789304082160885:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.079401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.079423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789304082160885:2048] [node 1] ICP22 created new session: [3:7577789308377128279:2048]
2025-11-28T14:52:42.079450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789308377128279:2048] [node 1] ICS09 handshake done sender: [3:7577789308377128263:2118] self: [3:7577789308377128271:0] peer: [1:7577789310730008331:0] socket: 100 qp: -1
2025-11-28T14:52:42.079466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789308377128279:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:42.079483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:42.079619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789308581226540:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:42.079807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789310730008326:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.079848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.079886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789310730008326:2107] [node 4] ICP22 created new session: [1:7577789310730008360:2107]
2025-11-28T14:52:42.079918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789310730008360:2107] [node 4] ICS09 handshake done sender: [1:7577789310730008334:2112] self: [1:7577789310730008333:0] peer: [4:7577789308581226555:0] socket: 96 qp: -1
2025-11-28T14:52:42.079940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789310730008360:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:42.079982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.080711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789308581226531:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.080737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.080764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308797413692:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:42.080763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789308581226531:2108] [node 5] ICP22 created new session: [4:7577789308581226561:2108]
2025-11-28T14:52:42.080797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789308581226561:2108] [node 5] ICS09 handshake done sender: [4:7577789308581226540:2113] self: [4:7577789308581226539:0] peer: [5:7577789310140657388:0] socket: 74 qp: -1
2025-11-28T14:52:42.080825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789308581226561:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:42.080848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:42.081231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308797413694:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.081671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789304502446328:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.081811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.081856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789304502446328:2049] [node 3] ICP22 created new session: [2:7577789308797413717:2049]
2025-11-28T14:52:42.082137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308797413717:2049] [node 3] ICS09 handshake done sender: [2:7577789308797413692:2111] self: [2:7577789308797413691:0] peer: [3:7577789308377128270:0] socket: 85 qp: -1
2025-11-28T14:52:42.082169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308797413717:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:42.082209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.082674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789305845690010:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.082796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789310140657389:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.083285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789310140657392:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-28T14:52:42.083466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789308797413686:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.083566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789308581226547:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:42.083631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.083682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789308797413686:2107] [node 4] ICP22 created new session: [2:7577789308797413719:2107]
2025-11-28T14:52:42.083741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308797413719:2107] [node 4] ICS09 handshake done sender: [2:7577789308797413694:2112] self: [2:7577789308797413693:0] peer: [4:7577789308581226553:0] socket: 87 qp: -1
2025-11-28T14:52:42.083786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308797413719:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:42.083818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.083798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789310140657385:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.083960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789304286259173:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.084030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:42.084055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789304286259173:2049] [node 2] ICP22 created new session: [4:7577789308581226564:2049]
2025-11-28T14:52:42.084186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789308581226564:2049] [node 2] ICS09 handshake done sender: [4:7577789308581226547:2116] self: [4:7577789308581226553:0] peer: [2:7577789308797413693:0] socket: 91 qp: -1
2025-11-28T14:52:42.084209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789308581226564:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:42.084216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789310140657367:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.084241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.084297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.084456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789310140657367:2108] [node 4] ICP22 created new session: [5:7577789310140657394:2108]
2025-11-28T14:52:42.084510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789310140657394:2108] [node 4] ICS09 handshake done sender: [5:7577789310140657385:2117] self: [5:7577789310140657388:0] peer: [4:7577789308581226539:0] socket: 113 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.084556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789310140657394:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:42.084512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789311515301623:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.084593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.084624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301624:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
2025-11-28T14:52:42.084752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789307220334251:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.084751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789308377128262:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:42.084982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789304082160886:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.085015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:42.085048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789304082160886:2049] [node 2] ICP22 created new session: [3:7577789308377128281:2049]
2025-11-28T14:52:42.085077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789308377128281:2049] [node 2] ICS09 handshake done sender: [3:7577789308377128262:2117] self: [3:7577789308377128270:0] peer: [2:7577789308797413691:0] socket: 89 qp: -1
2025-11-28T14:52:42.085100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789308377128281:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:42.085141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.086516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789308377128257:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.086548Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789308377128257:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.088167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789308581226566:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11617
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.088438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789308377128257:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:42.088630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301626:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.088931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789308377128247:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.088978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.088978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301627:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
2025-11-28T14:52:42.089003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789308377128247:2109] [node 6] ICP22 created new session: [3:7577789308377128284:2109]
2025-11-28T14:52:42.089043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789308377128284:2109] [node 6] ICS09 handshake done sender: [3:7577789308377128257:2115] self: [3:7577789308377128256:0] peer: [6:7577789311515301625:0] socket: 124 qp: -1
2025-11-28T14:52:42.089070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789310140657389:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.089063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789308377128284:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:42.089081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:42.089921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789311515301626:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:42.089947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789310140657397:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-28T14:52:42.089998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301628:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
2025-11-28T14:52:42.090021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789305845690008:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.090193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008336:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.090216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789310730008336:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.091467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789310730008336:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:42.091707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789311515301624:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.091794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789310730008327:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.091826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789311515301627:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.091863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.091923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789310730008327:2108] [node 5] ICP22 created new session: [1:7577789310730008364:2108]
2025-11-28T14:52:42.091978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789310730008364:2108] [node 5] ICS09 handshake done sender: [1:7577789310730008336:2113] self: [1:7577789310730008335:0] peer: [5:7577789310140657398:0] socket: 119 qp: -1
2025-11-28T14:52:42.092018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789310730008364:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:42.092089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:42.092213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789311515301623:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:42.092628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789311515301606:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.093049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789311515301609:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.093121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789307220334249:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.093181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789307220334251:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.093228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.093274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789307220334251:2050] [node 3] ICP22 created new session: [6:7577789311515301633:2050]
2025-11-28T14:52:42.093313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789311515301633:2050] [node 3] ICS09 handshake done sender: [6:7577789311515301623:2115] self: [6:7577789311515301625:0] peer: [3:7577789308377128256:0] socket: 125 qp: -1
2025-11-28T14:52:42.093345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789311515301633:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:42.093396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.095421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789310140657397:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.095511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789305845690008:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789310140657389:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.095846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301629:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
2025-11-28T14:52:42.095906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789308581226542:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.095929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789310730008338:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.095936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789308581226542:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.095939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789310140657399:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-28T14:52:42.095957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789310730008338:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.097862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789308581226542:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:42.098060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789311515301629:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.098180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789308581226532:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.098193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789310730008338:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:42.098231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789310140657378:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.098254Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789310140657378:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.098249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.098297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789308581226532:2109] [node 6] ICP22 created new session: [4:7577789308581226568:2109]
2025-11-28T14:52:42.098349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789308581226568:2109] [node 6] ICS09 handshake done sender: [4:7577789308581226542:2114] self: [4:7577789308581226541:0] peer: [6:7577789311515301631:0] socket: 80 qp: -1
2025-11-28T14:52:42.098423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789308581226568:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:42.098453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:42.099086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789310730008328:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.099237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.099301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789310140657389:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.099298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789310730008328:2109] [node 6] ICP22 created new session: [1:7577789310730008367:2109]
2025-11-28T14:52:42.099355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789310730008367:2109] [node 6] ICS09 handshake done sender: [1:7577789310730008338:2114] self: [1:7577789310730008337:0] peer: [6:7577789311515301632:0] socket: 78 qp: -1
2025-11-28T14:52:42.099397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789310730008367:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.099532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789307220334250:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.099434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:42.099591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789305845690009:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.099611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301635:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
2025-11-28T14:52:42.099704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789305845690008:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.099735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.099764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789305845690008:2048] [node 1] ICP22 created new session: [5:7577789310140657402:2048]
2025-11-28T14:52:42.099798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789310140657402:2048] [node 1] ICS09 handshake done sender: [5:7577789310140657389:2119] self: [5:7577789310140657398:0] peer: [1:7577789310730008335:0] socket: 121 qp: -1
2025-11-28T14:52:42.099826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789310140657402:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:42.099882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.099990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301636:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.100273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789308581226570:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11617
2025-11-28T14:52:42.100590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308797413698:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.100596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789311515301624:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.100623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789308797413698:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.101406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789311515301627:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.101526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308797413696:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.101568Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789308797413696:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.101617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789310140657378:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:42.102313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789310140657368:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.102391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.102459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789310140657368:2109] [node 6] ICP22 created new session: [5:7577789310140657404:2109]
2025-11-28T14:52:42.102583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789310140657404:2109] [node 6] ICS09 handshake done sender: [5:7577789310140657378:2114] self: [5:7577789310140657377:0] peer: [6:7577789311515301630:0] socket: 66 qp: -1
2025-11-28T14:52:42.102604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308797413698:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:42.102605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789310140657404:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:42.102645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:42.102689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789311515301609:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.102720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.102749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789311515301609:2107] [node 4] ICP22 created new session: [6:7577789311515301640:2107]
2025-11-28T14:52:42.102828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789311515301640:2107] [node 4] ICS09 handshake done sender: [6:7577789311515301624:2116] self: [6:7577789311515301631:0] peer: [4:7577789308581226541:0] socket: 103 qp: -1
2025-11-28T14:52:42.102866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789311515301640:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:42.102917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.103038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308797413696:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:42.103274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789308797413688:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.103321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.103367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789308797413688:2109] [node 6] ICP22 created new session: [2:7577789308797413725:2109]
2025-11-28T14:52:42.103417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308797413725:2109] [node 6] ICS09 handshake done sender: [2:7577789308797413698:2114] self: [2:7577789308797413697:0] peer: [6:7577789311515301638:0] socket: 128 qp: -1
2025-11-28T14:52:42.103445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308797413725:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:42.103475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.104018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789310140657406:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-28T14:52:42.104285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789310140657397:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:42.104472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789305845690009:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.104522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:42.104498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789308797413687:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.104551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789305845690009:2049] [node 2] ICP22 created new session: [5:7577789310140657407:2049]
2025-11-28T14:52:42.104549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.104593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789310140657407:2049] [node 2] ICS09 handshake done sender: [5:7577789310140657397:2121] self: [5:7577789310140657401:0] peer: [2:7577789308797413695:0] socket: 127 qp: -1
2025-11-28T14:52:42.104603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789308797413687:2108] [node 5] ICP22 created new session: [2:7577789308797413727:2108]
2025-11-28T14:52:42.104611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789310140657407:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:42.104647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308797413727:2108] [node 5] ICS09 handshake done sender: [2:7577789308797413696:2113] self: [2:7577789308797413695:0] peer: [5:7577789310140657401:0] socket: 70 qp: -1
2025-11-28T14:52:42.104629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.104674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308797413727:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:42.104698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:42.106125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789307220334249:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.106184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.106219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789307220334249:2048] [node 1] ICP22 created new session: [6:7577789311515301642:2048]
2025-11-28T14:52:42.106247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789311515301642:2048] [node 1] ICS09 handshake done sender: [6:7577789311515301627:2118] self: [6:7577789311515301632:0] peer: [1:7577789310730008337:0] socket: 67 qp: -1
2025-11-28T14:52:42.106270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789311515301642:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:42.106298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:42.107024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789307220334249:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.108351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301639:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
2025-11-28T14:52:42.109989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789308581226532:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:42.111062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789311515301606:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789311515301626:2117] held: no
2025-11-28T14:52:42.111758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789307220334250:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15818) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789311515301629:2120] held: no
2025-11-28T14:52:42.112162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789311515301626:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.112822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789311515301644:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9309
2025-11-28T14:52:42.129849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789311515301606:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.129907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.129952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789311515301606:2106] [node 5] ICP22 created new session: [6:7577789311515301645:2106]
2025-11-28T14:52:42.129999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789311515301645:2106] [node 5] ICS09 handshake done sender: [6:7577789311515301626:2117] self: [6:7577789311515301630:0] peer: [5:7577789310140657377:0] socket: 73 qp: -1
2025-11-28T14:52:42.130039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789311515301645:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:42.130071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:42.131336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789311515301629:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:42.131617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789307220334250:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.131656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:42.131695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789307220334250:2049] [node 2] ICP22 created new session: [6:7577789311515301647:2049]
2025-11-28T14:52:42.131732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789311515301647:2049] [node 2] ICS09 handshake done sender: [6:7577789311515301629:2120] self: [6:7577789311515301638:0] peer: [2:7577789308797413697:0] socket: 130 qp: -1
2025-11-28T14:52:42.131760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789311515301647:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:42.131790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.145235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:42.145495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789308797413714:2048] [node 1] ICS07 socket disconnect 94 reason# EndOfStream
2025-11-28T14:52:42.145534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789308797413714:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.145639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.145702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789308797413714:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:42.146363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:42.146528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789308377128279:2048] [node 1] ICS07 socket disconnect 100 reason# EndOfStream
2025-11-28T14:52:42.146555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789308377128279:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.146638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.146678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789308377128279:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:42.148845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789304502446327:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Socket error# Connection reset by peer state# SendExRequestHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
2025-11-28T14:52:42.148902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:42.148881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789308797413714:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:42.149190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789308581226559:2048] [node 1] ICS07 socket disconnect 102 reason# EndOfStream
2025-11-28T14:52:42.149225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789308581226559:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.149312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.149365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789308581226559:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:42.150285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789304082160885:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:42.150339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789308377128279:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:42.150344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789304502446327:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.150381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789308797413714:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.150403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789304502446327:2048] [node 1] ICP30 unregister session Session# [2:7577789308797413714:2048] VirtualId# [2:7577789308797413710:0]
2025-11-28T14:52:42.150479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.150505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789304502446327:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.150514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.150486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:42.150650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789310140657402:2048] [node 1] ICS01 socket: 121 reason# EndOfStream
2025-11-28T14:52:42.150685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789305845690008:2048] [node 1] ICP30 unregister session Session# [5:7577789310140657402:2048] VirtualId# [5:7577789310140657398:0]
2025-11-28T14:52:42.150711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789310140657402:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.150719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789304286259172:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.150761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.150797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.150780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789308581226559:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.150799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789304286259172:2048] [node 1] ICP30 unregister session Session# [4:7577789308581226559:2048] VirtualId# [4:7577789308581226555:0]
2025-11-28T14:52:42.150881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.150940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789304286259172:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.150955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.151160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789304082160885:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.151225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789308377128279:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.151274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789304082160885:2048] [node 1] ICP30 unregister session Session# [3:7577789308377128279:2048] VirtualId# [3:7577789308377128271:0]
2025-11-28T14:52:42.151383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.151418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789304082160885:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.151360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:42.151435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.151486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789311515301642:2048] [node 1] ICS07 socket disconnect 67 reason# EndOfStream
2025-11-28T14:52:42.151507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789311515301642:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.151548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.151576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789311515301642:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:42.152108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789307220334249:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.152140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789311515301642:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.152160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789307220334249:2048] [node 1] ICP30 unregister session Session# [6:7577789311515301642:2048] VirtualId# [6:7577789311515301632:0]
2025-11-28T14:52:42.152205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.152221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789307220334249:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.152229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.159764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789304286259172:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:42.160779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789304082160885:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:42.161832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789307220334249:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:42.168810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:42.168979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789308377128281:2049] [node 2] ICS01 socket: 89 reason# EndOfStream
2025-11-28T14:52:42.169015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789304082160886:2049] [node 2] ICP30 unregister session Session# [3:7577789308377128281:2049] VirtualId# [3:7577789308377128270:0]
2025-11-28T14:52:42.169043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789308377128281:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.169141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:42.169184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:42.171275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:42.171406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789308581226564:2049] [node 2] ICS07 socket disconnect 91 reason# EndOfStream
2025-11-28T14:52:42.171433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789308581226564:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.171493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:42.171537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789308581226564:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:42.172216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789304286259173:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15818) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.172252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789308581226564:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.172275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789304286259173:2049] [node 2] ICP30 unregister session Session# [4:7577789308581226564:2049] VirtualId# [4:7577789308581226553:0]
2025-11-28T14:52:42.172223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:42.172303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:42.172325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789304286259173:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15818) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.172334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15818) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.172348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789310140657407:2049] [node 2] ICS07 socket disconnect 127 reason# EndOfStream
2025-11-28T14:52:42.172392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789310140657407:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.172463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:42.172509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789310140657407:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:42.173111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:42.173220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789305845690009:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15818) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.173254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789310140657407:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.173242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789311515301647:2049] [node 2] ICS01 socket: 130 reason# EndOfStream
2025-11-28T14:52:42.173277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789305845690009:2049] [node 2] ICP30 unregister session Session# [5:7577789310140657407:2049] VirtualId# [5:7577789310140657401:0]
2025-11-28T14:52:42.173301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:42.173274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789307220334250:2049] [node 2] ICP30 unregister session Session# [6:7577789311515301647:2049] VirtualId# [6:7577789311515301638:0]
2025-11-28T14:52:42.173316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789305845690009:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15818) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.173293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789311515301647:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.173324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15818) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.173335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:42.173363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:42.182297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789304286259173:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:42.183298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789305845690009:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:42.199945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:42.200118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789308581226551:2050] [node 3] ICS07 socket disconnect 69 reason# EndOfStream
2025-11-28T14:52:42.200168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789308581226551:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.200268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:42.200325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789308581226551:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:42.200796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:42.201017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789310140657390:2050] [node 3] ICS07 socket disconnect 112 reason# EndOfStream
2025-11-28T14:52:42.201049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789310140657390:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.201218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789304286259174:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64960) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.201136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:42.201275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789308581226551:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.201225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789310140657390:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:42.201370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789304286259174:2050] [node 3] ICP30 unregister session Session# [4:7577789308581226551:2050] VirtualId# [4:7577789308581226549:0]
2025-11-28T14:52:42.201456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:42.201489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789304286259174:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64960) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.201499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64960) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.201680Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:42.201871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789311515301633:2050] [node 3] ICS07 socket disconnect 125 reason# EndOfStream
2025-11-28T14:52:42.201899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789311515301633:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.201973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:42.202035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789311515301633:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:42.202168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789305845690010:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64960) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.202230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789310140657390:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.202267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789305845690010:2050] [node 3] ICP30 unregister session Session# [5:7577789310140657390:2050] VirtualId# [5:7577789310140657386:0]
2025-11-28T14:52:42.202332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:42.202360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789305845690010:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64960) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.202373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64960) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.202625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789307220334251:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64960) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.202657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789311515301633:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.202704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789307220334251:2050] [node 3] ICP30 unregister session Session# [6:7577789311515301633:2050] VirtualId# [6:7577789311515301625:0]
2025-11-28T14:52:42.202775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:42.202793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789307220334251:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64960) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.202801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64960) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.212020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789305845690010:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:42.213017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789307220334251:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:42.220652Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:42.220906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789310140657394:2108] [node 4] ICS07 socket disconnect 113 reason# EndOfStream
2025-11-28T14:52:42.220957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789310140657394:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.221064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:42.221113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789310140657394:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:42.221320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:42.221458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789311515301640:2107] [node 4] ICS07 socket disconnect 103 reason# EndOfStream
2025-11-28T14:52:42.221483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789311515301640:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.221548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:42.221582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789311515301640:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:42.221934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789310140657367:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24877) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.221986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789310140657394:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.222020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789310140657367:2108] [node 4] ICP30 unregister session Session# [5:7577789310140657394:2108] VirtualId# [5:7577789310140657388:0]
2025-11-28T14:52:42.222060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:42.222093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789310140657367:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.222110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.222202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789311515301609:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24877) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.222233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789311515301640:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.222257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789311515301609:2107] [node 4] ICP30 unregister session Session# [6:7577789311515301640:2107] VirtualId# [6:7577789311515301631:0]
2025-11-28T14:52:42.222335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:42.222363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789311515301609:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.222380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.231473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789310140657367:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:42.232444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789311515301609:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:42.257030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:42.257213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789311515301645:2106] [node 5] ICS01 socket: 73 reason# EndOfStream
2025-11-28T14:52:42.257247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789311515301606:2106] [node 5] ICP30 unregister session Session# [6:7577789311515301645:2106] VirtualId# [6:7577789311515301630:0]
2025-11-28T14:52:42.257279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789311515301645:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.257405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:42.257463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:42.344404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789309469871148:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:42.344463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789309469871149:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:42.367307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789308194551133:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.367347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789308194551134:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:42.388994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789311830155251:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.389122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789311830155252:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:42.411148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789311645952514:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:42.411182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789311645952515:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:42.411283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789311645952513:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.432593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789308176654363:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.432643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789308176654365:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:42.432789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789308176654364:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:42.453793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789310128691251:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.453824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789310128691252:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:42.453843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789310128691253:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:42.509104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789310128691312:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:42.525915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789309469871211:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:42.527698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789311830155314:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:42.528060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789311645952576:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:42.527720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789309469871212:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:42.528010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789308176654426:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:42.528027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789308194551196:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:42.527770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789311830155315:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:42.528226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789309469871213:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:42.528359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789311645952577:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:42.528126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789308194551197:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:42.528408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789308176654427:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:42.528517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789311830155316:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:42.528791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789308194551198:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:42.529358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789310128691315:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.538410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308194551212:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.539600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789308176654441:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-28T14:52:42.539859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308194551213:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
2025-11-28T14:52:42.540090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789311830155330:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
2025-11-28T14:52:42.540110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789311830155331:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.542788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308194551214:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:42.543914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789309469871222:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6684
::ffff:127.1.0.0
2025-11-28T14:52:42.543929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789311645952589:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49147
2025-11-28T14:52:42.544228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789309469871223:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6684
2025-11-28T14:52:42.544730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308194551212:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.544934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789311830155330:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.544437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789311645952588:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49147
2025-11-28T14:52:42.545403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871223:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.545486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789309469871227:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6684
2025-11-28T14:52:42.546963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789308176654441:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.546523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789309469871226:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6684
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.546961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871222:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.547656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789311830155331:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.547879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308194551214:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.547992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789311830155332:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
2025-11-28T14:52:42.548098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308194551215:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
2025-11-28T14:52:42.548115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789311645952588:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.548023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789311830155252:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.548276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308194551213:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.548288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308194551202:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.548300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789311645952593:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49147
2025-11-28T14:52:42.548306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789308194551202:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.548751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789311830155333:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
2025-11-28T14:52:42.549058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871227:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.549256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308194551216:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
2025-11-28T14:52:42.549289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871226:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.550414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308194551202:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.550537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789309469871231:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6684
2025-11-28T14:52:42.550738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789311645952589:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.550884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789311645952514:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.550865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789308194551134:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.550918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.551164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789308194551134:2049] [node 3] ICP22 created new session: [2:7577789308194551221:2049]
2025-11-28T14:52:42.551022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789311830155336:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
2025-11-28T14:52:42.551341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308194551221:2049] [node 3] ICS09 handshake done sender: [2:7577789308194551202:2111] self: [2:7577789308194551201:0] peer: [3:7577789311830155334:0] socket: 74 qp: -1
2025-11-28T14:52:42.551380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308194551221:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:42.551418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.551541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789311645952515:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.551558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789311830155333:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.552452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308194551216:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.554261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308194551204:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.554294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789308194551204:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.554864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789311830155323:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.554900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871231:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.554896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789311830155323:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.554989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789311830155330:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:42.556142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789311830155252:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8087) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789311830155330:2115] held: no
2025-11-28T14:52:42.556424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789311645952593:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.556716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789311645952513:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.556677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308194551215:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.556992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789308176654443:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-28T14:52:42.557045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871219:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.557010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308194551204:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.557082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789309469871219:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.557087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789311830155332:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.557142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789311830155252:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.557306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789311645952597:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49147
2025-11-28T14:52:42.557216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:42.557578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789308194551197:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.557336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789311830155252:2049] [node 2] ICP22 created new session: [3:7577789311830155339:2049]
2025-11-28T14:52:42.557624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.557825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789308194551197:2107] [node 4] ICP22 created new session: [2:7577789308194551226:2107]
2025-11-28T14:52:42.557580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789311830155339:2049] [node 2] ICS09 handshake done sender: [3:7577789311830155330:2115] self: [3:7577789311830155334:0] peer: [2:7577789308194551201:0] socket: 79 qp: -1
2025-11-28T14:52:42.557885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308194551226:2107] [node 4] ICS09 handshake done sender: [2:7577789308194551204:2112] self: [2:7577789308194551203:0] peer: [4:7577789311645952594:0] socket: 76 qp: -1
2025-11-28T14:52:42.557827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789311830155339:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:42.557923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308194551226:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:42.557858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.558000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.558250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789309469871219:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.558282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789311830155323:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.558382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789311645952588:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.558777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789309469871211:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.558904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789308176654444:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-28T14:52:42.558958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789311645952598:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49147
2025-11-28T14:52:42.558839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.558997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789311830155251:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.559064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789311645952514:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.558885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789309469871211:2107] [node 4] ICP22 created new session: [1:7577789309469871238:2107]
2025-11-28T14:52:42.559125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:42.559194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789311645952514:2049] [node 2] ICP22 created new session: [4:7577789311645952600:2049]
2025-11-28T14:52:42.559001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789309469871238:2107] [node 4] ICS09 handshake done sender: [1:7577789309469871219:2112] self: [1:7577789309469871218:0] peer: [4:7577789311645952596:0] socket: 103 qp: -1
2025-11-28T14:52:42.559491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789311645952600:2049] [node 2] ICS09 handshake done sender: [4:7577789311645952588:2115] self: [4:7577789311645952594:0] peer: [2:7577789308194551203:0] socket: 96 qp: -1
2025-11-28T14:52:42.559525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789311645952600:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:42.559032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789309469871238:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:42.559547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.559085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.559615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789311830155314:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.559674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.559822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789311830155314:2107] [node 4] ICP22 created new session: [3:7577789311830155343:2107]
2025-11-28T14:52:42.560013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789311830155343:2107] [node 4] ICS09 handshake done sender: [3:7577789311830155323:2112] self: [3:7577789311830155321:0] peer: [4:7577789311645952595:0] socket: 85 qp: -1
2025-11-28T14:52:42.560547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789308194551133:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.560578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789311830155343:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.560733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.561229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789308176654445:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-28T14:52:42.561228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789308176654443:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.561332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871215:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.561359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789308176654364:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.561362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789309469871215:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.562534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789308176654445:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.562565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871217:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.562539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308194551206:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.562587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789309469871217:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.562947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691329:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.563219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789308176654363:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.562568Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789308194551206:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.563371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789311645952599:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49147
2025-11-28T14:52:42.563920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789311645952589:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:42.564027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871221:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.564045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789309469871221:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.564728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789311645952513:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789311645952593:2117] held: no
2025-11-28T14:52:42.564781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789308176654448:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-28T14:52:42.564860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789309469871215:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:42.564956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789309469871217:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:42.565047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308194551206:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:42.565123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789309469871148:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.565169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.565347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789309469871148:2048] [node 2] ICP22 created new session: [1:7577789309469871244:2048]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:42.565859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789309469871244:2048] [node 2] ICS09 handshake done sender: [1:7577789309469871215:2110] self: [1:7577789309469871214:0] peer: [2:7577789308194551228:0] socket: 100 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.565879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789309469871244:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:42.565944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789311830155346:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23637
2025-11-28T14:52:42.565905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.566543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789308176654449:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-28T14:52:42.566402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789310128691329:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.566537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789308194551196:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.566534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691330:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.566558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789308176654443:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:42.566630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.566905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789309469871221:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:42.566678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789308194551196:2108] [node 5] ICP22 created new session: [2:7577789308194551232:2108]
2025-11-28T14:52:42.566703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789310128691252:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.566849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308194551232:2108] [node 5] ICS09 handshake done sender: [2:7577789308194551206:2113] self: [2:7577789308194551205:0] peer: [5:7577789308176654446:0] socket: 83 qp: -1
2025-11-28T14:52:42.567248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789308176654364:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.566871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308194551232:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:42.567409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:42.567133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:42.567454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789311645952515:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.567450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789308176654364:2049] [node 2] ICP22 created new session: [5:7577789308176654450:2049]
2025-11-28T14:52:42.567679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789311830155332:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.567574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.567677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789308176654450:2049] [node 2] ICS09 handshake done sender: [5:7577789308176654443:2116] self: [5:7577789308176654446:0] peer: [2:7577789308194551205:0] socket: 86 qp: -1
2025-11-28T14:52:42.567814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789310128691330:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.567835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789308176654450:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:42.567843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789311645952515:2050] [node 3] ICP22 created new session: [4:7577789311645952602:2050]
2025-11-28T14:52:42.567876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789311830155251:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.567967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789310128691253:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.567867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.567886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789311645952602:2050] [node 3] ICS09 handshake done sender: [4:7577789311645952589:2116] self: [4:7577789311645952595:0] peer: [3:7577789311830155321:0] socket: 98 qp: -1
2025-11-28T14:52:42.568045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.569355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789309469871149:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.567910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789311645952602:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:42.569395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.568103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789311830155251:2048] [node 1] ICP22 created new session: [3:7577789311830155347:2048]
2025-11-28T14:52:42.567934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.569505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789309469871149:2049] [node 3] ICP22 created new session: [1:7577789309469871246:2049]
2025-11-28T14:52:42.568207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789311830155347:2048] [node 1] ICS09 handshake done sender: [3:7577789311830155332:2117] self: [3:7577789311830155342:0] peer: [1:7577789309469871216:0] socket: 104 qp: -1
2025-11-28T14:52:42.570355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789309469871246:2049] [node 3] ICS09 handshake done sender: [1:7577789309469871217:2111] self: [1:7577789309469871216:0] peer: [3:7577789311830155342:0] socket: 101 qp: -1
2025-11-28T14:52:42.570391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789309469871246:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:42.568314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789311830155347:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:42.570500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789311645952593:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.570502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.568333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:42.570519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789311830155329:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.570551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789308194551208:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.571372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789308176654444:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.570584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789308194551208:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.570534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789311830155329:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.571511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789308176654445:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.572884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789311645952513:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.572928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.573039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789311645952513:2048] [node 1] ICP22 created new session: [4:7577789311645952606:2048]
2025-11-28T14:52:42.573182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789311645952606:2048] [node 1] ICS09 handshake done sender: [4:7577789311645952593:2117] self: [4:7577789311645952596:0] peer: [1:7577789309469871218:0] socket: 106 qp: -1
2025-11-28T14:52:42.573186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789308176654363:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.573289Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789311830155329:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:42.573250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.573218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789311645952606:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:42.573226Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789311645952515:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:42.573453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789308194551231:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
2025-11-28T14:52:42.573551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789309469871212:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.573289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789308176654363:2048] [node 1] ICP22 created new session: [5:7577789308176654452:2048]
2025-11-28T14:52:42.573245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:42.573406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789308176654452:2048] [node 1] ICS09 handshake done sender: [5:7577789308176654445:2118] self: [5:7577789308176654447:0] peer: [1:7577789309469871220:0] socket: 109 qp: -1
2025-11-28T14:52:42.573637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.573439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789308176654452:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:42.573666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789309469871212:2108] [node 5] ICP22 created new session: [1:7577789309469871248:2108]
2025-11-28T14:52:42.573912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789311830155316:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.573522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:42.573697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789309469871248:2108] [node 5] ICS09 handshake done sender: [1:7577789309469871221:2113] self: [1:7577789309469871220:0] peer: [5:7577789308176654447:0] socket: 110 qp: -1
2025-11-28T14:52:42.573756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789309469871248:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:42.574000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.573816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:42.574128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789311830155316:2109] [node 6] ICP22 created new session: [3:7577789311830155350:2109]
2025-11-28T14:52:42.574253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789311830155350:2109] [node 6] ICS09 handshake done sender: [3:7577789311830155329:2114] self: [3:7577789311830155328:0] peer: [6:7577789310128691332:0] socket: 128 qp: -1
2025-11-28T14:52:42.574259Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789308194551133:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789308194551215:2118] held: no
2025-11-28T14:52:42.574260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691333:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.574274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789311830155350:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:42.574297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: 2025-11-28T14:52:42.574885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789310128691330:2116] [node 3] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:42.574966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691334:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.574998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308194551208:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:42.575378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789310128691253:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.575597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.575672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789310128691253:2050] [node 3] ICP22 created new session: [6:7577789310128691335:2050]
2025-11-28T14:52:42.575731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789310128691335:2050] [node 3] ICS09 handshake done sender: [6:7577789310128691330:2116] self: [6:7577789310128691332:0] peer: [3:7577789311830155328:0] socket: 131 qp: -1
2025-11-28T14:52:42.575758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789310128691335:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:42.575792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.576174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789308194551215:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.576218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789310128691329:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:42.576391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789308194551198:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.576790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.576953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789308194551133:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.576977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.577668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789310128691252:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.577837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:42.576983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789308194551198:2109] [node 6] ICP22 created new session: [2:7577789308194551235:2109]
2025-11-28T14:52:42.577859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789310128691252:2049] [node 2] ICP22 created new session: [6:7577789310128691338:2049]
2025-11-28T14:52:42.577254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789308194551133:2048] [node 1] ICP22 created new session: [2:7577789308194551236:2048]
2025-11-28T14:52:42.577891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789310128691338:2049] [node 2] ICS09 handshake done sender: [6:7577789310128691329:2115] self: [6:7577789310128691331:0] peer: [2:7577789308194551207:0] socket: 124 qp: -1
2025-11-28T14:52:42.577578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308194551235:2109] [node 6] ICS09 handshake done sender: [2:7577789308194551208:2114] self: [2:7577789308194551207:0] peer: [6:7577789310128691331:0] socket: 105 qp: -1
2025-11-28T14:52:42.577925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789310128691338:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:42.577658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789308194551236:2048] [node 1] ICS09 handshake done sender: [2:7577789308194551215:2118] self: [2:7577789308194551228:0] peer: [1:7577789309469871214:0] socket: 102 qp: -1
2025-11-28T14:52:42.578041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:42.577663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308194551235:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:42.577685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789308194551236:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:42.577704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:42.577745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:42.578488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789308176654365:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.579129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789308176654455:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.579193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789311830155324:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.579224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789311830155324:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.579325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691336:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.579609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789308176654455:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.579929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789308176654426:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.580000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789311830155324:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:42.580608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789311645952608:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49147
2025-11-28T14:52:42.580693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789311830155315:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.580734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.580858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789311830155315:2108] [node 5] ICP22 created new session: [3:7577789311830155353:2108]
2025-11-28T14:52:42.581204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789311830155353:2108] [node 5] ICS09 handshake done sender: [3:7577789311830155324:2113] self: [3:7577789311830155322:0] peer: [5:7577789308176654456:0] socket: 121 qp: -1
2025-11-28T14:52:42.581387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789311830155353:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:42.581407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:42.581651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789310128691336:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.581667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789311645952585:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.581695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789311645952585:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.582640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789311645952608:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.582613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789308176654365:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789308176654444:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.583400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691340:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.583459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789311645952585:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.583605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691341:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.583612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789311645952610:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49147
2025-11-28T14:52:42.583717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789311645952577:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.583760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.583770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789310128691315:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.583864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789311645952577:2108] [node 5] ICP22 created new session: [4:7577789311645952612:2108]
2025-11-28T14:52:42.583892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789311645952612:2108] [node 5] ICS09 handshake done sender: [4:7577789311645952585:2113] self: [4:7577789311645952584:0] peer: [5:7577789308176654458:0] socket: 108 qp: -1
2025-11-28T14:52:42.583920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789311645952612:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:42.583948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.583989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789310128691340:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.584165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789310128691251:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:42.584157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789308176654459:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-28T14:52:42.584280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789310128691341:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.584319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789311645952587:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.584342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789311645952587:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.584597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789309469871225:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.584618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789310128691312:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.584623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789309469871225:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:42.585360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789308176654426:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64447) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789308176654455:2121] held: no
2025-11-28T14:52:42.585953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789308176654460:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-28T14:52:42.586065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789311645952610:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:42.586255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789311645952587:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.586364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789308176654444:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:42.586478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691346:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.586629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789311645952576:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.586633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789309469871225:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:42.586655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.586690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789311645952576:2109] [node 6] ICP22 created new session: [4:7577789311645952615:2109]
2025-11-28T14:52:42.586729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789311645952615:2109] [node 6] ICS09 handshake done sender: [4:7577789311645952587:2114] self: [4:7577789311645952586:0] peer: [6:7577789310128691343:0] socket: 87 qp: -1
2025-11-28T14:52:42.586752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789311645952615:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:42.586778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:42.586788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789309469871213:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.586778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789310128691336:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.586830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789308176654437:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:42.586864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789308176654437:2114] [node 6] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-28T14:52:42.586873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.587273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789309469871213:2109] [node 6] ICP22 created new session: [1:7577789309469871252:2109]
2025-11-28T14:52:42.587558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789309469871252:2109] [node 6] ICS09 handshake done sender: [1:7577789309469871225:2116] self: [1:7577789309469871224:0] peer: [6:7577789310128691344:0] socket: 91 qp: -1
2025-11-28T14:52:42.587586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789309469871252:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:42.587590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691347:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.587605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789310128691315:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.587607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789308176654365:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.587634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:42.587632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.587611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789308176654455:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:42.587662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789310128691315:2107] [node 4] ICP22 created new session: [6:7577789310128691348:2107]
2025-11-28T14:52:42.587651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:42.587689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789310128691348:2107] [node 4] ICS09 handshake done sender: [6:7577789310128691336:2119] self: [6:7577789310128691343:0] peer: [4:7577789311645952586:0] socket: 93 qp: -1
2025-11-28T14:52:42.587706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789310128691348:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:42.587692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789308176654365:2050] [node 3] ICP22 created new session: [5:7577789308176654461:2050]
2025-11-28T14:52:42.587736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789308176654461:2050] [node 3] ICS09 handshake done sender: [5:7577789308176654444:2117] self: [5:7577789308176654456:0] peer: [3:7577789311830155322:0] socket: 123 qp: -1
2025-11-28T14:52:42.587727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.587808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789308176654461:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:42.587910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:42.588175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789308176654426:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.588201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:42.588224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789308176654426:2108] [node 4] ICP22 created new session: [5:7577789308176654464:2108]
2025-11-28T14:52:42.588251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789308176654464:2108] [node 4] ICS09 handshake done sender: [5:7577789308176654455:2121] self: [5:7577789308176654458:0] peer: [4:7577789311645952584:0] socket: 75 qp: -1
2025-11-28T14:52:42.588262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789311645952576:2109] [node 6] ICP18 handshake (actor [4:7577789311645952610:2122]) is too old
2025-11-28T14:52:42.588271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789308176654464:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:42.588317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:42.589492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789308176654437:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:42.589599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789310128691340:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:42.589897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789308176654427:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:42.589985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:42.590105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789308176654427:2109] [node 6] ICP22 created new session: [5:7577789308176654466:2109]
2025-11-28T14:52:42.590157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789308176654466:2109] [node 6] ICS09 handshake done sender: [5:7577789308176654437:2114] self: [5:7577789308176654436:0] peer: [6:7577789310128691345:0] socket: 94 qp: -1
2025-11-28T14:52:42.590194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789308176654466:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:42.590229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:42.590311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789310128691251:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789310128691340:2120] held: no
2025-11-28T14:52:42.590838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789310128691251:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.590900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:42.590947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789310128691251:2048] [node 1] ICP22 created new session: [6:7577789310128691351:2048]
2025-11-28T14:52:42.590990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789310128691351:2048] [node 1] ICS09 handshake done sender: [6:7577789310128691340:2120] self: [6:7577789310128691344:0] peer: [1:7577789309469871224:0] socket: 133 qp: -1
2025-11-28T14:52:42.591022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789310128691351:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:42.591049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:42.591691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789310128691350:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46875
2025-11-28T14:52:42.591940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789310128691312:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2002) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789310128691341:2121] held: no
2025-11-28T14:52:42.592395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789310128691341:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:42.592493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789310128691312:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:42.592597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:42.592637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789310128691312:2106] [node 5] ICP22 created new session: [6:7577789310128691353:2106]
2025-11-28T14:52:42.592675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789310128691353:2106] [node 5] ICS09 handshake done sender: [6:7577789310128691341:2121] self: [6:7577789310128691345:0] peer: [5:7577789308176654436:0] socket: 134 qp: -1
2025-11-28T14:52:42.592703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789310128691353:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:42.592745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:42.629111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:42.629273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789308194551236:2048] [node 1] ICS01 socket: 102 reason# EndOfStream
2025-11-28T14:52:42.629307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789308194551133:2048] [node 1] ICP30 unregister session Session# [2:7577789308194551236:2048] VirtualId# [2:7577789308194551228:0]
2025-11-28T14:52:42.629330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789308194551236:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.629390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.629431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.630443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:42.630620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789311830155347:2048] [node 1] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-28T14:52:42.630648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789311830155347:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.630730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.630769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789311830155347:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:42.632419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789311830155251:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:42.632842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789311830155347:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:42.633103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:42.633302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789311645952606:2048] [node 1] ICS01 socket: 106 reason# EndOfStream
2025-11-28T14:52:42.633341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789311645952513:2048] [node 1] ICP30 unregister session Session# [4:7577789311645952606:2048] VirtualId# [4:7577789311645952596:0]
2025-11-28T14:52:42.633365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789311645952606:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.633423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.633456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.633618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789311830155251:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.633709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789311830155347:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.633871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789311830155251:2048] [node 1] ICP30 unregister session Session# [3:7577789311830155347:2048] VirtualId# [3:7577789311830155342:0]
2025-11-28T14:52:42.633906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.633928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789311830155251:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.633943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.634597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:42.634832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789308176654452:2048] [node 1] ICS07 socket disconnect 109 reason# EndOfStream
2025-11-28T14:52:42.634873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789308176654452:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.634956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.635013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789308176654452:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:42.635447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:42.635574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789310128691351:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-11-28T14:52:42.635593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789308176654363:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.635633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789308176654452:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.635602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789310128691251:2048] [node 1] ICP30 unregister session Session# [6:7577789310128691351:2048] VirtualId# [6:7577789310128691344:0]
2025-11-28T14:52:42.635687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789308176654363:2048] [node 1] ICP30 unregister session Session# [5:7577789308176654452:2048] VirtualId# [5:7577789308176654447:0]
2025-11-28T14:52:42.635621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789310128691351:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.635766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.635679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:42.635800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789308176654363:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.635714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:42.635809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7194) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.644089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789311830155251:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:42.645155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789308176654363:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:42.677792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:42.677971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789311830155339:2049] [node 2] ICS01 socket: 79 reason# EndOfStream
2025-11-28T14:52:42.678007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789311830155252:2049] [node 2] ICP30 unregister session Session# [3:7577789311830155339:2049] VirtualId# [3:7577789311830155334:0]
2025-11-28T14:52:42.678030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789311830155339:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.678120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:42.678164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:42.680934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:42.681196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789311645952600:2049] [node 2] ICS07 socket disconnect 96 reason# EndOfStream
2025-11-28T14:52:42.681240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789311645952600:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.681326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:42.681373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789311645952600:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:42.682050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789311645952514:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8087) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.682026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:42.682088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789311645952600:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.682111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789311645952514:2049] [node 2] ICP30 unregister session Session# [4:7577789311645952600:2049] VirtualId# [4:7577789311645952594:0]
2025-11-28T14:52:42.682250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:42.682275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789311645952514:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8087) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.682285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8087) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.682270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789308176654450:2049] [node 2] ICS07 socket disconnect 86 reason# EndOfStream
2025-11-28T14:52:42.682308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789308176654450:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.682401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:42.682560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789308176654450:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:42.683079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:42.683269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789310128691338:2049] [node 2] ICS01 socket: 124 reason# EndOfStream
2025-11-28T14:52:42.683269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789308176654364:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8087) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.683306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789308176654450:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.683320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789310128691252:2049] [node 2] ICP30 unregister session Session# [6:7577789310128691338:2049] VirtualId# [6:7577789310128691331:0]
2025-11-28T14:52:42.683367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789308176654364:2049] [node 2] ICP30 unregister session Session# [5:7577789308176654450:2049] VirtualId# [5:7577789308176654446:0]
2025-11-28T14:52:42.683371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789310128691338:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.683440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:42.683507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:42.683497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:42.683528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789308176654364:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8087) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.683541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8087) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.692285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789311645952514:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:42.693277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789308176654364:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:42.740005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:42.740165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789311645952602:2050] [node 3] ICS07 socket disconnect 98 reason# EndOfStream
2025-11-28T14:52:42.740189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789311645952602:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.740252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:42.740301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789311645952602:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:42.740823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:42.740901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789311645952515:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.741000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789311645952602:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.741032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789311645952515:2050] [node 3] ICP30 unregister session Session# [4:7577789311645952602:2050] VirtualId# [4:7577789311645952595:0]
2025-11-28T14:52:42.741117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789308176654461:2050] [node 3] ICS01 socket: 123 reason# EndOfStream
2025-11-28T14:52:42.741138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:42.741166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789311645952515:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.741175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.741177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789308176654365:2050] [node 3] ICP30 unregister session Session# [5:7577789308176654461:2050] VirtualId# [5:7577789308176654456:0]
2025-11-28T14:52:42.741226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789308176654461:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.741325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:42.741402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:42.741557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:42.741703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789310128691335:2050] [node 3] ICS07 socket disconnect 131 reason# EndOfStream
2025-11-28T14:52:42.741729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789310128691335:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.741790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:42.741830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789310128691335:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:42.742359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789310128691253:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.742424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789310128691335:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.742457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789310128691253:2050] [node 3] ICP30 unregister session Session# [6:7577789310128691335:2050] VirtualId# [6:7577789310128691332:0]
2025-11-28T14:52:42.742544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:42.742569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789310128691253:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.742579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21852) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.752674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789310128691253:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:42.776988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:42.777271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789308176654464:2108] [node 4] ICS01 socket: 75 reason# EndOfStream
2025-11-28T14:52:42.777345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789308176654426:2108] [node 4] ICP30 unregister session Session# [5:7577789308176654464:2108] VirtualId# [5:7577789308176654458:0]
2025-11-28T14:52:42.777386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789308176654464:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.777516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:42.777582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:42.777796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:42.777947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789310128691348:2107] [node 4] ICS07 socket disconnect 93 reason# EndOfStream
2025-11-28T14:52:42.777973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789310128691348:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.778030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:42.778077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789310128691348:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:42.778653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789310128691315:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64447) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:42.778765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789310128691348:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:42.778834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789310128691315:2107] [node 4] ICP30 unregister session Session# [6:7577789310128691348:2107] VirtualId# [6:7577789310128691343:0]
2025-11-28T14:52:42.778948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:42.778979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789310128691315:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64447) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.778989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64447) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:42.788532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789310128691315:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:42.813655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:42.813901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789310128691353:2106] [node 5] ICS01 socket: 134 reason# EndOfStream
2025-11-28T14:52:42.813979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789310128691312:2106] [node 5] ICP30 unregister session Session# [6:7577789310128691353:2106] VirtualId# [6:7577789310128691345:0]
2025-11-28T14:52:42.814021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789310128691353:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:42.814118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:42.814167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:42.887388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789308658834992:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:42.887412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789308658834993:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:42.904272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789310248669093:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.904485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789310248669094:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:42.921323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789311146439794:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.921367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789311146439795:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:42.938113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789309208953804:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.938144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789309208953805:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:42.938169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789309208953806:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:42.956600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789308760134027:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.956638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789308760134028:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:42.956715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789308760134029:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:42.972624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789308243631418:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:42.972683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789308243631420:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:42.972705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789308243631419:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:43.041101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789312953802351:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:43.049289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789312953802352:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:43.049289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789312538598775:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:43.050246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789312953802353:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:43.050724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789315441407153:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:43.051079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789312538598778:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:43.050913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789315441407154:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:43.050909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789314543636453:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:43.051025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789314543636452:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:43.051637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789313055101386:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:43.051243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789315441407155:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:43.051943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789313055101387:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:43.051581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789313503921163:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:43.051901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789314543636454:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:43.051601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789313503921164:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:43.062921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313503921172:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44046
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.062930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313503921171:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44046
2025-11-28T14:52:43.063432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789312953802367:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32315
2025-11-28T14:52:43.063660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313503921171:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.063773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789309208953804:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.063945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313503921172:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.063966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789312953802368:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32315
2025-11-28T14:52:43.064001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312953802357:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.063944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314543636461:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37157
2025-11-28T14:52:43.064572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314543636462:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37157
2025-11-28T14:52:43.064022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312953802357:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.065248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314543636461:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.065424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312953802357:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:43.065473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314543636465:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37157
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.065680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314543636462:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.065962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789310248669093:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.067055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314543636466:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37157
2025-11-28T14:52:43.067093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789312953802351:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.068787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314543636465:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.067333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.067883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789312953802351:2107] [node 4] ICP22 created new session: [1:7577789312953802371:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.069320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313503921182:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44046
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.068187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312953802355:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.068212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312953802355:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.069671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789315441407162:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61936
2025-11-28T14:52:43.069705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598792:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.068947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312953802371:2107] [node 4] ICS09 handshake done sender: [1:7577789312953802357:2111] self: [1:7577789312953802356:0] peer: [4:7577789313503921180:0] socket: 82 qp: -1
2025-11-28T14:52:43.069727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789315441407163:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61936
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.070352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313503921171:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:43.070567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789315441407164:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61936
2025-11-28T14:52:43.068988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312953802371:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.070815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789315441407168:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61936
2025-11-28T14:52:43.069030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.071261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789315441407167:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61936
2025-11-28T14:52:43.069629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789312953802370:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32315
2025-11-28T14:52:43.071339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789309208953804:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.071641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789315441407162:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.069766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312953802368:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.069775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312953802367:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:43.070394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312953802355:2110] [node 2] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:43.071672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789308658834992:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.071656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598793:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-28T14:52:43.071701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:43.071750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.071770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314543636466:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.071786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789315441407163:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.071761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789309208953804:2048] [node 1] ICP22 created new session: [4:7577789313503921183:2048]
2025-11-28T14:52:43.071798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789308658834992:2048] [node 2] ICP22 created new session: [1:7577789312953802375:2048]
2025-11-28T14:52:43.073132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789312538598792:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.072766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313503921183:2048] [node 1] ICS09 handshake done sender: [4:7577789313503921171:2113] self: [4:7577789313503921180:0] peer: [1:7577789312953802356:0] socket: 86 qp: -1
2025-11-28T14:52:43.072509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314543636475:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37157
2025-11-28T14:52:43.071903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789315441407164:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.072975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313503921183:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:43.073476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314543636461:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:43.072534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312953802375:2048] [node 2] ICS09 handshake done sender: [1:7577789312953802355:2110] self: [1:7577789312953802354:0] peer: [2:7577789314543636472:0] socket: 80 qp: -1
2025-11-28T14:52:43.071907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789315441407168:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.072956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312953802375:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:43.073968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314543636476:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37157
2025-11-28T14:52:43.072553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789315441407167:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.073695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789308243631419:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.073077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789311146439795:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:52:43.073124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:43.072992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:43.073145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789311146439794:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-28T14:52:43.074197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314543636459:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.074233Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789314543636459:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.074371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312953802370:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.074423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313503921184:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44046
2025-11-28T14:52:43.074350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789310248669093:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.075185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312953802359:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.074738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314543636458:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.074756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789314543636458:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.075213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312953802359:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.075269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.075298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789310248669093:2048] [node 1] ICP22 created new session: [2:7577789314543636478:2048]
2025-11-28T14:52:43.077991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789312538598793:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.075869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314543636478:2048] [node 1] ICS09 handshake done sender: [2:7577789314543636461:2113] self: [2:7577789314543636472:0] peer: [1:7577789312953802354:0] socket: 87 qp: -1
2025-11-28T14:52:43.076036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314543636478:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:43.078313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789308243631418:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.078546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313503921184:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.076062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:43.078815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789309208953806:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.078404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598795:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-28T14:52:43.077824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314543636458:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.078808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314543636459:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:43.078922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789312538598795:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.079160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789315441407158:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.079465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789314543636453:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.079436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789312538598778:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.080507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312953802363:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.080593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313503921174:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.079209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789315441407158:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.080484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:43.079456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598796:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-28T14:52:43.080547Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312953802363:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.080255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789315441407179:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61936
2025-11-28T14:52:43.080626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789313503921174:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.081331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789314543636453:2109] [node 6] ICP22 created new session: [2:7577789314543636482:2109]
2025-11-28T14:52:43.082006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789312538598792:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:43.081363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314543636476:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.082168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789315441407158:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:43.081522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789310248669094:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.082353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313503921174:2115] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:43.082515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789315441407168:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.081718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314543636482:2109] [node 6] ICS09 handshake done sender: [2:7577789314543636458:2110] self: [2:7577789314543636455:0] peer: [6:7577789312538598794:0] socket: 107 qp: -1
2025-11-28T14:52:43.081996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314543636482:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.082540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789308243631419:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.082551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313055101401:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62576
2025-11-28T14:52:43.082869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789312953802351:2107] [node 4] ICP18 handshake (actor [1:7577789312953802368:2116]) is too old
2025-11-28T14:52:43.082944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.082015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.082022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:43.082869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789312953802378:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32315
2025-11-28T14:52:43.083114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789313503921164:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.082975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789308243631419:2049] [node 2] ICP22 created new session: [6:7577789312538598800:2049]
2025-11-28T14:52:43.082870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789315441407181:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61936
2025-11-28T14:52:43.083764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789311146439795:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.083113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312953802359:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:43.083740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313503921189:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44046
2025-11-28T14:52:43.082047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789310248669094:2049] [node 3] ICP22 created new session: [2:7577789314543636483:2049]
2025-11-28T14:52:43.084034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789312538598800:2049] [node 2] ICS09 handshake done sender: [6:7577789312538598792:2115] self: [6:7577789312538598794:0] peer: [2:7577789314543636455:0] socket: 114 qp: -1
2025-11-28T14:52:43.084675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789308658834993:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.083925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.083768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:43.082493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314543636483:2049] [node 3] ICS09 handshake done sender: [2:7577789314543636459:2111] self: [2:7577789314543636456:0] peer: [3:7577789315441407177:0] socket: 109 qp: -1
2025-11-28T14:52:43.083966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789311146439795:2049] [node 2] ICP22 created new session: [3:7577789315441407182:2049]
2025-11-28T14:52:43.083890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789313503921164:2109] [node 6] ICP22 created new session: [4:7577789313503921190:2109]
2025-11-28T14:52:43.082518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314543636483:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:43.085405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:43.084199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789312538598800:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:43.084207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313503921190:2109] [node 6] ICS09 handshake done sender: [4:7577789313503921174:2115] self: [4:7577789313503921173:0] peer: [6:7577789312538598798:0] socket: 125 qp: -1
2025-11-28T14:52:43.082542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.085454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789308658834993:2049] [node 3] ICP22 created new session: [1:7577789312953802382:2049]
2025-11-28T14:52:43.084037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789315441407153:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.084233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313503921190:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.084223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:43.084066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789315441407182:2049] [node 2] ICS09 handshake done sender: [3:7577789315441407168:2118] self: [3:7577789315441407177:0] peer: [2:7577789314543636456:0] socket: 111 qp: -1
2025-11-28T14:52:43.085819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312953802382:2049] [node 3] ICS09 handshake done sender: [1:7577789312953802359:2112] self: [1:7577789312953802358:0] peer: [3:7577789315441407174:0] socket: 83 qp: -1
2025-11-28T14:52:43.084634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:43.084219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789315441407182:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:43.086372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312953802382:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:43.086997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313503921184:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:43.085122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:43.086579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.085170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:43.088416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312953802363:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:43.085206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789315441407153:2107] [node 4] ICP22 created new session: [3:7577789315441407184:2107]
2025-11-28T14:52:43.088240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598799:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.086336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789315441407184:2107] [node 4] ICS09 handshake done sender: [3:7577789315441407158:2110] self: [3:7577789315441407157:0] peer: [4:7577789313503921187:0] socket: 120 qp: -1
2025-11-28T14:52:43.088975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312953802378:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.088817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789309208953806:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.086365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789315441407184:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:43.089208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598802:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-28T14:52:43.086556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.089305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:43.089463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789309208953806:2050] [node 3] ICP22 created new session: [4:7577789313503921193:2050]
2025-11-28T14:52:43.089454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789312953802353:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.089576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789308658834992:2048] [node 2] ICP18 handshake (actor [1:7577789312953802370:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.089619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313503921193:2050] [node 3] ICS09 handshake done sender: [4:7577789313503921184:2118] self: [4:7577789313503921187:0] peer: [3:7577789315441407157:0] socket: 122 qp: -1
2025-11-28T14:52:43.089577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313055101401:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.089671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:43.089659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313503921193:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:43.089677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789308760134027:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.089687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.089701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313055101402:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62576
2025-11-28T14:52:43.089743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789312953802353:2109] [node 6] ICP22 created new session: [1:7577789312953802384:2109]
2025-11-28T14:52:43.089731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789312538598795:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:43.089906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312953802384:2109] [node 6] ICS09 handshake done sender: [1:7577789312953802363:2114] self: [1:7577789312953802360:0] peer: [6:7577789312538598797:0] socket: 116 qp: -1
2025-11-28T14:52:43.089936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312953802384:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.090055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789312538598793:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:43.090086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.090761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789312538598778:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.090959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:43.090978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313055101404:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62576
2025-11-28T14:52:43.090987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789312538598778:2107] [node 4] ICP22 created new session: [6:7577789312538598804:2107]
2025-11-28T14:52:43.091021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789312538598804:2107] [node 4] ICS09 handshake done sender: [6:7577789312538598795:2117] self: [6:7577789312538598798:0] peer: [4:7577789313503921173:0] socket: 126 qp: -1
2025-11-28T14:52:43.091023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789315441407163:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:43.091021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313055101405:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62576
2025-11-28T14:52:43.091043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789312538598804:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:43.091062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789310248669094:2049] [node 3] ICP18 handshake (actor [2:7577789314543636476:2120]) is too old
2025-11-28T14:52:43.091064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.091269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789311146439794:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.091328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789308243631418:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.091428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:43.091459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:43.091464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789311146439794:2048] [node 1] ICP22 created new session: [3:7577789315441407187:2048]
2025-11-28T14:52:43.091495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789308243631418:2048] [node 1] ICP22 created new session: [6:7577789312538598806:2048]
2025-11-28T14:52:43.091500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789315441407187:2048] [node 1] ICS09 handshake done sender: [3:7577789315441407163:2114] self: [3:7577789315441407174:0] peer: [1:7577789312953802358:0] socket: 96 qp: -1
2025-11-28T14:52:43.091529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789315441407187:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:43.091523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789312538598806:2048] [node 1] ICS09 handshake done sender: [6:7577789312538598793:2116] self: [6:7577789312538598797:0] peer: [1:7577789312953802360:0] socket: 117 qp: -1
2025-11-28T14:52:43.091558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:43.091659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312953802362:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.091540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789312538598806:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:43.091692Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312953802362:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.091597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.093050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789308658834993:2049] [node 3] ICP18 handshake (actor [1:7577789312953802378:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.094871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313055101406:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62576
2025-11-28T14:52:43.095464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313055101402:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.095457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313503921195:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44046
2025-11-28T14:52:43.095674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789313055101386:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.096006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313503921178:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.096041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789313503921178:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.098363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312953802362:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.098476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789308243631418:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:43.098775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313055101408:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62576
2025-11-28T14:52:43.098777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313055101405:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.098829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789312953802352:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.098874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.098932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789312953802352:2108] [node 5] ICP22 created new session: [1:7577789312953802387:2108]
2025-11-28T14:52:43.098947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789308760134028:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.098989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312953802387:2108] [node 5] ICS09 handshake done sender: [1:7577789312953802362:2113] self: [1:7577789312953802361:0] peer: [5:7577789313055101403:0] socket: 110 qp: -1
2025-11-28T14:52:43.099035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312953802387:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:43.099071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.099126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313055101401:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:43.099235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313503921195:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.099529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313503921178:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:43.099502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314543636464:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.099948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789308760134027:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.100016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:43.099536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789314543636464:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.100043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789308760134027:2048] [node 1] ICP22 created new session: [5:7577789313055101411:2048]
2025-11-28T14:52:43.100272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313055101411:2048] [node 1] ICS09 handshake done sender: [5:7577789313055101401:2115] self: [5:7577789313055101403:0] peer: [1:7577789312953802361:0] socket: 132 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.100378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789313503921163:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.100478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314543636464:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:43.100307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313055101411:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:43.100391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789309208953805:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.100326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:43.100870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.101299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314543636468:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.101072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789313503921163:2108] [node 5] ICP22 created new session: [4:7577789313503921198:2108]
2025-11-28T14:52:43.101325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789314543636468:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.101069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313055101410:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62576
2025-11-28T14:52:43.102426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313503921198:2108] [node 5] ICS09 handshake done sender: [4:7577789313503921178:2116] self: [4:7577789313503921175:0] peer: [5:7577789313055101407:0] socket: 90 qp: -1
2025-11-28T14:52:43.102605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789314543636452:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.102469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313503921198:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:43.102878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.103036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.103090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789314543636452:2108] [node 5] ICP22 created new session: [2:7577789314543636490:2108]
2025-11-28T14:52:43.104405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314543636490:2108] [node 5] ICS09 handshake done sender: [2:7577789314543636464:2115] self: [2:7577789314543636463:0] peer: [5:7577789313055101409:0] socket: 140 qp: -1
2025-11-28T14:52:43.104751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314543636490:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:43.104758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314543636468:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.104802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.105261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598808:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-28T14:52:43.105273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313503921200:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44046
2025-11-28T14:52:43.105103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789313055101386:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3756) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789313055101402:2116] held: no
2025-11-28T14:52:43.105670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313503921195:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:43.105775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789312538598808:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.106530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789309208953805:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.106671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.106693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789314543636454:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.106023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789308243631420:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.106780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789309208953805:2049] [node 2] ICP22 created new session: [4:7577789313503921201:2049]
2025-11-28T14:52:43.107233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:43.107355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313503921201:2049] [node 2] ICS09 handshake done sender: [4:7577789313503921195:2120] self: [4:7577789313503921197:0] peer: [2:7577789314543636467:0] socket: 93 qp: -1
2025-11-28T14:52:43.107260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789314543636454:2107] [node 4] ICP22 created new session: [2:7577789314543636492:2107]
2025-11-28T14:52:43.107346Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313055101402:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:43.107366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789315441407170:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.107398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313503921201:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:43.107475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314543636492:2107] [node 4] ICS09 handshake done sender: [2:7577789314543636468:2118] self: [2:7577789314543636467:0] peer: [4:7577789313503921197:0] socket: 141 qp: -1
2025-11-28T14:52:43.107522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313055101406:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.107416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789315441407170:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.107569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314543636492:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:43.107693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:43.108984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789308760134028:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789313055101405:2118] held: no
2025-11-28T14:52:43.107693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.109203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598810:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-28T14:52:43.109341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789315441407170:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:43.109592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789315441407155:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.109763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:43.109744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598811:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-28T14:52:43.109881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789315441407155:2109] [node 6] ICP22 created new session: [3:7577789315441407191:2109]
2025-11-28T14:52:43.109941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789315441407191:2109] [node 6] ICS09 handshake done sender: [3:7577789315441407170:2119] self: [3:7577789315441407169:0] peer: [6:7577789312538598809:0] socket: 123 qp: -1
2025-11-28T14:52:43.110063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313055101404:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:43.109988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789312538598808:2121] [node 3] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:43.109980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789315441407191:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.110194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789313055101386:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.110099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:43.110910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313055101413:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62576
2025-11-28T14:52:43.110953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:43.111126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789313055101386:2108] [node 4] ICP22 created new session: [5:7577789313055101415:2108]
2025-11-28T14:52:43.110991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789308243631420:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789312538598808:2121] held: no
2025-11-28T14:52:43.111447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313055101415:2108] [node 4] ICS09 handshake done sender: [5:7577789313055101402:2116] self: [5:7577789313055101407:0] peer: [4:7577789313503921175:0] socket: 102 qp: -1
2025-11-28T14:52:43.111507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313055101415:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:43.111584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.111628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789308760134029:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.111842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313055101405:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:43.112035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789308243631420:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.112185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:43.112218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789308243631420:2050] [node 3] ICP22 created new session: [6:7577789312538598812:2050]
2025-11-28T14:52:43.112271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789312538598812:2050] [node 3] ICS09 handshake done sender: [6:7577789312538598808:2121] self: [6:7577789312538598809:0] peer: [3:7577789315441407169:0] socket: 144 qp: -1
2025-11-28T14:52:43.112274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789308760134028:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.112304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789312538598812:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:43.112383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.112359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.112402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789315441407166:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.112445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789315441407166:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.112438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789308760134028:2049] [node 2] ICP22 created new session: [5:7577789313055101419:2049]
2025-11-28T14:52:43.112833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313055101419:2049] [node 2] ICS09 handshake done sender: [5:7577789313055101405:2118] self: [5:7577789313055101409:0] peer: [2:7577789314543636463:0] socket: 108 qp: -1
2025-11-28T14:52:43.112868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313055101419:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:43.112908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:43.113497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789312538598810:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.113621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789312538598775:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.113774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313055101395:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.113805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789313055101395:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.113868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789315441407166:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.114352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313055101421:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62576
2025-11-28T14:52:43.114178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789315441407154:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.114429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.114501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789315441407154:2108] [node 5] ICP22 created new session: [3:7577789315441407194:2108]
2025-11-28T14:52:43.114548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789315441407194:2108] [node 5] ICS09 handshake done sender: [3:7577789315441407166:2116] self: [3:7577789315441407165:0] peer: [5:7577789313055101417:0] socket: 97 qp: -1
2025-11-28T14:52:43.114587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789315441407194:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:43.114634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.114600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313055101406:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:43.114622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313055101395:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:43.114962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789308760134029:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.115186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789313055101387:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.115201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:43.115253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789308760134029:2050] [node 3] ICP22 created new session: [5:7577789313055101423:2050]
2025-11-28T14:52:43.115302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:43.115315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313055101423:2050] [node 3] ICS09 handshake done sender: [5:7577789313055101406:2119] self: [5:7577789313055101417:0] peer: [3:7577789315441407165:0] socket: 112 qp: -1
2025-11-28T14:52:43.115343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789313055101387:2109] [node 6] ICP22 created new session: [5:7577789313055101424:2109]
2025-11-28T14:52:43.115295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789312538598815:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-28T14:52:43.115355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313055101423:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:43.115380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313055101424:2109] [node 6] ICS09 handshake done sender: [5:7577789313055101395:2113] self: [5:7577789313055101394:0] peer: [6:7577789312538598814:0] socket: 79 qp: -1
2025-11-28T14:52:43.115391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.115528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789312538598810:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:43.115409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313055101424:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.115688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789312538598775:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.115439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:43.116906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.116953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789312538598775:2106] [node 5] ICP22 created new session: [6:7577789312538598816:2106]
2025-11-28T14:52:43.116986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789312538598816:2106] [node 5] ICS09 handshake done sender: [6:7577789312538598810:2122] self: [6:7577789312538598814:0] peer: [5:7577789313055101394:0] socket: 81 qp: -1
2025-11-28T14:52:43.117008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789312538598816:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:43.117027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.118271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789308760134029:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:43.158280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:43.158557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789314543636478:2048] [node 1] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-28T14:52:43.158614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789314543636478:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.158728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.158789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789314543636478:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:43.159339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:43.159581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789315441407187:2048] [node 1] ICS07 socket disconnect 96 reason# EndOfStream
2025-11-28T14:52:43.159616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789315441407187:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.159703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.159758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789315441407187:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:43.160397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789311146439794:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.160744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789315441407187:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.160978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789311146439794:2048] [node 1] ICP30 unregister session Session# [3:7577789315441407187:2048] VirtualId# [3:7577789315441407174:0]
2025-11-28T14:52:43.161425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.161481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789311146439794:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.161398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789310248669093:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:43.161467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789314543636478:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:43.161501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.162212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:43.162406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789313503921183:2048] [node 1] ICS07 socket disconnect 86 reason# EndOfStream
2025-11-28T14:52:43.162444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789313503921183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.162503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.162565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789313503921183:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:43.162914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789310248669093:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.162959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789314543636478:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.163034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789310248669093:2048] [node 1] ICP30 unregister session Session# [2:7577789314543636478:2048] VirtualId# [2:7577789314543636472:0]
2025-11-28T14:52:43.163119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.163146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789310248669093:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.163155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.163162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789309208953804:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.163260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789313503921183:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.163300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789309208953804:2048] [node 1] ICP30 unregister session Session# [4:7577789313503921183:2048] VirtualId# [4:7577789313503921180:0]
2025-11-28T14:52:43.163319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:43.163388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.163420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789309208953804:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.163428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.163546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789313055101411:2048] [node 1] ICS07 socket disconnect 132 reason# EndOfStream
2025-11-28T14:52:43.163577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789313055101411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.163633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.163726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789313055101411:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:43.164115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:43.164194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789308760134027:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.164240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789313055101411:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.164292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789312538598806:2048] [node 1] ICS07 socket disconnect 117 reason# EndOfStream
2025-11-28T14:52:43.164265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789308760134027:2048] [node 1] ICP30 unregister session Session# [5:7577789313055101411:2048] VirtualId# [5:7577789313055101403:0]
2025-11-28T14:52:43.164406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.164321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789312538598806:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.164425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789308760134027:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.164437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.164411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.164449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789312538598806:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:43.164944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789308243631418:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.164972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789312538598806:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.165020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789308243631418:2048] [node 1] ICP30 unregister session Session# [6:7577789312538598806:2048] VirtualId# [6:7577789312538598797:0]
2025-11-28T14:52:43.165095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.165112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789308243631418:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.165120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15230) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.171510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789311146439794:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:43.172530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789310248669093:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:43.173480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789309208953804:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:43.174495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789308760134027:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:43.174518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789308243631418:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:43.208864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:43.209099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789315441407182:2049] [node 2] ICS07 socket disconnect 111 reason# EndOfStream
2025-11-28T14:52:43.209140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789315441407182:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.209273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:43.209330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789315441407182:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:43.210039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789311146439795:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.210122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789315441407182:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.210165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789311146439795:2049] [node 2] ICP30 unregister session Session# [3:7577789315441407182:2049] VirtualId# [3:7577789315441407177:0]
2025-11-28T14:52:43.210297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:43.210337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789311146439795:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.210356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.210914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:43.211152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789313503921201:2049] [node 2] ICS07 socket disconnect 93 reason# EndOfStream
2025-11-28T14:52:43.211193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789313503921201:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.211289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:43.211339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789313503921201:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:43.211558Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:43.211687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789313055101419:2049] [node 2] ICS01 socket: 108 reason# EndOfStream
2025-11-28T14:52:43.211717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789308760134028:2049] [node 2] ICP30 unregister session Session# [5:7577789313055101419:2049] VirtualId# [5:7577789313055101409:0]
2025-11-28T14:52:43.211738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789313055101419:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.211791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:43.211831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:43.212011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789309208953805:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.212098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789313503921201:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.212130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789309208953805:2049] [node 2] ICP30 unregister session Session# [4:7577789313503921201:2049] VirtualId# [4:7577789313503921197:0]
2025-11-28T14:52:43.212164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:43.212192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789309208953805:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.212200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.212261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:43.212449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789312538598800:2049] [node 2] ICS07 socket disconnect 114 reason# EndOfStream
2025-11-28T14:52:43.212502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789312538598800:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.212582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:43.212639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789312538598800:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:43.213263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789308243631419:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.213343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789312538598800:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.213371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789308243631419:2049] [node 2] ICP30 unregister session Session# [6:7577789312538598800:2049] VirtualId# [6:7577789312538598794:0]
2025-11-28T14:52:43.213447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:43.213464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789308243631419:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.213473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.221634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789309208953805:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:43.223701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789308243631419:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:43.273193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:43.273353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789313503921193:2050] [node 3] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-28T14:52:43.273381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789313503921193:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.273484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:43.273526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789313503921193:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:43.273858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:43.273993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789313055101423:2050] [node 3] ICS07 socket disconnect 112 reason# EndOfStream
2025-11-28T14:52:43.274014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789313055101423:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.274082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:43.274125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789313055101423:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:43.274146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789309208953806:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61681) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.274302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789313503921193:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.274357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789309208953806:2050] [node 3] ICP30 unregister session Session# [4:7577789313503921193:2050] VirtualId# [4:7577789313503921187:0]
2025-11-28T14:52:43.274439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:43.274469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789309208953806:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.274478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.274514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:43.274648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789312538598812:2050] [node 3] ICS01 socket: 144 reason# EndOfStream
2025-11-28T14:52:43.274727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789308243631420:2050] [node 3] ICP30 unregister session Session# [6:7577789312538598812:2050] VirtualId# [6:7577789312538598809:0]
2025-11-28T14:52:43.274783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789308760134029:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61681) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.274754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789312538598812:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.275015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789313055101423:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.274823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:43.275081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789308760134029:2050] [node 3] ICP30 unregister session Session# [5:7577789313055101423:2050] VirtualId# [5:7577789313055101417:0]
2025-11-28T14:52:43.274874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:43.275170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:43.275201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789308760134029:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.275211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.285141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789308760134029:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:43.307038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:43.307219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789313055101415:2108] [node 4] ICS01 socket: 102 reason# EndOfStream
2025-11-28T14:52:43.307255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789313055101386:2108] [node 4] ICP30 unregister session Session# [5:7577789313055101415:2108] VirtualId# [5:7577789313055101407:0]
2025-11-28T14:52:43.307277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789313055101415:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.307349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:43.307413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:43.307878Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:43.308024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789312538598804:2107] [node 4] ICS07 socket disconnect 126 reason# EndOfStream
2025-11-28T14:52:43.308049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789312538598804:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.308140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:43.308177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789312538598804:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:43.308779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789312538598778:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3756) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.308860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789312538598804:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.308898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789312538598778:2107] [node 4] ICP30 unregister session Session# [6:7577789312538598804:2107] VirtualId# [6:7577789312538598798:0]
2025-11-28T14:52:43.308995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:43.309038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789312538598778:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.309053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.319290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789312538598778:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:43.334458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:43.334615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789312538598816:2106] [node 5] ICS07 socket disconnect 81 reason# EndOfStream
2025-11-28T14:52:43.334641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789312538598816:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.334719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:43.334757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789312538598816:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:43.335351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789312538598775:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28916) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.335405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789312538598816:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.335449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789312538598775:2106] [node 5] ICP30 unregister session Session# [6:7577789312538598816:2106] VirtualId# [6:7577789312538598814:0]
2025-11-28T14:52:43.335549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:43.335573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789312538598775:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28916) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.335585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28916) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:43.402599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789312808134962:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:43.402652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789312808134963:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:43.420209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789314121672871:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:43.420245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789314121672872:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:43.437880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789312159277262:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:43.437916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789312159277263:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:43.454796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789313775499337:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:43.454832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789313775499339:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:43.455115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789313775499338:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:43.471342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789313918557679:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:43.471371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789313918557681:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:43.471633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789313918557680:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:43.488187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789315035851315:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:43.488268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789315035851316:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:43.488274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789315035851314:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:43.576722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789313775499400:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:43.576976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789312159277325:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:43.576164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789314121672934:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:43.576895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789315035851376:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:43.576661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789312808135025:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:43.576539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789314121672935:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:43.577330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789313918557743:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:43.576626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789314121672936:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:43.577527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789312159277327:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:43.577453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789313918557742:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:43.576934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789312808135026:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:43.577054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789315035851377:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:43.577578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789312159277326:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:43.577268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789313775499401:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:43.577558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789312808135027:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.592024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314121672948:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16727
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.592356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789312808135037:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29998
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.594351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789312159277336:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1293
2025-11-28T14:52:43.593393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313775499410:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2668
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.592686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789312808135036:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29998
2025-11-28T14:52:43.595506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789312159277337:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1293
2025-11-28T14:52:43.595572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312808135036:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.594869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313775499411:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2668
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.595848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313775499410:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.596225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314121672948:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.596614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789312159277336:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.596722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313918557755:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54313
2025-11-28T14:52:43.596626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789313775499338:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.597673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789312159277338:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1293
2025-11-28T14:52:43.597644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314121672941:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.597546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313918557755:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.598876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312808135037:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.597677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789314121672941:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.597220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313775499411:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.598861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314121672951:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16727
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.598934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314121672952:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16727
2025-11-28T14:52:43.598926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789312159277337:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.599089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789312159277338:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.598976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314121672941:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:43.599352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789312159277263:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.599445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851392:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.599448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789312808135041:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29998
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.599501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789312159277343:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1293
2025-11-28T14:52:43.599450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851390:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.601336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789312159277342:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1293
2025-11-28T14:52:43.602008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313775499416:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2668
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.602150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789312808135042:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29998
2025-11-28T14:52:43.602756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312808135041:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.602217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789314121672934:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.602525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313775499420:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2668
2025-11-28T14:52:43.602289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789312159277342:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.602262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:43.602811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789312159277343:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.603200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789315035851390:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.602300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789314121672934:2107] [node 4] ICP22 created new session: [2:7577789314121672955:2107]
2025-11-28T14:52:43.603737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789315035851314:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.602740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314121672955:2107] [node 4] ICS09 handshake done sender: [2:7577789314121672941:2110] self: [2:7577789314121672940:0] peer: [4:7577789313775499415:0] socket: 84 qp: -1
2025-11-28T14:52:43.606890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789312159277262:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.602740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314121672942:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.606908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312808135035:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.607051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313775499410:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:43.606909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312808135034:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.602767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789314121672942:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.608013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789313775499338:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.608527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313918557759:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54313
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.602775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314121672955:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:43.606940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312808135034:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.608764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789315035851392:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.608767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.602857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.606951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312808135035:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.609329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313918557759:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.608900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313775499416:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.608552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312808135042:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.609588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789315035851316:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.609485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789313775499338:2049] [node 2] ICP22 created new session: [4:7577789313775499421:2049]
2025-11-28T14:52:43.602889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314121672951:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.608635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312808135032:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.609619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789313918557680:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.609588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789312159277351:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1293
2025-11-28T14:52:43.602948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314121672952:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.609696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313775499421:2049] [node 2] ICS09 handshake done sender: [4:7577789313775499410:2114] self: [4:7577789313775499415:0] peer: [2:7577789314121672940:0] socket: 87 qp: -1
2025-11-28T14:52:43.608660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312808135032:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.609888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789313775499339:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.610813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789312159277334:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.603678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789314121672871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.610811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313775499421:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:43.608940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312808135035:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:43.606782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314121672942:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:43.611493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:43.610841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789312159277334:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.607389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789314121672872:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.609599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312808135034:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:43.608544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.609668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789312808134962:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.612851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851396:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.608757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789314121672872:2049] [node 3] ICP22 created new session: [2:7577789314121672960:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.611790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789312159277336:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:43.610803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.611826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789312159277333:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.609346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314121672960:2049] [node 3] ICS09 handshake done sender: [2:7577789314121672942:2111] self: [2:7577789314121672938:0] peer: [3:7577789312159277346:0] socket: 85 qp: -1
2025-11-28T14:52:43.613730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851397:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.611839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789312159277333:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.609521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314121672960:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:43.610824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789312808135025:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.609553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.612641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789312159277334:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:43.610838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789312808134962:2048] [node 2] ICP22 created new session: [1:7577789312808135051:2048]
2025-11-28T14:52:43.613941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789312159277333:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:43.611565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314121672944:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.614505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789312159277263:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.615240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789312159277325:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.611596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789314121672944:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.611490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312808135051:2048] [node 2] ICS09 handshake done sender: [1:7577789312808135035:2113] self: [1:7577789312808135030:0] peer: [2:7577789314121672958:0] socket: 109 qp: -1
2025-11-28T14:52:43.616206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313918557761:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54313
2025-11-28T14:52:43.615279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:43.615038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789314121672963:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16727
2025-11-28T14:52:43.611517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312808135051:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.615532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789312159277325:2109] [node 6] ICP22 created new session: [3:7577789312159277355:2109]
2025-11-28T14:52:43.611520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.616256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789315035851390:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:43.616205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313775499425:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2668
2025-11-28T14:52:43.615546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.611547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:43.616932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313918557761:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.611590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789312808135025:2109] [node 6] ICP22 created new session: [1:7577789312808135052:2109]
2025-11-28T14:52:43.615712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789312159277263:2049] [node 2] ICP22 created new session: [3:7577789312159277356:2049]
2025-11-28T14:52:43.612640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312808135052:2109] [node 6] ICS09 handshake done sender: [1:7577789312808135034:2112] self: [1:7577789312808135028:0] peer: [6:7577789315035851394:0] socket: 106 qp: -1
2025-11-28T14:52:43.616129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789312159277355:2109] [node 6] ICS09 handshake done sender: [3:7577789312159277334:2112] self: [3:7577789312159277331:0] peer: [6:7577789315035851395:0] socket: 118 qp: -1
2025-11-28T14:52:43.616629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851398:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.616888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789314121672871:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789314121672951:2116] held: no
2025-11-28T14:52:43.616642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313775499416:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:43.616992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789315035851397:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.616218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789312159277355:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.616979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313775499426:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2668
2025-11-28T14:52:43.613026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312808135052:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.617314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789315035851314:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.617234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313918557762:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54313
2025-11-28T14:52:43.617690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314121672944:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:43.617667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789313918557743:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.613626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:43.616432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789312159277356:2049] [node 2] ICS09 handshake done sender: [3:7577789312159277336:2114] self: [3:7577789312159277346:0] peer: [2:7577789314121672938:0] socket: 89 qp: -1
2025-11-28T14:52:43.617318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789315035851315:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.618199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789313775499339:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.618257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789314121672950:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.616587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:43.618243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789315035851392:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:43.613967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312808135032:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:43.618249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:43.616626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789312159277356:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:43.618279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789314121672950:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.618330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:43.618654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789313775499339:2050] [node 3] ICP22 created new session: [4:7577789313775499427:2050]
2025-11-28T14:52:43.616664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:43.615736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789312808134963:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.618702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789314121672934:2107] [node 4] ICP18 handshake (actor [2:7577789314121672952:2117]) is too old
2025-11-28T14:52:43.619782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313775499427:2050] [node 3] ICS09 handshake done sender: [4:7577789313775499416:2116] self: [4:7577789313775499422:0] peer: [3:7577789312159277329:0] socket: 121 qp: -1
2025-11-28T14:52:43.616666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789312159277327:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.618698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789315035851314:2048] [node 1] ICP22 created new session: [6:7577789315035851400:2048]
2025-11-28T14:52:43.615777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:43.619964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313775499409:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.623325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789314121672935:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.617720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:43.619632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789315035851400:2048] [node 1] ICS09 handshake done sender: [6:7577789315035851390:2114] self: [6:7577789315035851394:0] peer: [1:7577789312808135028:0] socket: 115 qp: -1
2025-11-28T14:52:43.619979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789313775499409:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.615810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789312808134963:2049] [node 3] ICP22 created new session: [1:7577789312808135058:2049]
2025-11-28T14:52:43.623571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.620027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313775499427:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:43.616400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312808135058:2049] [node 3] ICS09 handshake done sender: [1:7577789312808135032:2110] self: [1:7577789312808135031:0] peer: [3:7577789312159277350:0] socket: 107 qp: -1
2025-11-28T14:52:43.617944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789312159277327:2107] [node 4] ICP22 created new session: [3:7577789312159277359:2107]
2025-11-28T14:52:43.616904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312808135058:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:43.620030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789315035851400:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.616941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.622091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.618225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789312159277359:2107] [node 4] ICS09 handshake done sender: [3:7577789312159277333:2111] self: [3:7577789312159277329:0] peer: [4:7577789313775499422:0] socket: 120 qp: -1
2025-11-28T14:52:43.623636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789314121672935:2108] [node 5] ICP22 created new session: [2:7577789314121672965:2108]
2025-11-28T14:52:43.620030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789315035851316:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3333) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789315035851392:2115] held: no
2025-11-28T14:52:43.618258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789312159277359:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:43.618279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.620068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:43.623950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314121672965:2108] [node 5] ICS09 handshake done sender: [2:7577789314121672944:2113] self: [2:7577789314121672939:0] peer: [5:7577789313918557760:0] socket: 128 qp: -1
2025-11-28T14:52:43.624191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313775499426:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.624375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314121672965:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:43.624411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.624476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789315035851316:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.624527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:43.624766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789315035851316:2050] [node 3] ICP22 created new session: [6:7577789315035851403:2050]
2025-11-28T14:52:43.624741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314121672951:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.624792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313775499409:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.624805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789315035851403:2050] [node 3] ICS09 handshake done sender: [6:7577789315035851392:2115] self: [6:7577789315035851395:0] peer: [3:7577789312159277331:0] socket: 123 qp: -1
2025-11-28T14:52:43.624920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789314121672871:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.624455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313918557764:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54313
2025-11-28T14:52:43.624842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789315035851403:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:43.625356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313775499429:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2668
2025-11-28T14:52:43.625031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789314121672950:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:43.624617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313918557762:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.625425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789312159277354:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1293
2025-11-28T14:52:43.625514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789313775499337:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.625072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.625702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312808135044:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.624760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789313918557679:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.625806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:43.625770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789313775499400:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.625913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789312159277262:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789312159277343:2118] held: no
2025-11-28T14:52:43.625734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312808135044:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.625937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313918557759:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:43.626490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789312808135033:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.625955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789314121672936:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.627103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.626516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789312808135033:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.627270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789313775499400:2108] [node 5] ICP22 created new session: [4:7577789313775499433:2108]
2025-11-28T14:52:43.626555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789314121672871:2048] [node 1] ICP22 created new session: [2:7577789314121672968:2048]
2025-11-28T14:52:43.626932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312808135044:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:43.627102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314121672968:2048] [node 1] ICS09 handshake done sender: [2:7577789314121672951:2116] self: [2:7577789314121672958:0] peer: [1:7577789312808135030:0] socket: 110 qp: -1
2025-11-28T14:52:43.627285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789313918557680:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.627393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851405:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.627381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314121672968:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:43.627351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.627532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313775499433:2108] [node 5] ICS09 handshake done sender: [4:7577789313775499409:2113] self: [4:7577789313775499403:0] peer: [5:7577789313918557763:0] socket: 138 qp: -1
2025-11-28T14:52:43.628619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313775499433:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:43.628857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.628621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789312808135027:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.627420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:43.627426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:43.627481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789314121672936:2109] [node 6] ICP22 created new session: [2:7577789314121672970:2109]
2025-11-28T14:52:43.627389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789313918557680:2049] [node 2] ICP22 created new session: [5:7577789313918557767:2049]
2025-11-28T14:52:43.628650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789314121672970:2109] [node 6] ICS09 handshake done sender: [2:7577789314121672950:2115] self: [2:7577789314121672949:0] peer: [6:7577789315035851399:0] socket: 134 qp: -1
2025-11-28T14:52:43.629251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.629261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789313775499338:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16433592ns) {HandleHandshakeStatus:329(16431722ns) {DropIncomingHandshake:442(354ns) DropOutgoingHandshake:457(11692ns) SwitchToState:179(108ns) ProcessPendingSessionEvents:495(12832928ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12594809ns max# 1255984ns}}}
2025-11-28T14:52:43.629500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789315035851397:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:43.627430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313918557767:2049] [node 2] ICS09 handshake done sender: [5:7577789313918557759:2116] self: [5:7577789313918557760:0] peer: [2:7577789314121672939:0] socket: 129 qp: -1
2025-11-28T14:52:43.629301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789312808135027:2108] [node 5] ICP22 created new session: [1:7577789312808135062:2108]
2025-11-28T14:52:43.628692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789314121672970:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.630219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313775499429:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.629492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789312808135033:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:43.627558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313918557767:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:43.628867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:43.628324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313918557766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54313
2025-11-28T14:52:43.630457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312808135062:2108] [node 5] ICS09 handshake done sender: [1:7577789312808135044:2118] self: [1:7577789312808135043:0] peer: [5:7577789313918557765:0] socket: 144 qp: -1
2025-11-28T14:52:43.628627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:43.630531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789315035851315:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789315035851397:2118] held: no
2025-11-28T14:52:43.630487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312808135062:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:43.630720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.630758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789312808135026:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.632017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:43.631084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789312159277343:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:43.631253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789313918557743:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789313918557761:2117] held: no
2025-11-28T14:52:43.632218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789312808135026:2107] [node 4] ICP22 created new session: [1:7577789312808135064:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.633336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789312159277262:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.632720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789312808135064:2107] [node 4] ICS09 handshake done sender: [1:7577789312808135033:2111] self: [1:7577789312808135029:0] peer: [4:7577789313775499431:0] socket: 142 qp: -1
2025-11-28T14:52:43.633372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789313918557679:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789313918557762:2118] held: no
2025-11-28T14:52:43.633552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851406:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.632967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789312808135064:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:43.632995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.633603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:43.633652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789315035851315:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-28T14:52:43.633657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789312159277262:2048] [node 1] ICP22 created new session: [3:7577789312159277362:2048]
2025-11-28T14:52:43.633937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789312159277362:2048] [node 1] ICS09 handshake done sender: [3:7577789312159277343:2118] self: [3:7577789312159277350:0] peer: [1:7577789312808135031:0] socket: 116 qp: -1
2025-11-28T14:52:43.633986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:43.634016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789315035851315:2049] [node 2] ICP22 created new session: [6:7577789315035851407:2049]
2025-11-28T14:52:43.634022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789312159277362:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:43.633997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313918557769:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54313
2025-11-28T14:52:43.634100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789315035851407:2049] [node 2] ICS09 handshake done sender: [6:7577789315035851397:2118] self: [6:7577789315035851399:0] peer: [2:7577789314121672949:0] socket: 112 qp: -1
2025-11-28T14:52:43.634127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789315035851407:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:43.634094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313918557770:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54313
2025-11-28T14:52:43.634148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:43.634133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:43.634389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313918557761:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.635541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789315035851406:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.635410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313918557762:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:43.635627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789313775499435:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2668
2025-11-28T14:52:43.635532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789313918557743:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.635676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789315035851377:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.635954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313918557770:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.635988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789313918557679:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.635987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789313775499337:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789313775499426:2120] held: no
2025-11-28T14:52:43.635989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:43.636041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789313918557743:2108] [node 4] ICP22 created new session: [5:7577789313918557772:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.636075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:43.636099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313918557772:2108] [node 4] ICS09 handshake done sender: [5:7577789313918557761:2117] self: [5:7577789313918557763:0] peer: [4:7577789313775499403:0] socket: 92 qp: -1
2025-11-28T14:52:43.636131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313918557772:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:43.636172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851410:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.636127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789313918557679:2048] [node 1] ICP22 created new session: [5:7577789313918557773:2048]
2025-11-28T14:52:43.636158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.636339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789313775499419:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.636382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789313775499419:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.636176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313918557773:2048] [node 1] ICS09 handshake done sender: [5:7577789313918557762:2118] self: [5:7577789313918557765:0] peer: [1:7577789312808135043:0] socket: 147 qp: -1
2025-11-28T14:52:43.636204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313918557773:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:43.636297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:43.636857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313775499426:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:43.636996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789313775499337:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.637206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:43.637406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789313775499337:2048] [node 1] ICP22 created new session: [4:7577789313775499437:2048]
2025-11-28T14:52:43.637452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313775499437:2048] [node 1] ICS09 handshake done sender: [4:7577789313775499426:2120] self: [4:7577789313775499431:0] peer: [1:7577789312808135029:0] socket: 145 qp: -1
2025-11-28T14:52:43.637481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313775499437:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:43.637573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:43.637669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789313775499419:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:43.637837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789313775499401:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.637805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789313918557681:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.638045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:43.638068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789312159277345:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.638096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789312159277345:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.638140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789313775499401:2109] [node 6] ICP22 created new session: [4:7577789313775499440:2109]
2025-11-28T14:52:43.638192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789313775499440:2109] [node 6] ICS09 handshake done sender: [4:7577789313775499419:2117] self: [4:7577789313775499418:0] peer: [6:7577789315035851409:0] socket: 149 qp: -1
2025-11-28T14:52:43.638148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789315035851410:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.638275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789313775499440:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.638303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:43.638238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851411:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.638405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789315035851376:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:43.639119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789315035851406:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:43.639288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789313918557757:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:43.639317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789313918557757:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:43.639764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789312159277345:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:43.639757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789315035851377:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789315035851406:2121] held: no
2025-11-28T14:52:43.640233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313918557757:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:43.640656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789312159277326:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:43.640773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.640811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789313918557778:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54313
No RDMA context found for GID: 2025-11-28T14:52:43.640819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789312159277326:2108] [node 5] ICP22 created new session: [3:7577789312159277365:2108]
::ffff:127.1.0.0
2025-11-28T14:52:43.640970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789312159277365:2108] [node 5] ICS09 handshake done sender: [3:7577789312159277345:2119] self: [3:7577789312159277344:0] peer: [5:7577789313918557776:0] socket: 152 qp: -1
2025-11-28T14:52:43.640841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789313918557742:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:43.641379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789315035851413:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54796
2025-11-28T14:52:43.641375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789312159277365:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:43.641132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:43.641422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.641552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789313918557742:2109] [node 6] ICP22 created new session: [5:7577789313918557779:2109]
2025-11-28T14:52:43.641595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789315035851376:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10708) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789315035851410:2122] held: no
2025-11-28T14:52:43.641836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789313918557681:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3333) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789313918557770:2122] held: no
2025-11-28T14:52:43.643466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789315035851377:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.641838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789313918557770:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:43.643813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:43.643333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313918557779:2109] [node 6] ICS09 handshake done sender: [5:7577789313918557757:2115] self: [5:7577789313918557756:0] peer: [6:7577789315035851412:0] socket: 90 qp: -1
2025-11-28T14:52:43.643436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313918557779:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:43.643854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789315035851377:2108] [node 4] ICP22 created new session: [6:7577789315035851414:2108]
2025-11-28T14:52:43.643464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:43.643893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789315035851414:2108] [node 4] ICS09 handshake done sender: [6:7577789315035851406:2121] self: [6:7577789315035851409:0] peer: [4:7577789313775499418:0] socket: 150 qp: -1
2025-11-28T14:52:43.643910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789315035851414:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:43.643932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:43.644203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789315035851410:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:43.644204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789313918557681:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.644299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:43.644409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789315035851376:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:43.644415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789313918557681:2050] [node 3] ICP22 created new session: [5:7577789313918557781:2050]
2025-11-28T14:52:43.644529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:43.644523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789313918557781:2050] [node 3] ICS09 handshake done sender: [5:7577789313918557770:2122] self: [5:7577789313918557776:0] peer: [3:7577789312159277344:0] socket: 100 qp: -1
2025-11-28T14:52:43.644579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789313918557781:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:43.644602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789315035851376:2106] [node 5] ICP22 created new session: [6:7577789315035851416:2106]
2025-11-28T14:52:43.644612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:43.644652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789315035851416:2106] [node 5] ICS09 handshake done sender: [6:7577789315035851410:2122] self: [6:7577789315035851412:0] peer: [5:7577789313918557756:0] socket: 95 qp: -1
2025-11-28T14:52:43.644687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789315035851416:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:43.644725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:43.680062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:43.680229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789314121672968:2048] [node 1] ICS01 socket: 110 reason# EndOfStream
2025-11-28T14:52:43.680263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789314121672871:2048] [node 1] ICP30 unregister session Session# [2:7577789314121672968:2048] VirtualId# [2:7577789314121672958:0]
2025-11-28T14:52:43.680293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789314121672968:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.680387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.680438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.681139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:43.681333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789312159277362:2048] [node 1] ICS01 socket: 116 reason# EndOfStream
2025-11-28T14:52:43.681381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789312159277262:2048] [node 1] ICP30 unregister session Session# [3:7577789312159277362:2048] VirtualId# [3:7577789312159277350:0]
2025-11-28T14:52:43.681414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789312159277362:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.681498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.681556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.683134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:43.683261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789313775499437:2048] [node 1] ICS01 socket: 145 reason# EndOfStream
2025-11-28T14:52:43.683289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789313775499337:2048] [node 1] ICP30 unregister session Session# [4:7577789313775499437:2048] VirtualId# [4:7577789313775499431:0]
2025-11-28T14:52:43.683308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789313775499437:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.683369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.683415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.683737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:43.683917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789313918557773:2048] [node 1] ICS01 socket: 147 reason# EndOfStream
2025-11-28T14:52:43.683982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789313918557679:2048] [node 1] ICP30 unregister session Session# [5:7577789313918557773:2048] VirtualId# [5:7577789313918557765:0]
2025-11-28T14:52:43.684016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789313918557773:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.684101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.684151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.684413Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:43.684569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789315035851400:2048] [node 1] ICS07 socket disconnect 115 reason# EndOfStream
2025-11-28T14:52:43.684593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789315035851400:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.684663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:43.684695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789315035851400:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:43.685362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789315035851314:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11893) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.685540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789315035851400:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.685588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789315035851314:2048] [node 1] ICP30 unregister session Session# [6:7577789315035851400:2048] VirtualId# [6:7577789315035851394:0]
2025-11-28T14:52:43.685661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:43.685689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789315035851314:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.685698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.695776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789315035851314:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:43.742621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:43.742838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789312159277356:2049] [node 2] ICS07 socket disconnect 89 reason# EndOfStream
2025-11-28T14:52:43.742872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789312159277356:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.742988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:43.743045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789312159277356:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:43.743725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789312159277263:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.743763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789312159277356:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.743819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789312159277263:2049] [node 2] ICP30 unregister session Session# [3:7577789312159277356:2049] VirtualId# [3:7577789312159277346:0]
2025-11-28T14:52:43.743908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:43.743933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789312159277263:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.743945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.745058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:43.745191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789313775499421:2049] [node 2] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-28T14:52:43.745212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789313775499421:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.745264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:43.745292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789313775499421:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:43.745780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789313775499338:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.745787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:43.745836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789313775499421:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.745873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789313775499338:2049] [node 2] ICP30 unregister session Session# [4:7577789313775499421:2049] VirtualId# [4:7577789313775499415:0]
2025-11-28T14:52:43.745957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:43.745982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789313775499338:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.745990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789313918557767:2049] [node 2] ICS07 socket disconnect 129 reason# EndOfStream
2025-11-28T14:52:43.746023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789313918557767:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.745990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.746104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:43.746149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789313918557767:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:43.746491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:43.746641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789315035851407:2049] [node 2] ICS01 socket: 112 reason# EndOfStream
2025-11-28T14:52:43.746633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789313918557680:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.746736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789313918557767:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.746713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789315035851315:2049] [node 2] ICP30 unregister session Session# [6:7577789315035851407:2049] VirtualId# [6:7577789315035851399:0]
2025-11-28T14:52:43.746849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789315035851407:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.746771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789313918557680:2049] [node 2] ICP30 unregister session Session# [5:7577789313918557767:2049] VirtualId# [5:7577789313918557760:0]
2025-11-28T14:52:43.746867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:43.746903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789313918557680:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.746929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:43.746969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:43.746924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.756192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789313775499338:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:43.757188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789313918557680:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:43.787686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:43.787914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789313775499427:2050] [node 3] ICS07 socket disconnect 121 reason# EndOfStream
2025-11-28T14:52:43.787946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789313775499427:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.788043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:43.788089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789313775499427:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:43.788465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:43.788647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789313918557781:2050] [node 3] ICS01 socket: 100 reason# EndOfStream
2025-11-28T14:52:43.788768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789313775499339:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3333) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:43.788699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789313918557681:2050] [node 3] ICP30 unregister session Session# [5:7577789313918557781:2050] VirtualId# [5:7577789313918557776:0]
2025-11-28T14:52:43.788734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789313918557781:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.788823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789313775499427:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:43.788819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:43.788876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:43.788899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789313775499339:2050] [node 3] ICP30 unregister session Session# [4:7577789313775499427:2050] VirtualId# [4:7577789313775499422:0]
2025-11-28T14:52:43.788936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:43.788957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789313775499339:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3333) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.788969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3333) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:43.789210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:43.789371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789315035851403:2050] [node 3] ICS01 socket: 123 reason# EndOfStream
2025-11-28T14:52:43.789408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789315035851316:2050] [node 3] ICP30 unregister session Session# [6:7577789315035851403:2050] VirtualId# [6:7577789315035851395:0]
2025-11-28T14:52:43.789433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789315035851403:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.789521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:43.789558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:43.799199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789313775499339:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:43.834226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:43.834442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789313918557772:2108] [node 4] ICS01 socket: 92 reason# EndOfStream
2025-11-28T14:52:43.834492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789313918557743:2108] [node 4] ICP30 unregister session Session# [5:7577789313918557772:2108] VirtualId# [5:7577789313918557763:0]
2025-11-28T14:52:43.834536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789313918557772:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.834641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:43.834687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:43.834996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:43.835145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789315035851414:2108] [node 4] ICS01 socket: 150 reason# EndOfStream
2025-11-28T14:52:43.835214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789315035851377:2108] [node 4] ICP30 unregister session Session# [6:7577789315035851414:2108] VirtualId# [6:7577789315035851409:0]
2025-11-28T14:52:43.835244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789315035851414:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.835328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:43.835367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:43.888949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:43.889131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789315035851416:2106] [node 5] ICS01 socket: 95 reason# EndOfStream
2025-11-28T14:52:43.889166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789315035851376:2106] [node 5] ICP30 unregister session Session# [6:7577789315035851416:2106] VirtualId# [6:7577789315035851412:0]
2025-11-28T14:52:43.889199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789315035851416:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:43.889293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:43.889338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:43.987963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789313621151521:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:43.988004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789313621151522:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:44.004810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789319058796682:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.004840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789319058796683:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:44.022637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789317297610602:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.022749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789317297610603:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:44.040109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789318662406820:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.040141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789318662406821:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:44.040161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789318662406822:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:44.059463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789318802875120:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:44.059466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789318802875119:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.059491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789318802875121:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:44.076445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319878332979:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.076511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319878332980:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:44.076562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319878332981:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:44.150467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789317916118880:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:44.161704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319878333040:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:44.162153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789317297610666:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:44.162194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789317916118881:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:44.162277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789317297610668:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:44.162219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789319058796745:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:44.162465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789317297610667:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:44.162502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789318662406883:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:44.162369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789318802875182:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:44.162863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789318662406884:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:44.162832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319878333041:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:44.162860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789318802875183:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:44.162947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789317916118882:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:44.163227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789319058796746:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:44.163273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789319058796747:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.179200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789318662406895:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3682
2025-11-28T14:52:44.179376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789319058796760:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59703
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.179332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333056:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.180167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789318802875198:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:44.182002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789317297610682:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23412
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.180999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789318802875197:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-11-28T14:52:44.181995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333059:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.182011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789319058796762:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59703
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.185083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333058:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.183035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789319058796760:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.186022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789318802875198:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.185085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789318662406900:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3682
2025-11-28T14:52:44.185996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789317297610683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23412
2025-11-28T14:52:44.186501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319878333059:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.185654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789319058796762:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.185618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789318662406901:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3682
2025-11-28T14:52:44.186468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789318802875119:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.186034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789317297610682:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.186788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789317916118896:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4418
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.186854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789319058796763:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59703
2025-11-28T14:52:44.185766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789318662406895:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.187136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789317297610683:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.186746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333060:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
2025-11-28T14:52:44.186547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789318802875197:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.186847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789317916118886:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.188250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789319058796751:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.186200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789318662406899:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3682
2025-11-28T14:52:44.186777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319878333056:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.186875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789317916118886:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.187124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789318802875182:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.186884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789317916118897:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4418
2025-11-28T14:52:44.187118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319878333058:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.188275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789319058796751:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.186485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789318662406900:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.189684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789318802875189:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.187814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319878333041:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.191585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789317297610674:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.186874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789318662406901:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.188172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789317916118896:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.188273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319878333040:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.189718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789318802875189:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.188506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789319058796763:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.191621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789317297610674:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.193581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789318802875189:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:44.189848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789319058796682:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.188389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789317916118897:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.187071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789318662406821:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.188773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319878333060:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.193796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789317297610676:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.189547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789317916118898:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4418
2025-11-28T14:52:44.187156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789318662406820:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.191270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789319058796751:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:44.189658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319878332981:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.188419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789318662406899:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.193812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789317297610676:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.189689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789317916118887:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.193370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789319058796755:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.188443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789318662406889:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.189895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319878332980:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.196001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789317297610674:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:44.189706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789317916118887:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.193405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789319058796755:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.188458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789318662406887:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.191598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789317916118886:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:44.198335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333065:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
2025-11-28T14:52:44.197685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789318802875183:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.197277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789317297610676:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:44.194489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789319058796745:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.188463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789318662406889:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.199516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333066:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
2025-11-28T14:52:44.191603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789317916118891:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.198274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789318802875202:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.198347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789317297610667:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.195786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.188489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789318662406887:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.202166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319878333059:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:44.191631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789317916118891:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.199506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789318802875203:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-11-28T14:52:44.199485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789317297610666:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.196124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789319058796745:2107] [node 4] ICP22 created new session: [2:7577789319058796768:2107]
2025-11-28T14:52:44.192448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789318662406822:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.192471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789317916118898:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.199524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.192503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789318662406889:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:44.199184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:44.196824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789319058796768:2107] [node 4] ICS09 handshake done sender: [2:7577789319058796751:2110] self: [2:7577789319058796748:0] peer: [4:7577789318662406902:0] socket: 104 qp: -1
2025-11-28T14:52:44.201271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: 2025-11-28T14:52:44.199920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789318802875183:2109] [node 6] ICP22 created new session: [5:7577789318802875205:2109]
2025-11-28T14:52:44.204317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319878333056:2115] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.02025-11-28T14:52:44.196859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789319058796768:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:44.196401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789317916118887:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:44.196634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789318662406887:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:44.201322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789317297610667:2110] [node 6] ICP22 created new session: [3:7577789317297610691:2110]
2025-11-28T14:52:44.205593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319878332981:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29787) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789319878333058:2116] held: no
2025-11-28T14:52:44.196432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789317916118891:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:44.201303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789318802875205:2109] [node 6] ICS09 handshake done sender: [5:7577789318802875189:2110] self: [5:7577789318802875184:0] peer: [6:7577789319878333062:0] socket: 90 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.197694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.197080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789318662406884:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.197773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789319058796755:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:44.196813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789317916118882:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.201365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789317297610691:2110] [node 6] ICS09 handshake done sender: [3:7577789317297610674:2111] self: [3:7577789317297610672:0] peer: [6:7577789319878333063:0] socket: 97 qp: -1
2025-11-28T14:52:44.198340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:44.207220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319878333041:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.201335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789318802875205:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:44.199835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789319058796769:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59703
2025-11-28T14:52:44.197120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:44.199526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789318662406884:2108] [node 5] ICP22 created new session: [4:7577789318662406911:2108]
2025-11-28T14:52:44.201394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789317297610691:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:44.207978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319878332980:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14313) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789319878333060:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.201369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.199544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789318662406883:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.199883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789319058796746:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.199871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789318662406907:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3682
2025-11-28T14:52:44.201783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789317297610666:2107] [node 4] ICP22 created new session: [3:7577789317297610690:2107]
2025-11-28T14:52:44.197371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789317916118880:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.209754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.201281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:44.199890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:44.203370Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789318802875182:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25102) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789318802875197:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.197725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789317916118882:2108] [node 5] ICP22 created new session: [1:7577789317916118906:2108]
2025-11-28T14:52:44.201835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789319058796746:2109] [node 6] ICP22 created new session: [2:7577789319058796773:2109]
2025-11-28T14:52:44.201849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:44.209834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319878333040:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.203051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789319058796763:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:44.201248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789318662406883:2109] [node 6] ICP22 created new session: [4:7577789318662406912:2109]
2025-11-28T14:52:44.203317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789319058796773:2109] [node 6] ICS09 handshake done sender: [2:7577789319058796755:2112] self: [2:7577789319058796753:0] peer: [6:7577789319878333064:0] socket: 105 qp: -1
2025-11-28T14:52:44.203008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789317297610690:2107] [node 4] ICS09 handshake done sender: [3:7577789317297610676:2113] self: [3:7577789317297610670:0] peer: [4:7577789318662406905:0] socket: 102 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:44.197731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789313621151521:2048] [node 2] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:44.204303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789318802875198:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:44.202222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789318662406911:2108] [node 5] ICS09 handshake done sender: [4:7577789318662406889:2111] self: [4:7577789318662406886:0] peer: [5:7577789318802875200:0] socket: 103 qp: -1
2025-11-28T14:52:44.204323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789317297610690:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:44.212575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319878333041:2107] [node 4] ICP22 created new session: [6:7577789319878333070:2107]
2025-11-28T14:52:44.204321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789319058796773:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:44.198161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789317916118906:2108] [node 5] ICS09 handshake done sender: [1:7577789317916118886:2110] self: [1:7577789317916118883:0] peer: [5:7577789318802875199:0] socket: 114 qp: -1
2025-11-28T14:52:44.213359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333067:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.205592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.205983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789318802875119:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.202246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789318662406912:2109] [node 6] ICS09 handshake done sender: [4:7577789318662406887:2110] self: [4:7577789318662406885:0] peer: [6:7577789319878333061:0] socket: 99 qp: -1
2025-11-28T14:52:44.198183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.205604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.207636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789317297610688:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23412
2025-11-28T14:52:44.207985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.202647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789318662406911:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:44.212600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789317297610688:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.208016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789318802875119:2048] [node 1] ICP22 created new session: [5:7577789318802875210:2048]
No RDMA context found for GID: 2025-11-28T14:52:44.203009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789318662406912:2109] [node 6] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:44.215148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789317297610603:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.198220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789313621151521:2048] [node 2] ICP22 created new session: [1:7577789317916118907:2048]
2025-11-28T14:52:44.213360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.205654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789319058796682:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789319058796763:2117] held: no
2025-11-28T14:52:44.209727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789318802875210:2048] [node 1] ICS09 handshake done sender: [5:7577789318802875198:2116] self: [5:7577789318802875199:0] peer: [1:7577789317916118883:0] socket: 117 qp: -1
2025-11-28T14:52:44.213405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319878333070:2107] [node 4] ICS09 handshake done sender: [6:7577789319878333059:2117] self: [6:7577789319878333061:0] peer: [4:7577789318662406885:0] socket: 119 qp: -1
2025-11-28T14:52:44.203030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:44.214289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789319058796776:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59703
2025-11-28T14:52:44.198386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789317916118907:2048] [node 2] ICS09 handshake done sender: [1:7577789317916118891:2113] self: [1:7577789317916118885:0] peer: [2:7577789319058796766:0] socket: 116 qp: -1
2025-11-28T14:52:44.214647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789319058796682:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.210178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789318802875206:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-11-28T14:52:44.203327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:44.198938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789317916118906:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:44.214674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.212552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789318802875210:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.217533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789317297610695:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23412
2025-11-28T14:52:44.213687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319878333070:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:44.205604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789318662406908:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3682
2025-11-28T14:52:44.198974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789317916118907:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.212607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.207972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789318662406900:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:44.214975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789318802875206:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.214707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789319058796682:2048] [node 1] ICP22 created new session: [2:7577789319058796778:2048]
2025-11-28T14:52:44.209691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789318662406910:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3682
2025-11-28T14:52:44.215519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789318802875197:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:44.213729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.217581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789317297610696:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23412
2025-11-28T14:52:44.199180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.215151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789319058796778:2048] [node 1] ICS09 handshake done sender: [2:7577789319058796763:2117] self: [2:7577789319058796766:0] peer: [1:7577789317916118885:0] socket: 123 qp: -1
2025-11-28T14:52:44.215548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789318802875209:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-11-28T14:52:44.218655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789317297610678:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.213780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333068:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
2025-11-28T14:52:44.215925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789318802875121:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.209730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789318662406822:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29787) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789318662406899:2116] held: no
2025-11-28T14:52:44.199226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:44.215650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789319058796754:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:44.218681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789317297610678:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.217669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789318802875182:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.215673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789319058796754:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.218791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789318802875209:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.215708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789319058796778:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.220447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789317297610695:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.213819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319878333040:2106] [node 5] ICP22 created new session: [6:7577789319878333071:2106]
2025-11-28T14:52:44.213741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789318662406821:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.218817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.199547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:44.215896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.220917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789317297610696:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.214623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319878333071:2106] [node 5] ICS09 handshake done sender: [6:7577789319878333056:2115] self: [6:7577789319878333062:0] peer: [5:7577789318802875184:0] socket: 95 qp: -1
2025-11-28T14:52:44.199876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789317916118880:2107] [node 4] ICP22 created new session: [1:7577789317916118910:2107]
2025-11-28T14:52:44.220081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789319058796776:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.214062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.214938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319878333071:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:44.220934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789317297610678:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:44.220237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789319058796754:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.201786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789317916118910:2107] [node 4] ICS09 handshake done sender: [1:7577789317916118887:2111] self: [1:7577789317916118884:0] peer: [4:7577789318662406904:0] socket: 111 qp: -1
2025-11-28T14:52:44.219281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789318802875120:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.214106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789318662406821:2049] [node 2] ICP22 created new session: [4:7577789318662406917:2049]
2025-11-28T14:52:44.221084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789317297610602:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.214968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:44.220545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789319058796777:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59703
2025-11-28T14:52:44.219291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789318802875182:2108] [node 4] ICP22 created new session: [5:7577789318802875213:2108]
2025-11-28T14:52:44.202651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789317916118910:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:44.214965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789318662406917:2049] [node 2] ICS09 handshake done sender: [4:7577789318662406900:2117] self: [4:7577789318662406902:0] peer: [2:7577789319058796748:0] socket: 122 qp: -1
2025-11-28T14:52:44.221564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789317297610698:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23412
2025-11-28T14:52:44.214990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333069:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.220263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789318802875213:2108] [node 4] ICS09 handshake done sender: [5:7577789318802875197:2115] self: [5:7577789318802875200:0] peer: [4:7577789318662406886:0] socket: 109 qp: -1
2025-11-28T14:52:44.221753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789319058796757:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.202683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.220295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789318802875213:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:44.213384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789317916118911:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4418
2025-11-28T14:52:44.220336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.215156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789318662406917:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:44.214704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789317916118914:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4418
2025-11-28T14:52:44.215196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:44.218464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319878333060:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:44.222309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789317916118892:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.215372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789318662406901:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:44.221778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789319058796757:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.222330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789317916118892:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.218731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319878333058:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.222347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789317916118890:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.224492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789318802875216:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-11-28T14:52:44.218990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319878333069:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.222375Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789317916118890:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.218957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789318662406820:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.219015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319878332980:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.222381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789317297610668:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.221806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789319058796745:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(20560807ns) {HandleHandshakeStatus:329(20558551ns) {DropIncomingHandshake:442(239ns) DropOutgoingHandshake:457(570ns) SwitchToState:179(184ns) ProcessPendingSessionEvents:495(14662984ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14581195ns max# 2552871ns}}}
2025-11-28T14:52:44.220274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.220327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789318662406820:2048] [node 1] ICP22 created new session: [4:7577789318662406919:2048]
2025-11-28T14:52:44.219174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319878332981:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.221883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789319058796683:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.222420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789317297610688:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:44.220387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789318662406919:2048] [node 1] ICS09 handshake done sender: [4:7577789318662406901:2118] self: [4:7577789318662406904:0] peer: [1:7577789317916118884:0] socket: 128 qp: -1
2025-11-28T14:52:44.221913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.219200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.222300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789319058796683:2049] [node 3] ICP22 created new session: [2:7577789319058796781:2049]
2025-11-28T14:52:44.220592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789318662406919:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.223805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:44.219303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.222344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789319058796781:2049] [node 3] ICS09 handshake done sender: [2:7577789319058796754:2111] self: [2:7577789319058796750:0] peer: [3:7577789317297610697:0] socket: 150 qp: -1
2025-11-28T14:52:44.220624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.226827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789317916118892:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.224388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789317297610603:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.227955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789318802875183:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23512751ns) {HandleHandshakeStatus:329(23302305ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(848ns) SwitchToState:179(300ns) ProcessPendingSessionEvents:495(17289194ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17216232ns max# 2919573ns}}}
2025-11-28T14:52:44.219453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319878332981:2050] [node 3] ICP22 created new session: [6:7577789319878333075:2050]
2025-11-28T14:52:44.222378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789319058796781:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:44.227740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789317916118880:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22977726ns) {HandleHandshakeStatus:329(22975544ns) {DropIncomingHandshake:442(200ns) DropOutgoingHandshake:457(3913ns) SwitchToState:179(349ns) ProcessPendingSessionEvents:495(14702875ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14623782ns max# 2310293ns}}}
2025-11-28T14:52:44.224521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.228917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789318802875206:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.226014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789318662406899:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.222417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:44.227946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789317916118890:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:44.220268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319878332980:2049] [node 2] ICP22 created new session: [6:7577789319878333074:2049]
2025-11-28T14:52:44.225373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789317297610668:2109] [node 5] ICP22 created new session: [3:7577789317297610702:2109]
2025-11-28T14:52:44.228972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789318802875121:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29787) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789318802875206:2119] held: no
2025-11-28T14:52:44.225922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789319058796757:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:44.228969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789318662406822:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.228911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789313621151521:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(22248677ns) {HandleHandshakeStatus:329(22247220ns) {DropIncomingHandshake:442(10718ns) DropOutgoingHandshake:457(152ns) SwitchToState:179(476ns) ProcessPendingSessionEvents:495(17293146ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16838419ns max# 2763800ns}}}
2025-11-28T14:52:44.227727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789319058796747:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.225446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789317297610603:2049] [node 2] ICP22 created new session: [3:7577789317297610703:2049]
2025-11-28T14:52:44.220336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319878333075:2050] [node 3] ICS09 handshake done sender: [6:7577789319878333058:2116] self: [6:7577789319878333063:0] peer: [3:7577789317297610672:0] socket: 101 qp: -1
2025-11-28T14:52:44.229108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789313621151522:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.229193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789318662406884:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(23875808ns) {HandleHandshakeStatus:329(23873962ns) {DropIncomingHandshake:442(13183ns) DropOutgoingHandshake:457(522ns) SwitchToState:179(277ns) ProcessPendingSessionEvents:495(15275696ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14425873ns max# 2313503ns}}}
2025-11-28T14:52:44.229795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.229802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.226137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789317297610702:2109] [node 5] ICS09 handshake done sender: [3:7577789317297610678:2115] self: [3:7577789317297610673:0] peer: [5:7577789318802875212:0] socket: 157 qp: -1
2025-11-28T14:52:44.226175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789317297610702:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.230013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.229973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789319058796747:2108] [node 5] ICP22 created new session: [2:7577789319058796784:2108]
2025-11-28T14:52:44.229971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789313621151522:2049] [node 3] ICP22 created new session: [1:7577789317916118917:2049]
2025-11-28T14:52:44.220358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319878333075:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:44.226180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789317297610703:2049] [node 2] ICS09 handshake done sender: [3:7577789317297610688:2118] self: [3:7577789317297610697:0] peer: [2:7577789319058796750:0] socket: 155 qp: -1
2025-11-28T14:52:44.230648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789319058796784:2108] [node 5] ICS09 handshake done sender: [2:7577789319058796757:2114] self: [2:7577789319058796749:0] peer: [5:7577789318802875214:0] socket: 92 qp: -1
2025-11-28T14:52:44.230670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789318662406822:2050] [node 3] ICP22 created new session: [4:7577789318662406921:2050]
2025-11-28T14:52:44.230610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789317916118917:2049] [node 3] ICS09 handshake done sender: [1:7577789317916118892:2114] self: [1:7577789317916118889:0] peer: [3:7577789317297610700:0] socket: 127 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.226357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789317297610703:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:44.230629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789317916118881:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.226392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:44.220396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:44.230786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789319058796784:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:44.230643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789317916118917:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:44.226665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:44.230847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789318662406883:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(21216535ns) {HandleHandshakeStatus:329(21214739ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(271ns) SwitchToState:179(142ns) ProcessPendingSessionEvents:495(14754823ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13853804ns max# 2313117ns}}}
2025-11-28T14:52:44.230824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:44.220445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319878333074:2049] [node 2] ICS09 handshake done sender: [6:7577789319878333060:2118] self: [6:7577789319878333064:0] peer: [2:7577789319058796753:0] socket: 121 qp: -1
2025-11-28T14:52:44.230990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789319058796746:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23071831ns) {HandleHandshakeStatus:329(23069580ns) {DropIncomingHandshake:442(11709ns) DropOutgoingHandshake:457(209ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(14607520ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13875839ns max# 2852191ns}}}
2025-11-28T14:52:44.231053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789318802875121:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.220616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319878333074:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:44.230673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:44.230544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789317297610666:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22124464ns) {HandleHandshakeStatus:329(22122887ns) {DropIncomingHandshake:442(9031ns) DropOutgoingHandshake:457(184ns) SwitchToState:179(232ns) ProcessPendingSessionEvents:495(11903185ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11443112ns max# 2618059ns}}}
2025-11-28T14:52:44.220644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:44.231091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789318802875217:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-11-28T14:52:44.230936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789318662406921:2050] [node 3] ICS09 handshake done sender: [4:7577789318662406899:2116] self: [4:7577789318662406905:0] peer: [3:7577789317297610670:0] socket: 106 qp: -1
2025-11-28T14:52:44.230768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:44.230791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789317297610667:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(22624144ns) {HandleHandshakeStatus:329(22622181ns) {DropIncomingHandshake:442(12425ns) DropOutgoingHandshake:457(165ns) SwitchToState:179(1379ns) ProcessPendingSessionEvents:495(15267403ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15165314ns max# 2543924ns}}}
2025-11-28T14:52:44.231103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789318662406921:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:44.221247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319878332979:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.230814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789317916118881:2109] [node 6] ICP22 created new session: [1:7577789317916118919:2109]
2025-11-28T14:52:44.231476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789319058796683:2049] [node 3] ICP18 handshake (actor [2:7577789319058796776:2119]) is too old
2025-11-28T14:52:44.231265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:44.231248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789317297610706:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23412
2025-11-28T14:52:44.232444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.231078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319878333079:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8222
2025-11-28T14:52:44.232490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789318802875121:2050] [node 3] ICP22 created new session: [5:7577789318802875218:2050]
2025-11-28T14:52:44.232783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789317297610695:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:44.231242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789317916118919:2109] [node 6] ICS09 handshake done sender: [1:7577789317916118890:2112] self: [1:7577789317916118888:0] peer: [6:7577789319878333078:0] socket: 154 qp: -1
2025-11-28T14:52:44.232298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319878333069:2123] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:44.232755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789318802875119:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18804918ns) {HandleHandshakeStatus:329(18551548ns) {DropIncomingHandshake:442(634ns) DropOutgoingHandshake:457(1672083ns) SwitchToState:179(200ns) ProcessPendingSessionEvents:495(10646263ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10394258ns max# 1692429ns}}}
2025-11-28T14:52:44.233623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789317297610602:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.231266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789317916118919:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:44.233355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319878332979:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.231286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789317916118882:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(26029065ns) {HandleHandshakeStatus:329(26026885ns) {DropIncomingHandshake:442(983ns) DropOutgoingHandshake:457(270ns) SwitchToState:179(239ns) ProcessPendingSessionEvents:495(18981821ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17597866ns max# 2593988ns}}}
2025-11-28T14:52:44.233800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.232944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789318802875120:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14313) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789318802875209:2120] held: no
2025-11-28T14:52:44.233055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789318802875218:2050] [node 3] ICS09 handshake done sender: [5:7577789318802875206:2119] self: [5:7577789318802875212:0] peer: [3:7577789317297610673:0] socket: 158 qp: -1
2025-11-28T14:52:44.233779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789319878333041:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17314786ns) {HandleHandshakeStatus:329(17311202ns) {DropIncomingHandshake:442(837ns) DropOutgoingHandshake:457(8755ns) SwitchToState:179(385ns) ProcessPendingSessionEvents:495(10639720ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10542291ns max# 1486706ns}}}
2025-11-28T14:52:44.231529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789317916118880:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:44.234048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789317297610602:2048] [node 1] ICP22 created new session: [3:7577789317297610707:2048]
2025-11-28T14:52:44.233360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789318802875218:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:44.234317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.233398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:44.234537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319878332979:2048] [node 1] ICP22 created new session: [6:7577789319878333080:2048]
2025-11-28T14:52:44.232113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:44.234107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789317297610707:2048] [node 1] ICS09 handshake done sender: [3:7577789317297610695:2119] self: [3:7577789317297610700:0] peer: [1:7577789317916118889:0] socket: 112 qp: -1
2025-11-28T14:52:44.234581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319878333080:2048] [node 1] ICS09 handshake done sender: [6:7577789319878333069:2123] self: [6:7577789319878333078:0] peer: [1:7577789317916118888:0] socket: 156 qp: -1
2025-11-28T14:52:44.234588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789319878333040:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(16390312ns) {HandleHandshakeStatus:329(16388685ns) {DropIncomingHandshake:442(721ns) DropOutgoingHandshake:457(5223ns) SwitchToState:179(530ns) ProcessPendingSessionEvents:495(9241775ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9143208ns max# 1361325ns}}}
2025-11-28T14:52:44.234135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789317297610707:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.234614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319878333080:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.234642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.234420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.236518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789318802875209:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:44.236868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789318802875120:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.237368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.237426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789317916118882:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:44.237427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789318802875120:2049] [node 2] ICP22 created new session: [5:7577789318802875220:2049]
2025-11-28T14:52:44.237473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789318802875220:2049] [node 2] ICS09 handshake done sender: [5:7577789318802875209:2120] self: [5:7577789318802875214:0] peer: [2:7577789319058796749:0] socket: 126 qp: -1
2025-11-28T14:52:44.237580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789318802875220:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:44.237609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:44.237751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789317297610602:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:44.253215Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:44.253381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789319058796778:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-11-28T14:52:44.253413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789319058796682:2048] [node 1] ICP30 unregister session Session# [2:7577789319058796778:2048] VirtualId# [2:7577789319058796766:0]
2025-11-28T14:52:44.253437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789319058796778:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.253505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.253545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.254193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:44.254352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789317297610707:2048] [node 1] ICS07 socket disconnect 112 reason# EndOfStream
2025-11-28T14:52:44.254380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789317297610707:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.254479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.254526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789317297610707:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:44.255123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789317297610602:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.255193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789317297610707:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.255236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789317297610602:2048] [node 1] ICP30 unregister session Session# [3:7577789317297610707:2048] VirtualId# [3:7577789317297610700:0]
2025-11-28T14:52:44.255270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.255293Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789317297610602:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.255302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.256021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:44.256217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789318662406919:2048] [node 1] ICS07 socket disconnect 128 reason# EndOfStream
2025-11-28T14:52:44.256262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789318662406919:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.256375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.256429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789318662406919:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:44.256738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:44.256950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789318802875210:2048] [node 1] ICS07 socket disconnect 117 reason# EndOfStream
2025-11-28T14:52:44.257000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789318802875210:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.257054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789318662406820:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.257121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.257143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789318662406919:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.257175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789318802875210:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:44.257184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789318662406820:2048] [node 1] ICP30 unregister session Session# [4:7577789318662406919:2048] VirtualId# [4:7577789318662406904:0]
2025-11-28T14:52:44.257267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.257296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789318662406820:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.257304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.257397Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:44.257532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789319878333080:2048] [node 1] ICS07 socket disconnect 156 reason# EndOfStream
2025-11-28T14:52:44.257563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319878333080:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.257604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.257635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789319878333080:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:44.257691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789318802875119:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.257760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789318802875210:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.257789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789318802875119:2048] [node 1] ICP30 unregister session Session# [5:7577789318802875210:2048] VirtualId# [5:7577789318802875199:0]
2025-11-28T14:52:44.257815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.257875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789318802875119:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.257884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.258223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319878332979:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.258257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319878333080:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.258304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319878332979:2048] [node 1] ICP30 unregister session Session# [6:7577789319878333080:2048] VirtualId# [6:7577789319878333078:0]
2025-11-28T14:52:44.258382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.258405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789319878332979:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.258412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.265134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789317297610602:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:44.267143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789318662406820:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:44.268158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789319878332979:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:44.268155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789318802875119:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:44.292161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:44.292331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789317297610703:2049] [node 2] ICS07 socket disconnect 155 reason# EndOfStream
2025-11-28T14:52:44.292360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789317297610703:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.292475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:44.292544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789317297610703:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:44.293388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789317297610603:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.293441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789317297610703:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.293964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789317297610603:2049] [node 2] ICP30 unregister session Session# [3:7577789317297610703:2049] VirtualId# [3:7577789317297610697:0]
2025-11-28T14:52:44.294057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:44.294087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789317297610603:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.294102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.294147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:44.294289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789318662406917:2049] [node 2] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-28T14:52:44.294465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789318662406917:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.294542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:44.294580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789318662406917:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:44.295031Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:44.295085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789318662406821:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.295121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789318662406917:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.295184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789318662406821:2049] [node 2] ICP30 unregister session Session# [4:7577789318662406917:2049] VirtualId# [4:7577789318662406902:0]
2025-11-28T14:52:44.295232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789318802875220:2049] [node 2] ICS01 socket: 126 reason# EndOfStream
2025-11-28T14:52:44.295280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789318802875120:2049] [node 2] ICP30 unregister session Session# [5:7577789318802875220:2049] VirtualId# [5:7577789318802875214:0]
2025-11-28T14:52:44.295275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:44.295301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789318662406821:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.295323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789318802875220:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.295309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.295394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:44.295437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:44.295752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:44.295896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319878333074:2049] [node 2] ICS01 socket: 121 reason# EndOfStream
2025-11-28T14:52:44.295927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319878332980:2049] [node 2] ICP30 unregister session Session# [6:7577789319878333074:2049] VirtualId# [6:7577789319878333064:0]
2025-11-28T14:52:44.295946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319878333074:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.296020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:44.296062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:44.304009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789317297610603:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:44.305020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789318662406821:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:44.348056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:44.348220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789318662406921:2050] [node 3] ICS01 socket: 106 reason# EndOfStream
2025-11-28T14:52:44.348249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789318662406822:2050] [node 3] ICP30 unregister session Session# [4:7577789318662406921:2050] VirtualId# [4:7577789318662406905:0]
2025-11-28T14:52:44.348274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789318662406921:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.348336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:44.348399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:44.348829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:44.348959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789318802875218:2050] [node 3] ICS01 socket: 158 reason# EndOfStream
2025-11-28T14:52:44.348984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789318802875121:2050] [node 3] ICP30 unregister session Session# [5:7577789318802875218:2050] VirtualId# [5:7577789318802875212:0]
2025-11-28T14:52:44.349004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789318802875218:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.349062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:44.349106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:44.349451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:44.349601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319878333075:2050] [node 3] ICS01 socket: 101 reason# EndOfStream
2025-11-28T14:52:44.349630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319878332981:2050] [node 3] ICP30 unregister session Session# [6:7577789319878333075:2050] VirtualId# [6:7577789319878333063:0]
2025-11-28T14:52:44.349655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319878333075:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.349738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:44.349771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:44.418340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:44.418585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789318802875213:2108] [node 4] ICS01 socket: 109 reason# EndOfStream
2025-11-28T14:52:44.418696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789318802875182:2108] [node 4] ICP30 unregister session Session# [5:7577789318802875213:2108] VirtualId# [5:7577789318802875200:0]
2025-11-28T14:52:44.418741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789318802875213:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.418828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:44.418880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:44.419139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:44.419284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789319878333070:2107] [node 4] ICS07 socket disconnect 119 reason# EndOfStream
2025-11-28T14:52:44.419304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319878333070:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.419387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:44.419436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789319878333070:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:44.420072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319878333041:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25102) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.420118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319878333070:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.420172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319878333041:2107] [node 4] ICP30 unregister session Session# [6:7577789319878333070:2107] VirtualId# [6:7577789319878333061:0]
2025-11-28T14:52:44.420276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:44.420302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789319878333041:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25102) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.420310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25102) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.430069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789319878333041:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:44.492513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:44.492693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789319878333071:2106] [node 5] ICS07 socket disconnect 95 reason# EndOfStream
2025-11-28T14:52:44.492723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319878333071:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.492809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:44.492848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789319878333071:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:44.493478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319878333040:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7534) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.493541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319878333071:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.493571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319878333040:2106] [node 5] ICP30 unregister session Session# [6:7577789319878333071:2106] VirtualId# [6:7577789319878333062:0]
2025-11-28T14:52:44.493654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:44.493677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789319878333040:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7534) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.493687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7534) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:44.583975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789316605457981:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:44.584016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789316605457982:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:44.599123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789320547800594:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.599163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789320547800595:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:44.615593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789319884695216:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.615627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789319884695217:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:44.630577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789317292875452:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.630616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789317292875453:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:44.630648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789317292875454:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:44.646491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789319874114524:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.646535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789319874114525:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:44.646561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789319874114526:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:44.663912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319649885090:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:44.663951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319649885091:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:44.663978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319649885092:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:44.747780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789319884695279:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:44.747991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789316605458044:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:44.748009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789320547800657:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:44.748251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789319884695280:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:44.748336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789316605458045:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:44.748399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789317292875515:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:44.748405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319649885153:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:44.748470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789320547800658:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:44.748515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789319874114587:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:44.749406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789319884695281:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:44.750096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789319649885154:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:44.750201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789317292875516:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:44.750202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789320547800659:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:44.750264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789316605458046:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:44.750301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789319874114588:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.759599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789316605458057:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13879
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.760716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789319884695292:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62000
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.761584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789319884695293:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62000
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.762503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789317292875527:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38500
2025-11-28T14:52:44.762722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789317292875528:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38500
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.764148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789320547800670:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55378
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.764353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789320547800671:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55378
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.764825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789320547800672:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55378
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.766326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789320547800670:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.766381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789320547800673:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55378
2025-11-28T14:52:44.766430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789320547800674:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55378
2025-11-28T14:52:44.768047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458057:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.768466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789320547800671:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.769258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789316605458061:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13879
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.769664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789316605458062:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13879
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.770127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789316605458063:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13879
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.770227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789316605458064:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13879
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:44.770285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789319884695292:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.770391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458061:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.770392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789319884695297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62000
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.770521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789319884695298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62000
2025-11-28T14:52:44.770534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458062:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:44.770600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458063:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:44.770680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789319884695293:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.770599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789317292875527:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.770770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789317292875532:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38500
2025-11-28T14:52:44.770967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789317292875452:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.771036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789317292875528:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.771053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789319884695297:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.771127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789319884695299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62000
2025-11-28T14:52:44.771164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789317292875454:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.771184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458052:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.771212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789316605458052:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.771287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789319884695298:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.771515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789319884695287:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.771547Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789319884695287:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.771777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789316605458052:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:44.772243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789319884695216:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.772415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789320547800672:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.772457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789316605458044:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.772503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.772484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789317292875535:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38500
2025-11-28T14:52:44.772538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789316605458044:2107] [node 4] ICP22 created new session: [1:7577789316605458070:2107]
2025-11-28T14:52:44.772580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789316605458070:2107] [node 4] ICS09 handshake done sender: [1:7577789316605458052:2112] self: [1:7577789316605458051:0] peer: [4:7577789317292875533:0] socket: 104 qp: -1
2025-11-28T14:52:44.772613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789316605458070:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:44.772647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.772957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789319884695299:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.773001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789320547800674:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.773080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789317292875532:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.773107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789319884695287:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:44.773118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789317292875527:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:44.773113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789320547800673:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.773184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789320547800594:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.773363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789317292875452:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.773483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.773525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789317292875452:2048] [node 1] ICP22 created new session: [4:7577789317292875537:2048]
2025-11-28T14:52:44.773562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789317292875537:2048] [node 1] ICS09 handshake done sender: [4:7577789317292875527:2115] self: [4:7577789317292875533:0] peer: [1:7577789316605458051:0] socket: 112 qp: -1
2025-11-28T14:52:44.773603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789317292875537:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.773618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789319884695279:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.773629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.773701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.773755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789319884695279:2107] [node 4] ICP22 created new session: [3:7577789319884695305:2107]
2025-11-28T14:52:44.773797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789319884695305:2107] [node 4] ICS09 handshake done sender: [3:7577789319884695287:2112] self: [3:7577789319884695286:0] peer: [4:7577789317292875534:0] socket: 109 qp: -1
2025-11-28T14:52:44.773846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789319884695305:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:44.773880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.774418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458050:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.774439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789316605458050:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.774875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789319884695217:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.775053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458064:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.775361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458048:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.775386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789316605458048:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.775731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789317292875536:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38500
2025-11-28T14:52:44.775768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789320547800663:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.775792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789320547800663:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.776523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789317292875453:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.776599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789320547800663:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.776590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789319874114602:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64013
2025-11-28T14:52:44.777316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789320547800595:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.777658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789317292875454:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12530) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789317292875528:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.777516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.777549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789320547800595:2049] [node 3] ICP22 created new session: [2:7577789320547800684:2049]
2025-11-28T14:52:44.777788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789319884695309:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62000
2025-11-28T14:52:44.777586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789320547800684:2049] [node 3] ICS09 handshake done sender: [2:7577789320547800663:2111] self: [2:7577789320547800662:0] peer: [3:7577789319884695307:0] socket: 130 qp: -1
2025-11-28T14:52:44.777615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789320547800684:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:44.777642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:44.777839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789319874114602:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.777939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789319874114526:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.778202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789319884695289:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.778227Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789319884695289:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.778342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789317292875528:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.778627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789316605458050:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.778743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789319884695299:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.778794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789320547800665:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.778815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789320547800665:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.779171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789316605458048:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:44.779157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789319874114604:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64013
2025-11-28T14:52:44.779425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789317292875454:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.779522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.779553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789317292875454:2050] [node 3] ICP22 created new session: [4:7577789317292875542:2050]
2025-11-28T14:52:44.779577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789317292875542:2050] [node 3] ICS09 handshake done sender: [4:7577789317292875528:2116] self: [4:7577789317292875534:0] peer: [3:7577789319884695286:0] socket: 114 qp: -1
2025-11-28T14:52:44.779599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789317292875542:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:44.779656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:44.779612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789316605457982:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.779659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.779724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789316605457982:2049] [node 3] ICP22 created new session: [1:7577789316605458076:2049]
2025-11-28T14:52:44.779753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789316605458076:2049] [node 3] ICS09 handshake done sender: [1:7577789316605458050:2111] self: [1:7577789316605458049:0] peer: [3:7577789319884695301:0] socket: 103 qp: -1
2025-11-28T14:52:44.779775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789316605458076:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:44.779820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.780119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789320547800686:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55378
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.780384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789319884695217:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.780423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789319874114605:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64013
2025-11-28T14:52:44.780532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.780603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789319884695217:2049] [node 2] ICP22 created new session: [3:7577789319884695312:2049]
2025-11-28T14:52:44.780646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789319884695312:2049] [node 2] ICS09 handshake done sender: [3:7577789319884695299:2119] self: [3:7577789319884695307:0] peer: [2:7577789320547800662:0] socket: 136 qp: -1
2025-11-28T14:52:44.780673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789319884695312:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:44.780720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789320547800665:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:44.780731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:44.781141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789320547800657:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.781220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789316605457981:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.781236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.781295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.781291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789320547800657:2107] [node 4] ICP22 created new session: [2:7577789320547800689:2107]
2025-11-28T14:52:44.781255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789319874114606:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64013
2025-11-28T14:52:44.781319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789316605457981:2048] [node 2] ICP22 created new session: [1:7577789316605458078:2048]
2025-11-28T14:52:44.781347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789316605458078:2048] [node 2] ICS09 handshake done sender: [1:7577789316605458048:2110] self: [1:7577789316605458047:0] peer: [2:7577789320547800679:0] socket: 102 qp: -1
2025-11-28T14:52:44.781336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789320547800689:2107] [node 4] ICS09 handshake done sender: [2:7577789320547800665:2112] self: [2:7577789320547800664:0] peer: [4:7577789317292875539:0] socket: 131 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.781364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789316605458078:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:44.781372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789320547800689:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:44.781384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:44.781439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.781433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885168:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
2025-11-28T14:52:44.781498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789319884695217:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:44.782254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789320547800671:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.783428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789319874114607:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64013
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.783734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789317292875544:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38500
2025-11-28T14:52:44.785011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789319874114606:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.785129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789320547800594:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.785349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789319874114587:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.785385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.785438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789320547800594:2048] [node 1] ICP22 created new session: [2:7577789320547800691:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.785495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789320547800691:2048] [node 1] ICS09 handshake done sender: [2:7577789320547800671:2116] self: [2:7577789320547800679:0] peer: [1:7577789316605458047:0] socket: 113 qp: -1
2025-11-28T14:52:44.785531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789320547800691:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.785541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789319884695310:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62000
2025-11-28T14:52:44.785557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.785591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789319874114604:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.785665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789319874114524:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.785874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458054:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.785905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789316605458054:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.786171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789319884695289:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:44.786252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885169:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
2025-11-28T14:52:44.786268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789317292875453:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789317292875532:2117] held: no
2025-11-28T14:52:44.786388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319649885168:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.786701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319649885153:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.786748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789319874114602:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.786810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789319884695216:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789319884695293:2116] held: no
2025-11-28T14:52:44.787016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789319874114526:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.787060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789317292875524:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.787089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.787080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789317292875524:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.787148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789319874114526:2050] [node 3] ICP22 created new session: [5:7577789319874114610:2050]
2025-11-28T14:52:44.787182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789319874114610:2050] [node 3] ICS09 handshake done sender: [5:7577789319874114602:2115] self: [5:7577789319874114603:0] peer: [3:7577789319884695288:0] socket: 125 qp: -1
2025-11-28T14:52:44.787207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789319874114610:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:44.787229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:44.787284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789319884695280:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.787588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789317292875532:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:44.787469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:44.787557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789319884695280:2108] [node 5] ICP22 created new session: [3:7577789319884695315:2108]
2025-11-28T14:52:44.787700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789319884695315:2108] [node 5] ICS09 handshake done sender: [3:7577789319884695289:2113] self: [3:7577789319884695288:0] peer: [5:7577789319874114603:0] socket: 120 qp: -1
2025-11-28T14:52:44.787724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789319884695315:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:44.787746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:44.788093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789317292875526:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.788111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789317292875526:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.788939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789317292875453:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.788975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.789005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789317292875453:2049] [node 2] ICP22 created new session: [4:7577789317292875547:2049]
2025-11-28T14:52:44.789033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789317292875547:2049] [node 2] ICS09 handshake done sender: [4:7577789317292875532:2117] self: [4:7577789317292875539:0] peer: [2:7577789320547800664:0] socket: 137 qp: -1
2025-11-28T14:52:44.789051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789317292875547:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:44.789070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:44.789371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319649885169:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.789468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319649885090:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.789603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789316605458054:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:44.789639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789320547800594:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:44.790015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789316605458056:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.790034Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789316605458056:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.790391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789316605458045:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.790447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:44.790421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789319884695293:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:44.790498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789316605458045:2108] [node 5] ICP22 created new session: [1:7577789316605458082:2108]
2025-11-28T14:52:44.790536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789316605458082:2108] [node 5] ICS09 handshake done sender: [1:7577789316605458054:2113] self: [1:7577789316605458053:0] peer: [5:7577789319874114609:0] socket: 144 qp: -1
2025-11-28T14:52:44.790531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789319874114605:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.790559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789316605458082:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:44.790648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.790849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789317292875524:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:44.790969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885172:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.791144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789317292875526:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:44.791077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885173:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
2025-11-28T14:52:44.791513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789317292875515:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.791549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:44.791576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789317292875515:2108] [node 5] ICP22 created new session: [4:7577789317292875549:2108]
2025-11-28T14:52:44.791608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789317292875549:2108] [node 5] ICS09 handshake done sender: [4:7577789317292875524:2113] self: [4:7577789317292875523:0] peer: [5:7577789319874114608:0] socket: 108 qp: -1
2025-11-28T14:52:44.791626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789317292875549:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:44.791644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:44.791791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789319884695216:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.791937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.792020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789319884695216:2048] [node 1] ICP22 created new session: [3:7577789319884695317:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.792055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789319884695317:2048] [node 1] ICS09 handshake done sender: [3:7577789319884695293:2116] self: [3:7577789319884695301:0] peer: [1:7577789316605458049:0] socket: 105 qp: -1
2025-11-28T14:52:44.792087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789319884695317:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.792104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.792121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885174:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
2025-11-28T14:52:44.792178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319649885168:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:44.792465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319649885172:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.792544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319649885153:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.792589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.792597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789316605458056:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:44.792573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789317292875516:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.792604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:44.792637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789317292875516:2109] [node 6] ICP22 created new session: [4:7577789317292875551:2109]
2025-11-28T14:52:44.792662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319649885153:2106] [node 4] ICP22 created new session: [6:7577789319649885176:2106]
2025-11-28T14:52:44.792658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789317292875551:2109] [node 6] ICS09 handshake done sender: [4:7577789317292875526:2114] self: [4:7577789317292875525:0] peer: [6:7577789319649885170:0] socket: 151 qp: -1
2025-11-28T14:52:44.792713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319649885176:2106] [node 4] ICS09 handshake done sender: [6:7577789319649885168:2115] self: [6:7577789319649885170:0] peer: [4:7577789317292875525:0] socket: 152 qp: -1
2025-11-28T14:52:44.792678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789317292875551:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:44.792738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319649885176:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:44.792694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:44.792769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.793494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789316605458046:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.793553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:44.793600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789316605458046:2109] [node 6] ICP22 created new session: [1:7577789316605458085:2109]
2025-11-28T14:52:44.793707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789316605458085:2109] [node 6] ICS09 handshake done sender: [1:7577789316605458056:2114] self: [1:7577789316605458055:0] peer: [6:7577789319649885171:0] socket: 107 qp: -1
2025-11-28T14:52:44.793749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789316605458085:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:44.793783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:44.793883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789319874114524:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789319874114604:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.794398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789319874114612:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64013
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.794851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789319874114613:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64013
2025-11-28T14:52:44.795286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789319874114525:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.795686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789317292875553:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38500
2025-11-28T14:52:44.796078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789319874114604:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:44.796118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789320547800667:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.796141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789320547800667:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.796721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789319874114606:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.797217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789319874114524:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.797251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.797203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885175:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
2025-11-28T14:52:44.797277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789319874114524:2048] [node 1] ICP22 created new session: [5:7577789319874114617:2048]
2025-11-28T14:52:44.797271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789320547800667:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:44.797305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789319874114617:2048] [node 1] ICS09 handshake done sender: [5:7577789319874114604:2116] self: [5:7577789319874114609:0] peer: [1:7577789316605458053:0] socket: 145 qp: -1
2025-11-28T14:52:44.797324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789319874114617:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.797339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.797602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319649885090:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789319649885169:2116] held: no
2025-11-28T14:52:44.797631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789320547800658:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.797713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:44.797762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789320547800658:2108] [node 5] ICP22 created new session: [2:7577789320547800695:2108]
2025-11-28T14:52:44.797798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789320547800695:2108] [node 5] ICS09 handshake done sender: [2:7577789320547800667:2113] self: [2:7577789320547800666:0] peer: [5:7577789319874114615:0] socket: 148 qp: -1
2025-11-28T14:52:44.797834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789320547800695:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:44.797865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:44.797990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789319874114587:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.798018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:44.798040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789319874114587:2108] [node 4] ICP22 created new session: [5:7577789319874114619:2108]
2025-11-28T14:52:44.798066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789319874114619:2108] [node 4] ICS09 handshake done sender: [5:7577789319874114606:2118] self: [5:7577789319874114608:0] peer: [4:7577789317292875523:0] socket: 150 qp: -1
2025-11-28T14:52:44.798100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789319874114619:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:44.798096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319649885091:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.798113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:44.798298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319649885174:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.798586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319649885169:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:44.798981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319649885090:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.799013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:44.799046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319649885090:2048] [node 1] ICP22 created new session: [6:7577789319649885181:2048]
2025-11-28T14:52:44.799075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319649885181:2048] [node 1] ICS09 handshake done sender: [6:7577789319649885169:2116] self: [6:7577789319649885171:0] peer: [1:7577789316605458055:0] socket: 135 qp: -1
2025-11-28T14:52:44.799141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319649885181:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:44.799168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:44.799364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789320547800669:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.799398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789320547800669:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.800434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789320547800669:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.800478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319649885092:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.800551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885179:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
2025-11-28T14:52:44.800729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789320547800659:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.800820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:44.800867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789320547800659:2109] [node 6] ICP22 created new session: [2:7577789320547800698:2109]
2025-11-28T14:52:44.800916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789320547800698:2109] [node 6] ICS09 handshake done sender: [2:7577789320547800669:2114] self: [2:7577789320547800668:0] peer: [6:7577789319649885178:0] socket: 119 qp: -1
2025-11-28T14:52:44.800954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789320547800698:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:44.800985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:44.800982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789319884695291:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.801007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789319884695291:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.801248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789319874114525:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789319874114605:2117] held: no
2025-11-28T14:52:44.801601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789319884695291:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:44.801823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789319884695281:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.802043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:44.802192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789319884695281:2109] [node 6] ICP22 created new session: [3:7577789319884695320:2109]
2025-11-28T14:52:44.802247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789319884695320:2109] [node 6] ICS09 handshake done sender: [3:7577789319884695291:2114] self: [3:7577789319884695290:0] peer: [6:7577789319649885183:0] socket: 159 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.802270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789319884695320:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:44.802289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:44.802336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885184:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
2025-11-28T14:52:44.802387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789319874114621:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64013
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.804192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789319874114622:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64013
2025-11-28T14:52:44.804206Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319649885091:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789319649885172:2117] held: no
2025-11-28T14:52:44.804273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789319874114605:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:44.804443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789319874114525:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.804471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.804496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789319874114525:2049] [node 2] ICP22 created new session: [5:7577789319874114623:2049]
2025-11-28T14:52:44.804524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789319874114623:2049] [node 2] ICS09 handshake done sender: [5:7577789319874114605:2117] self: [5:7577789319874114615:0] peer: [2:7577789320547800666:0] socket: 149 qp: -1
2025-11-28T14:52:44.804540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789319874114623:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:44.804558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:44.805849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319649885172:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:44.806182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789319874114622:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.806367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319649885092:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12530) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789319649885174:2119] held: no
2025-11-28T14:52:44.806844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319649885091:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.806875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:44.806905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319649885091:2049] [node 2] ICP22 created new session: [6:7577789319649885186:2049]
2025-11-28T14:52:44.806944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319649885186:2049] [node 2] ICS09 handshake done sender: [6:7577789319649885172:2117] self: [6:7577789319649885178:0] peer: [2:7577789320547800668:0] socket: 155 qp: -1
2025-11-28T14:52:44.806973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319649885186:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:44.807000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: 2025-11-28T14:52:44.807777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789319649885179:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:44.807848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885185:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
2025-11-28T14:52:44.808742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319649885174:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:44.808929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789319649885154:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:44.808962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319649885092:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.808986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:44.809016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319649885092:2050] [node 3] ICP22 created new session: [6:7577789319649885189:2050]
2025-11-28T14:52:44.809057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319649885189:2050] [node 3] ICS09 handshake done sender: [6:7577789319649885174:2119] self: [6:7577789319649885183:0] peer: [3:7577789319884695290:0] socket: 160 qp: -1
2025-11-28T14:52:44.809084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319649885189:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:44.809104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:44.809948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789319874114598:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:44.809970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789319874114598:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:44.812126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789319874114598:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:44.812422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789319874114588:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:44.812499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:44.812543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789319874114588:2109] [node 6] ICP22 created new session: [5:7577789319874114627:2109]
2025-11-28T14:52:44.812594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789319874114627:2109] [node 6] ICS09 handshake done sender: [5:7577789319874114598:2114] self: [5:7577789319874114597:0] peer: [6:7577789319649885188:0] socket: 161 qp: -1
2025-11-28T14:52:44.812624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789319874114627:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:44.812668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:44.813777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789319649885191:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1334
2025-11-28T14:52:44.814147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789319649885179:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:44.814335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789319649885154:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:44.814384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:44.814429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789319649885154:2107] [node 5] ICP22 created new session: [6:7577789319649885192:2107]
2025-11-28T14:52:44.814461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789319649885192:2107] [node 5] ICS09 handshake done sender: [6:7577789319649885179:2121] self: [6:7577789319649885188:0] peer: [5:7577789319874114597:0] socket: 162 qp: -1
2025-11-28T14:52:44.814484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789319649885192:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:44.814515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:44.817119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789319649885154:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:44.836759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:44.836966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789320547800691:2048] [node 1] ICS07 socket disconnect 113 reason# EndOfStream
2025-11-28T14:52:44.836996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789320547800691:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.837059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.837098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789320547800691:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:44.837742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:44.837874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789319884695317:2048] [node 1] ICS01 socket: 105 reason# EndOfStream
2025-11-28T14:52:44.837905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789319884695216:2048] [node 1] ICP30 unregister session Session# [3:7577789319884695317:2048] VirtualId# [3:7577789319884695301:0]
2025-11-28T14:52:44.837928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789319884695317:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.837971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.838011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.839595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789320547800594:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:44.839622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789320547800691:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:44.840224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:44.840386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789317292875537:2048] [node 1] ICS07 socket disconnect 112 reason# EndOfStream
2025-11-28T14:52:44.840411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789317292875537:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.840458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.840490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789317292875537:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:44.840717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789320547800594:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.840757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789320547800691:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.840784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789320547800594:2048] [node 1] ICP30 unregister session Session# [2:7577789320547800691:2048] VirtualId# [2:7577789320547800679:0]
2025-11-28T14:52:44.840869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.840896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789320547800594:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.840904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.841213Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:44.841282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789317292875452:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.841316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789317292875537:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.841339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789317292875452:2048] [node 1] ICP30 unregister session Session# [4:7577789317292875537:2048] VirtualId# [4:7577789317292875533:0]
2025-11-28T14:52:44.841360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789319874114617:2048] [node 1] ICS01 socket: 145 reason# EndOfStream
2025-11-28T14:52:44.841388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789319874114524:2048] [node 1] ICP30 unregister session Session# [5:7577789319874114617:2048] VirtualId# [5:7577789319874114609:0]
2025-11-28T14:52:44.841394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.841407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789319874114617:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.841418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789317292875452:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.841441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.841457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.841501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.841904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:44.842057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319649885181:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-11-28T14:52:44.842090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319649885090:2048] [node 1] ICP30 unregister session Session# [6:7577789319649885181:2048] VirtualId# [6:7577789319649885171:0]
2025-11-28T14:52:44.842120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319649885181:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.842204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:44.842248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:44.850830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789320547800594:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:44.850830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789317292875452:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:44.873763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:44.873909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789319884695312:2049] [node 2] ICS07 socket disconnect 136 reason# EndOfStream
2025-11-28T14:52:44.873930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789319884695312:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.873996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:44.874031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789319884695312:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:44.875760Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789319884695217:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21208) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:44.875790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789319884695312:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:44.876160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789319884695217:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21208) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.876192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789319884695312:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.876211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789319884695217:2049] [node 2] ICP30 unregister session Session# [3:7577789319884695312:2049] VirtualId# [3:7577789319884695307:0]
2025-11-28T14:52:44.876283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:44.876307Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789319884695217:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21208) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.876318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21208) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.878102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:44.878230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789317292875547:2049] [node 2] ICS01 socket: 137 reason# EndOfStream
2025-11-28T14:52:44.878264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789317292875453:2049] [node 2] ICP30 unregister session Session# [4:7577789317292875547:2049] VirtualId# [4:7577789317292875539:0]
2025-11-28T14:52:44.878285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789317292875547:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.878346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:44.878379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:44.878859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:44.878984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789319874114623:2049] [node 2] ICS01 socket: 149 reason# EndOfStream
2025-11-28T14:52:44.879008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789319874114525:2049] [node 2] ICP30 unregister session Session# [5:7577789319874114623:2049] VirtualId# [5:7577789319874114615:0]
2025-11-28T14:52:44.879025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789319874114623:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.879069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:44.879099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:44.879502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:44.879654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319649885186:2049] [node 2] ICS01 socket: 155 reason# EndOfStream
2025-11-28T14:52:44.879684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319649885091:2049] [node 2] ICP30 unregister session Session# [6:7577789319649885186:2049] VirtualId# [6:7577789319649885178:0]
2025-11-28T14:52:44.879720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319649885186:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.879811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:44.879846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:44.896524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:44.896694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789317292875542:2050] [node 3] ICS01 socket: 114 reason# EndOfStream
2025-11-28T14:52:44.896740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789317292875454:2050] [node 3] ICP30 unregister session Session# [4:7577789317292875542:2050] VirtualId# [4:7577789317292875534:0]
2025-11-28T14:52:44.896769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789317292875542:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.896845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:44.896882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:44.897315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:44.897447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789319874114610:2050] [node 3] ICS07 socket disconnect 125 reason# EndOfStream
2025-11-28T14:52:44.897475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789319874114610:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.897534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:44.897575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789319874114610:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:44.897938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:44.898091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319649885189:2050] [node 3] ICS01 socket: 160 reason# EndOfStream
2025-11-28T14:52:44.898131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319649885092:2050] [node 3] ICP30 unregister session Session# [6:7577789319649885189:2050] VirtualId# [6:7577789319649885183:0]
2025-11-28T14:52:44.898123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789319874114526:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12530) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.898155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319649885189:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.898158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789319874114610:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.898179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789319874114526:2050] [node 3] ICP30 unregister session Session# [5:7577789319874114610:2050] VirtualId# [5:7577789319874114603:0]
2025-11-28T14:52:44.898216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:44.898231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:44.898245Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789319874114526:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12530) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.898258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12530) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.898277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:44.908167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789319874114526:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:44.925852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:44.926024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789319874114619:2108] [node 4] ICS07 socket disconnect 150 reason# EndOfStream
2025-11-28T14:52:44.926049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789319874114619:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.926110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:44.926142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789319874114619:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:44.926739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789319874114587:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25750) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.926775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789319874114619:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.926801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789319874114587:2108] [node 4] ICP30 unregister session Session# [5:7577789319874114619:2108] VirtualId# [5:7577789319874114608:0]
2025-11-28T14:52:44.926867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:44.926888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789319874114587:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.926897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.927084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:44.927237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789319649885176:2106] [node 4] ICS07 socket disconnect 152 reason# EndOfStream
2025-11-28T14:52:44.927260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319649885176:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.927333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:44.927370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789319649885176:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:44.927999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319649885153:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25750) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.928038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319649885176:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.928065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319649885153:2106] [node 4] ICP30 unregister session Session# [6:7577789319649885176:2106] VirtualId# [6:7577789319649885170:0]
2025-11-28T14:52:44.928145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:44.928167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789319649885153:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.928177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.937880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789319649885153:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:44.946545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:44.946774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789319649885192:2107] [node 5] ICS07 socket disconnect 162 reason# EndOfStream
2025-11-28T14:52:44.946810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789319649885192:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:44.946909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:44.946965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789319649885192:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:44.947789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789319649885154:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3578) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:44.947843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789319649885192:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:44.947880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789319649885154:2107] [node 5] ICP30 unregister session Session# [6:7577789319649885192:2107] VirtualId# [6:7577789319649885188:0]
2025-11-28T14:52:44.947955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:44.947992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789319649885154:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3578) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.948012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3578) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:44.957334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789319649885154:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:45.008452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789323835915111:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:45.008489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789323835915112:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:45.025314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789323755428727:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:45.025349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789323755428726:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.043706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789323876454737:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.043807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789323876454738:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:45.060788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789323689580481:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.060834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789323689580483:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:45.060884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789323689580482:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:45.077610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789324418672275:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.077655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789324418672277:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:45.077677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789324418672276:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:45.094571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789320832923631:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.094615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789320832923633:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:45.094695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789320832923632:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:45.183589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789323876454800:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:45.184825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789323876454801:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:45.184784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789320832923695:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:45.184857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789323835915175:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:45.185119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789324418672338:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:45.184999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789323835915176:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:45.185207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789323689580544:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:45.185190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789323755428790:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:45.185188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789323876454802:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:45.185416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789324418672339:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:45.185808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789323835915177:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:45.185216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789323755428791:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:45.185581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789323689580545:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:45.186068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789323755428792:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:45.186499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789320832923696:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.194609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789323876454816:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.194762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789323689580556:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14973
2025-11-28T14:52:45.194730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789324418672353:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:365
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.194871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789323689580557:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14973
2025-11-28T14:52:45.195220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789323835915191:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40474
2025-11-28T14:52:45.195337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789323835915192:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40474
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.199051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789323835915193:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40474
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.200255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789323876454817:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-28T14:52:45.200935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789324418672353:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.200833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789323835915194:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40474
2025-11-28T14:52:45.200905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915193:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.202159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789323876454816:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.202159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789323689580557:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.202160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915191:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.202371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915192:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.202376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789323876454817:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.202540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915194:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.202553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789323689580481:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.202716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789323876454818:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-28T14:52:45.202747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915183:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.202756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789323689580556:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.202726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789323876454737:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.202772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789323835915183:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.203139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789323876454818:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.203538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915179:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.203174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789323689580483:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.203570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789323835915179:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.203626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789323876454808:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.203733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789323755428802:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30078
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.203849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789323835915183:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:45.204922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789323755428801:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30078
2025-11-28T14:52:45.203649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789323876454808:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.205133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789323755428803:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30078
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.205563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789323755428805:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30078
2025-11-28T14:52:45.205571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789323876454808:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.205622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789323755428804:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30078
2025-11-28T14:52:45.205795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789323689580563:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14973
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.205869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789323835915175:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.206092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789323755428801:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.205967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:45.206122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789323755428803:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.206112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789323835915175:2107] [node 4] ICP22 created new session: [1:7577789323835915201:2107]
2025-11-28T14:52:45.206137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789323689580564:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14973
2025-11-28T14:52:45.206669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789323876454800:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.206293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789323755428802:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.206281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789323835915201:2107] [node 4] ICS09 handshake done sender: [1:7577789323835915183:2113] self: [1:7577789323835915182:0] peer: [4:7577789323689580561:0] socket: 107 qp: -1
2025-11-28T14:52:45.206798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:45.207073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789323689580557:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:45.206560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789323755428805:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.206906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789323876454800:2107] [node 4] ICP22 created new session: [3:7577789323876454824:2107]
2025-11-28T14:52:45.206788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789323835915201:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:45.207556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789323689580556:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:45.207203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789323876454824:2107] [node 4] ICS09 handshake done sender: [3:7577789323876454808:2112] self: [3:7577789323876454807:0] peer: [4:7577789323689580562:0] socket: 105 qp: -1
2025-11-28T14:52:45.206565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789323755428804:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.206985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.207291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789323876454824:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:45.207239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789323755428726:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.207433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:45.208210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789323689580481:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.208602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789324418672355:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:365
2025-11-28T14:52:45.208289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.208729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789323689580481:2048] [node 1] ICP22 created new session: [4:7577789323689580565:2048]
2025-11-28T14:52:45.208755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789323689580565:2048] [node 1] ICS09 handshake done sender: [4:7577789323689580557:2116] self: [4:7577789323689580561:0] peer: [1:7577789323835915182:0] socket: 119 qp: -1
2025-11-28T14:52:45.208777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789323689580565:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:45.209223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915181:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.209256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789323835915181:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.208851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.209212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789323689580483:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.209244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.209480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789323689580483:2050] [node 3] ICP22 created new session: [4:7577789323689580567:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.209598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789323689580567:2050] [node 3] ICS09 handshake done sender: [4:7577789323689580556:2115] self: [4:7577789323689580562:0] peer: [3:7577789323876454807:0] socket: 117 qp: -1
2025-11-28T14:52:45.210647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789323876454823:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-28T14:52:45.209964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789323689580567:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:45.209982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:45.210763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789323835915179:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.210858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789324418672355:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.210906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789323876454826:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-28T14:52:45.211042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789323689580481:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:45.211202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789324418672276:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.211709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789323876454816:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:45.211866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789323835915202:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40474
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.211845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789323755428798:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.211890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789323755428798:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.212087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789323876454737:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.212122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.212548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789323876454737:2048] [node 1] ICP22 created new session: [3:7577789323876454828:2048]
2025-11-28T14:52:45.212591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789323835915112:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.212746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.212585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789323876454828:2048] [node 1] ICS09 handshake done sender: [3:7577789323876454816:2115] self: [3:7577789323876454819:0] peer: [1:7577789323835915178:0] socket: 109 qp: -1
2025-11-28T14:52:45.212550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789324418672357:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:365
2025-11-28T14:52:45.212643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789323876454828:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:45.212775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789323835915112:2049] [node 3] ICP22 created new session: [1:7577789323835915206:2049]
2025-11-28T14:52:45.212739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.212810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789323835915206:2049] [node 3] ICS09 handshake done sender: [1:7577789323835915179:2111] self: [1:7577789323835915178:0] peer: [3:7577789323876454819:0] socket: 104 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.212839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789323835915206:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:45.212895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.213039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789323755428798:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:45.212970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789324418672358:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:365
2025-11-28T14:52:45.213123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789323755428817:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30078
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.213245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789323755428790:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.213313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789324418672359:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:365
2025-11-28T14:52:45.213405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:45.213455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789323755428790:2109] [node 5] ICP22 created new session: [2:7577789323755428818:2109]
2025-11-28T14:52:45.213507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789323755428818:2109] [node 5] ICS09 handshake done sender: [2:7577789323755428798:2113] self: [2:7577789323755428797:0] peer: [5:7577789324418672356:0] socket: 142 qp: -1
2025-11-28T14:52:45.213547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789323755428818:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:45.213581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:45.213616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789324418672355:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.214283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789324418672276:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.214331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.214375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789324418672276:2049] [node 2] ICP22 created new session: [5:7577789324418672360:2049]
2025-11-28T14:52:45.214422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789324418672360:2049] [node 2] ICS09 handshake done sender: [5:7577789324418672355:2116] self: [5:7577789324418672356:0] peer: [2:7577789323755428797:0] socket: 144 qp: -1
2025-11-28T14:52:45.214473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789323876454823:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.215180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915202:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.214492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789324418672360:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:45.214518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:45.217121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789323689580483:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:45.217528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789323835915181:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.217605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789324418672276:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.217971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789323755428802:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:45.218059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789323835915111:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.218097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.218090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789323689580569:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14973
2025-11-28T14:52:45.218138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789323755428726:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.218233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789323835915111:2048] [node 2] ICP22 created new session: [1:7577789323835915209:2048]
2025-11-28T14:52:45.218094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789323689580570:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14973
2025-11-28T14:52:45.218285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.218446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789323835915209:2048] [node 2] ICS09 handshake done sender: [1:7577789323835915181:2112] self: [1:7577789323835915180:0] peer: [2:7577789323755428813:0] socket: 106 qp: -1
2025-11-28T14:52:45.218527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789323835915209:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:45.218490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789323755428726:2048] [node 1] ICP22 created new session: [2:7577789323755428821:2048]
2025-11-28T14:52:45.218562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:45.218590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789323755428821:2048] [node 1] ICS09 handshake done sender: [2:7577789323755428802:2116] self: [2:7577789323755428813:0] peer: [1:7577789323835915180:0] socket: 131 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.218627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789323755428821:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:45.218661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.218670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789324418672357:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.218684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923710:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
2025-11-28T14:52:45.218881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789323876454738:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.219844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789323689580570:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.219834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923711:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.220194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789320832923710:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.220200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789324418672363:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:365
2025-11-28T14:52:45.220280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923712:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
2025-11-28T14:52:45.220430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789324418672277:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.220479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789320832923696:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.220653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789323835915111:2048] [node 2] ICP18 handshake (actor [1:7577789323835915202:2120]) is too old
2025-11-28T14:52:45.220687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789323689580553:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.220730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789324418672363:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.220714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789323689580553:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.220794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789323876454810:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.220832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789324418672338:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.220821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789323876454810:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.221091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789320832923712:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.221755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789323689580555:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.221775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789323689580555:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.222177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789320832923633:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.222246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789323876454810:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:45.222341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789323689580553:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:45.222419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789323689580555:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:45.222514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789323876454815:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.222541Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789323876454815:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.222583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789323876454801:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.222621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: 2025-11-28T14:52:45.222783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789324418672358:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:45.222803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789323755428796:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.223373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789320832923711:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.222798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789323876454801:2108] [node 5] ICP22 created new session: [3:7577789323876454833:2108]
2025-11-28T14:52:45.223824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923715:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
2025-11-28T14:52:45.222826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789323755428796:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.223835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789323876454833:2108] [node 5] ICS09 handshake done sender: [3:7577789323876454810:2113] self: [3:7577789323876454809:0] peer: [5:7577789324418672364:0] socket: 125 qp: -1
2025-11-28T14:52:45.223845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789323689580569:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.223881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789323689580544:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.223901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789324418672366:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:365
2025-11-28T14:52:45.223861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789323876454833:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:45.223912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789320832923631:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.223886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:45.223907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.224069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789324418672275:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:52:45.224059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789323689580544:2109] [node 6] ICP22 created new session: [4:7577789323689580576:2109]
::ffff:127.1.0.0
2025-11-28T14:52:45.224082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915190:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.224094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789323689580576:2109] [node 6] ICS09 handshake done sender: [4:7577789323689580553:2113] self: [4:7577789323689580552:0] peer: [6:7577789320832923713:0] socket: 128 qp: -1
2025-11-28T14:52:45.224255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789324418672367:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:365
2025-11-28T14:52:45.224107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789323835915190:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.224661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789320832923710:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:45.224113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789323689580576:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:45.224126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789323689580545:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.224132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.225155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789323876454815:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:45.224162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:45.224211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789323689580545:2108] [node 5] ICP22 created new session: [4:7577789323689580578:2108]
2025-11-28T14:52:45.225418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789320832923696:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.224723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789323689580578:2108] [node 5] ICS09 handshake done sender: [4:7577789323689580555:2114] self: [4:7577789323689580554:0] peer: [5:7577789324418672365:0] socket: 156 qp: -1
2025-11-28T14:52:45.225529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789323755428796:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:45.225440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789324418672357:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:45.224846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789323689580578:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:45.225046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:45.226199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:45.226214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789323835915185:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.226809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789323876454802:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.226762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789320832923696:2109] [node 4] ICP22 created new session: [6:7577789320832923718:2109]
2025-11-28T14:52:45.226234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789323835915185:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.227004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.227018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789320832923718:2109] [node 4] ICS09 handshake done sender: [6:7577789320832923710:2116] self: [6:7577789320832923713:0] peer: [4:7577789323689580552:0] socket: 155 qp: -1
2025-11-28T14:52:45.227040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789324418672277:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789324418672357:2117] held: no
2025-11-28T14:52:45.227119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789323755428727:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.227082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789323876454802:2109] [node 6] ICP22 created new session: [3:7577789323876454837:2109]
2025-11-28T14:52:45.227088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789320832923718:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:45.227157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789323876454837:2109] [node 6] ICS09 handshake done sender: [3:7577789323876454815:2114] self: [3:7577789323876454814:0] peer: [6:7577789320832923714:0] socket: 160 qp: -1
2025-11-28T14:52:45.227183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789323876454837:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:45.227362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.227124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:45.227269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.227475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789323755428727:2049] [node 3] ICP22 created new session: [2:7577789323755428825:2049]
2025-11-28T14:52:45.227501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789324418672363:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:45.227509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789323835915190:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:45.227518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789323755428825:2049] [node 3] ICS09 handshake done sender: [2:7577789323755428796:2112] self: [2:7577789323755428795:0] peer: [3:7577789323876454830:0] socket: 143 qp: -1
2025-11-28T14:52:45.227637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789323755428825:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:45.227664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.228073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789323835915185:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:45.228178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923717:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
2025-11-28T14:52:45.228231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789323835915177:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.228152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789324418672277:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.228236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.228291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.228343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789324418672277:2050] [node 3] ICP22 created new session: [5:7577789324418672370:2050]
2025-11-28T14:52:45.228375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789324418672338:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.228416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789323835915177:2110] [node 6] ICP22 created new session: [1:7577789323835915213:2110]
2025-11-28T14:52:45.228398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789324418672370:2050] [node 3] ICS09 handshake done sender: [5:7577789324418672357:2117] self: [5:7577789324418672364:0] peer: [3:7577789323876454809:0] socket: 136 qp: -1
2025-11-28T14:52:45.228403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:45.228465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789323835915213:2110] [node 6] ICS09 handshake done sender: [1:7577789323835915190:2115] self: [1:7577789323835915186:0] peer: [6:7577789320832923716:0] socket: 158 qp: -1
2025-11-28T14:52:45.228426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789324418672370:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:45.228433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789324418672338:2108] [node 4] ICP22 created new session: [5:7577789324418672371:2108]
2025-11-28T14:52:45.228509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789323835915213:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:45.228468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:45.228475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789324418672371:2108] [node 4] ICS09 handshake done sender: [5:7577789324418672363:2120] self: [5:7577789324418672365:0] peer: [4:7577789323689580554:0] socket: 157 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.228499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789324418672371:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:45.228523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:45.228537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.229333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789323689580482:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.228657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789323835915176:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.228689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:45.229084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789323835915176:2109] [node 5] ICP22 created new session: [1:7577789323835915215:2109]
2025-11-28T14:52:45.229517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923719:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
2025-11-28T14:52:45.229502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789323835915215:2109] [node 5] ICS09 handshake done sender: [1:7577789323835915185:2114] self: [1:7577789323835915184:0] peer: [5:7577789324418672368:0] socket: 149 qp: -1
2025-11-28T14:52:45.229524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789323835915215:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:45.229542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.229999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789323876454836:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-28T14:52:45.230010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789323876454738:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32373) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789323876454823:2118] held: no
2025-11-28T14:52:45.230081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789320832923712:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:45.230174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789323755428800:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.230217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789323755428800:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.231235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789323876454823:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.231269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923721:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
2025-11-28T14:52:45.231504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789323755428800:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:45.231571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789323876454738:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.231597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.231910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789323755428791:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.231736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789323876454738:2049] [node 2] ICP22 created new session: [3:7577789323876454840:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.231780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789323876454840:2049] [node 2] ICS09 handshake done sender: [3:7577789323876454823:2118] self: [3:7577789323876454830:0] peer: [2:7577789323755428795:0] socket: 145 qp: -1
2025-11-28T14:52:45.232718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789320832923633:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.231907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789323876454840:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:45.232863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:45.232982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.232993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789323755428791:2107] [node 4] ICP22 created new session: [2:7577789323755428828:2107]
2025-11-28T14:52:45.231924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:45.232986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789323876454839:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-28T14:52:45.233140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789323755428828:2107] [node 4] ICS09 handshake done sender: [2:7577789323755428800:2114] self: [2:7577789323755428799:0] peer: [4:7577789323689580580:0] socket: 132 qp: -1
2025-11-28T14:52:45.233126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789320832923633:2050] [node 3] ICP22 created new session: [6:7577789320832923723:2050]
2025-11-28T14:52:45.233170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789323755428828:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:45.233179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789320832923723:2050] [node 3] ICS09 handshake done sender: [6:7577789320832923712:2118] self: [6:7577789320832923714:0] peer: [3:7577789323876454814:0] socket: 161 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:45.233202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:45.233178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789320832923719:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:45.233202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789320832923723:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:45.233234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:45.233296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789323689580581:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14973
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.235032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789324418672369:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:365
2025-11-28T14:52:45.235582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789323689580482:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32373) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789323689580569:2119] held: no
2025-11-28T14:52:45.235837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789324418672275:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6814) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789324418672358:2118] held: no
2025-11-28T14:52:45.235960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789320832923711:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:45.236276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789320832923631:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6814) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789320832923711:2117] held: no
2025-11-28T14:52:45.236292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789323689580569:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.236407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789323689580482:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.236450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.236487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789323689580482:2049] [node 2] ICP22 created new session: [4:7577789323689580582:2049]
2025-11-28T14:52:45.236553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789323689580582:2049] [node 2] ICS09 handshake done sender: [4:7577789323689580569:2119] self: [4:7577789323689580580:0] peer: [2:7577789323755428799:0] socket: 108 qp: -1
2025-11-28T14:52:45.236607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789323689580582:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:45.236648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:45.236658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789324418672358:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:45.236784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789320832923631:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.236839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.236927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789320832923631:2048] [node 1] ICP22 created new session: [6:7577789320832923725:2048]
2025-11-28T14:52:45.236974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789320832923725:2048] [node 1] ICS09 handshake done sender: [6:7577789320832923711:2117] self: [6:7577789320832923716:0] peer: [1:7577789323835915186:0] socket: 159 qp: -1
2025-11-28T14:52:45.236989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789324418672275:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.237004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789320832923725:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:45.237024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.237016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.237048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789324418672275:2048] [node 1] ICP22 created new session: [5:7577789324418672375:2048]
2025-11-28T14:52:45.237074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789324418672375:2048] [node 1] ICS09 handshake done sender: [5:7577789324418672358:2118] self: [5:7577789324418672368:0] peer: [1:7577789323835915184:0] socket: 150 qp: -1
2025-11-28T14:52:45.237096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789324418672375:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:45.237139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.237857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789320832923632:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.238312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923727:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
2025-11-28T14:52:45.238392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789323755428807:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.238423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789323755428807:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.238606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789320832923727:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.238711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789320832923695:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.238921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789324418672349:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.238954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789324418672349:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.239359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789323755428807:2120] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.239523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789323755428792:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.239574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923730:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
2025-11-28T14:52:45.239610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.239665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789323755428792:2110] [node 6] ICP22 created new session: [2:7577789323755428831:2110]
2025-11-28T14:52:45.239692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789324418672349:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.239721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789323755428831:2110] [node 6] ICS09 handshake done sender: [2:7577789323755428807:2120] self: [2:7577789323755428806:0] peer: [6:7577789320832923728:0] socket: 168 qp: -1
2025-11-28T14:52:45.239761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789323755428831:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:45.239815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789320832923731:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36666
2025-11-28T14:52:45.239848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.239963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789324418672339:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.240079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789320832923719:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.240138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.240203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789324418672339:2109] [node 6] ICP22 created new session: [5:7577789324418672378:2109]
2025-11-28T14:52:45.240837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789320832923632:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32373) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789320832923719:2121] held: no
2025-11-28T14:52:45.240334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789324418672378:2109] [node 6] ICS09 handshake done sender: [5:7577789324418672349:2114] self: [5:7577789324418672348:0] peer: [6:7577789320832923729:0] socket: 102 qp: -1
2025-11-28T14:52:45.240391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789324418672378:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:45.240421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.241448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789320832923632:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.241492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.241589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789320832923632:2049] [node 2] ICP22 created new session: [6:7577789320832923732:2049]
2025-11-28T14:52:45.241634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789320832923732:2049] [node 2] ICS09 handshake done sender: [6:7577789320832923719:2121] self: [6:7577789320832923728:0] peer: [2:7577789323755428806:0] socket: 169 qp: -1
2025-11-28T14:52:45.241673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789320832923732:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:45.241712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:45.241752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789320832923727:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:45.243109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789320832923695:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.243160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:45.243200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789320832923695:2107] [node 5] ICP22 created new session: [6:7577789320832923734:2107]
2025-11-28T14:52:45.243239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789320832923734:2107] [node 5] ICS09 handshake done sender: [6:7577789320832923727:2123] self: [6:7577789320832923729:0] peer: [5:7577789324418672348:0] socket: 103 qp: -1
2025-11-28T14:52:45.243268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789320832923734:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:45.243297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:45.243910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789320832923695:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:45.265000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:45.265213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789323755428821:2048] [node 1] ICS01 socket: 131 reason# EndOfStream
2025-11-28T14:52:45.265261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789323755428726:2048] [node 1] ICP30 unregister session Session# [2:7577789323755428821:2048] VirtualId# [2:7577789323755428813:0]
2025-11-28T14:52:45.265301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789323755428821:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.265396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.265469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.266122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:45.266290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789323876454828:2048] [node 1] ICS07 socket disconnect 109 reason# EndOfStream
2025-11-28T14:52:45.266322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789323876454828:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.266416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.266461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789323876454828:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:45.267199Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789323876454737:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6814) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.267264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789323876454828:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.267340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789323876454737:2048] [node 1] ICP30 unregister session Session# [3:7577789323876454828:2048] VirtualId# [3:7577789323876454819:0]
2025-11-28T14:52:45.268060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.268082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789323876454737:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6814) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.268093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6814) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.268311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:45.268535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789323689580565:2048] [node 1] ICS07 socket disconnect 119 reason# EndOfStream
2025-11-28T14:52:45.268559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789323689580565:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.268633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.268685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789323689580565:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:45.269220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789323689580481:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6814) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.269174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:45.269257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789323689580565:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.269323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789323689580481:2048] [node 1] ICP30 unregister session Session# [4:7577789323689580565:2048] VirtualId# [4:7577789323689580561:0]
2025-11-28T14:52:45.269423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.269452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789323689580481:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6814) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.269472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6814) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.269469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789324418672375:2048] [node 1] ICS01 socket: 150 reason# EndOfStream
2025-11-28T14:52:45.269531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789324418672275:2048] [node 1] ICP30 unregister session Session# [5:7577789324418672375:2048] VirtualId# [5:7577789324418672368:0]
2025-11-28T14:52:45.269570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789324418672375:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.269650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.269721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.269945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:45.270126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789320832923725:2048] [node 1] ICS01 socket: 159 reason# EndOfStream
2025-11-28T14:52:45.270176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789320832923631:2048] [node 1] ICP30 unregister session Session# [6:7577789320832923725:2048] VirtualId# [6:7577789320832923716:0]
2025-11-28T14:52:45.270211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789320832923725:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.270276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.270320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.278868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789323876454737:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:45.278868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789323689580481:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:45.317154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:45.317326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789323876454840:2049] [node 2] ICS01 socket: 145 reason# EndOfStream
2025-11-28T14:52:45.317363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789323876454738:2049] [node 2] ICP30 unregister session Session# [3:7577789323876454840:2049] VirtualId# [3:7577789323876454830:0]
2025-11-28T14:52:45.317388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789323876454840:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.317488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:45.317550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:45.319371Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:45.319510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789323689580582:2049] [node 2] ICS01 socket: 108 reason# EndOfStream
2025-11-28T14:52:45.319543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789323689580482:2049] [node 2] ICP30 unregister session Session# [4:7577789323689580582:2049] VirtualId# [4:7577789323689580580:0]
2025-11-28T14:52:45.319562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789323689580582:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.319617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:45.319657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:45.320222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:45.320458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789324418672360:2049] [node 2] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-28T14:52:45.320498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789324418672360:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.320597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:45.320656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789324418672360:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:45.320902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:45.321136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789320832923732:2049] [node 2] ICS01 socket: 169 reason# EndOfStream
2025-11-28T14:52:45.321186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789320832923632:2049] [node 2] ICP30 unregister session Session# [6:7577789320832923732:2049] VirtualId# [6:7577789320832923728:0]
2025-11-28T14:52:45.321222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789320832923732:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.321304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:45.321349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:45.321415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789324418672276:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32373) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.321457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789324418672360:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.321483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789324418672276:2049] [node 2] ICP30 unregister session Session# [5:7577789324418672360:2049] VirtualId# [5:7577789324418672356:0]
2025-11-28T14:52:45.321586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:45.321613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789324418672276:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32373) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.321621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32373) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.331147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789324418672276:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:45.356356Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:45.356552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789323689580567:2050] [node 3] ICS07 socket disconnect 117 reason# EndOfStream
2025-11-28T14:52:45.356603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789323689580567:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.356696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:45.356734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789323689580567:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:45.357119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:45.357325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789324418672370:2050] [node 3] ICS01 socket: 136 reason# EndOfStream
2025-11-28T14:52:45.357367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789323689580483:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.357389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789324418672277:2050] [node 3] ICP30 unregister session Session# [5:7577789324418672370:2050] VirtualId# [5:7577789324418672364:0]
2025-11-28T14:52:45.357408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789323689580567:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.357431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789324418672370:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.357462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789323689580483:2050] [node 3] ICP30 unregister session Session# [4:7577789323689580567:2050] VirtualId# [4:7577789323689580562:0]
2025-11-28T14:52:45.357533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:45.357574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:45.357606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789323689580483:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.357603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:45.357616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.357776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:45.357992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789320832923723:2050] [node 3] ICS07 socket disconnect 161 reason# EndOfStream
2025-11-28T14:52:45.358037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789320832923723:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.358119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:45.358162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789320832923723:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:45.358800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789320832923633:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.358901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789320832923723:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.358953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789320832923633:2050] [node 3] ICP30 unregister session Session# [6:7577789320832923723:2050] VirtualId# [6:7577789320832923714:0]
2025-11-28T14:52:45.359041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:45.359078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789320832923633:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.359097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.367986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789323689580483:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:45.369085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789320832923633:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:45.388426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:45.388695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789324418672371:2108] [node 4] ICS07 socket disconnect 157 reason# EndOfStream
2025-11-28T14:52:45.388760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789324418672371:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.388868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:45.388914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789324418672371:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:45.389246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:45.389395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789320832923718:2109] [node 4] ICS07 socket disconnect 155 reason# EndOfStream
2025-11-28T14:52:45.389417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789320832923718:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.389503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:45.389540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789320832923718:2109] [node 4] ICS15 start handshake
2025-11-28T14:52:45.389615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789324418672338:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32058) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.389665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789324418672371:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.389754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789324418672338:2108] [node 4] ICP30 unregister session Session# [5:7577789324418672371:2108] VirtualId# [5:7577789324418672365:0]
2025-11-28T14:52:45.389850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:45.389878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789324418672338:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.389893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.390233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789320832923696:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32058) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.390356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789320832923718:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.390387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789320832923696:2109] [node 4] ICP30 unregister session Session# [6:7577789320832923718:2109] VirtualId# [6:7577789320832923713:0]
2025-11-28T14:52:45.390489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:45.390532Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789320832923696:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.390546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.399767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789324418672338:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:45.400733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789320832923696:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:52:45.440390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:45.440621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789320832923734:2107] [node 5] ICS07 socket disconnect 103 reason# EndOfStream
2025-11-28T14:52:45.440657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789320832923734:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.440750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:45.440797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789320832923734:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:45.441564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789320832923695:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27905) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.441610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789320832923734:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.441689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789320832923695:2107] [node 5] ICP30 unregister session Session# [6:7577789320832923734:2107] VirtualId# [6:7577789320832923729:0]
2025-11-28T14:52:45.441776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:45.441807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789320832923695:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27905) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.441818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27905) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:45.520668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789322822324461:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:45.520714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789322822324462:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:45.537836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789321839382609:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.538023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789321839382610:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:45.555391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789322319168104:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:45.555395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789322319168103:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.572593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789322361770279:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.572641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789322361770281:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:45.572731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789322361770280:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:45.589780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789323055202156:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.589819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789323055202157:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:45.590062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789323055202158:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:45.607251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789321284556981:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:45.607282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789321284556982:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:45.607318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789321284556983:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:45.688683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789321839382672:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:45.688318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789322822324525:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:45.688329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789322822324526:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:45.688233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789321284557042:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:45.688840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789323055202219:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:45.689050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789321839382673:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:45.688965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789322319168166:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:45.688527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789322822324524:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:45.689191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789322361770342:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:45.689715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789322319168167:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:45.689553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789321284557045:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:45.689710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789322361770343:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:45.689795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789323055202220:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:45.689803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789321839382674:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:45.689736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789322319168168:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.700176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789322319168175:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-28T14:52:45.700986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789322822324533:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33916
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.702750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324533:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.704068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789322319168175:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.704183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789321839382681:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19750
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.704341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789322822324534:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33916
2025-11-28T14:52:45.704382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789322822324535:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33916
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.704923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789322319168180:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-28T14:52:45.704931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789322319168179:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-28T14:52:45.706791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557059:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.708921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789321839382682:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19750
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.708985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789321839382684:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19750
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.709040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789321839382681:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.709638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789323055202233:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:45.709295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789322319168179:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.709655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789321839382682:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.709530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789322822324543:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33916
2025-11-28T14:52:45.709712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789321839382692:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19750
2025-11-28T14:52:45.709721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789322319168186:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-28T14:52:45.709719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789322822324545:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33916
2025-11-28T14:52:45.710077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789322319168185:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-28T14:52:45.709969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789321839382691:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19750
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.710157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789321839382684:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.710494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789322319168180:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.710561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789323055202235:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-28T14:52:45.710624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324534:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.710871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324535:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.712746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789322319168185:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.712781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789321284557059:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.712879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324543:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.712888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789323055202233:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.712962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789322319168103:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.713005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789321284556981:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.713097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789323055202158:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.713142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324530:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.713173Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789322822324530:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.713198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324532:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.713291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789322319168173:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.713326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789322319168173:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.713221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789322822324532:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.714765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789321839382692:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.715241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789323055202235:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.714825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789321839382691:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.715363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789322319168173:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:45.715316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324545:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.715480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789322319168186:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.715397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789323055202157:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.715585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789321839382609:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.715746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789322319168104:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.715805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324531:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.715807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789321839382678:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.715831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789321839382678:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.715808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789322822324530:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:45.716448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789321839382679:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.716469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789321839382679:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.715834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789322822324531:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.716423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789322319168166:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.716692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789323055202237:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-28T14:52:45.716489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:45.717654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789321839382678:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.715857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789322822324532:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.716705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789322319168166:2108] [node 5] ICP22 created new session: [3:7577789322319168194:2108]
2025-11-28T14:52:45.719445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789322822324462:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.719849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557061:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
2025-11-28T14:52:45.718578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789322319168194:2108] [node 5] ICS09 handshake done sender: [3:7577789322319168173:2111] self: [3:7577789322319168169:0] peer: [5:7577789323055202236:0] socket: 130 qp: -1
2025-11-28T14:52:45.718752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789322319168194:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:45.719509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.718973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:45.719592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789322822324526:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.719958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789322319168193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-28T14:52:45.720129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789321839382679:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:45.719849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789322822324462:2049] [node 3] ICP22 created new session: [1:7577789322822324553:2049]
2025-11-28T14:52:45.720139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789323055202158:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8674) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789323055202233:2114] held: no
2025-11-28T14:52:45.719903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789322822324553:2049] [node 3] ICS09 handshake done sender: [1:7577789322822324530:2110] self: [1:7577789322822324527:0] peer: [3:7577789322319168190:0] socket: 132 qp: -1
2025-11-28T14:52:45.722203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789323055202233:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:45.720155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789321839382672:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.722287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789321284557059:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:45.720201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:45.719937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.719950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789322822324553:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:45.723764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789321284556981:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789321284557059:2115] held: no
2025-11-28T14:52:45.721546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789321839382672:2108] [node 5] ICP22 created new session: [2:7577789321839382701:2108]
2025-11-28T14:52:45.725847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789322319168185:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:45.724459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789323055202239:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-28T14:52:45.722318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789321839382701:2108] [node 5] ICS09 handshake done sender: [2:7577789321839382678:2110] self: [2:7577789321839382675:0] peer: [5:7577789323055202238:0] socket: 141 qp: -1
2025-11-28T14:52:45.719975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789322822324526:2109] [node 6] ICP22 created new session: [1:7577789322822324554:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.727093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789323055202158:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.722442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789321839382701:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:45.727159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.722476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:45.727467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789323055202158:2050] [node 3] ICP22 created new session: [5:7577789323055202240:2050]
2025-11-28T14:52:45.719994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:45.724016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789321839382610:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.720111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789322822324554:2109] [node 6] ICS09 handshake done sender: [1:7577789322822324532:2112] self: [1:7577789322822324529:0] peer: [6:7577789321284557060:0] socket: 134 qp: -1
2025-11-28T14:52:45.727437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789322319168103:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789322319168185:2118] held: no
2025-11-28T14:52:45.727583Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789323055202157:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789323055202235:2116] held: no
2025-11-28T14:52:45.724071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.720378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789322822324554:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:45.727680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789322361770357:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43631
2025-11-28T14:52:45.727712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789322319168196:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-28T14:52:45.727736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789323055202240:2050] [node 3] ICS09 handshake done sender: [5:7577789323055202233:2114] self: [5:7577789323055202236:0] peer: [3:7577789322319168169:0] socket: 136 qp: -1
2025-11-28T14:52:45.720724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.724400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789321839382610:2049] [node 3] ICP22 created new session: [2:7577789321839382703:2049]
2025-11-28T14:52:45.727788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789323055202240:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:45.725532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789322822324531:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.724451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789321839382703:2049] [node 3] ICS09 handshake done sender: [2:7577789321839382679:2111] self: [2:7577789321839382676:0] peer: [3:7577789322319168192:0] socket: 142 qp: -1
2025-11-28T14:52:45.727836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:45.727829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789321284556981:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.725512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789321839382703:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:45.727866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.725820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:45.727902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789321284556981:2048] [node 1] ICP22 created new session: [6:7577789321284557063:2048]
2025-11-28T14:52:45.727888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789322822324461:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.727932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.727958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789321284557063:2048] [node 1] ICS09 handshake done sender: [6:7577789321284557059:2115] self: [6:7577789321284557060:0] peer: [1:7577789322822324529:0] socket: 135 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.728047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789322822324461:2048] [node 2] ICP22 created new session: [1:7577789322822324559:2048]
2025-11-28T14:52:45.728050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789321284557063:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:45.728203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.728206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789322822324559:2048] [node 2] ICS09 handshake done sender: [1:7577789322822324531:2111] self: [1:7577789322822324528:0] peer: [2:7577789321839382697:0] socket: 133 qp: -1
2025-11-28T14:52:45.728226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789322822324559:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:45.728290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.728345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789321839382700:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19750
2025-11-28T14:52:45.728592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789322361770358:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43631
2025-11-28T14:52:45.728648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789322319168103:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.728721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.728861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789323055202235:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.729560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789322319168103:2048] [node 1] ICP22 created new session: [3:7577789322319168198:2048]
2025-11-28T14:52:45.729722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789322319168198:2048] [node 1] ICS09 handshake done sender: [3:7577789322319168185:2118] self: [3:7577789322319168190:0] peer: [1:7577789322822324527:0] socket: 138 qp: -1
2025-11-28T14:52:45.729746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789322319168198:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.729765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.730408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789323055202157:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.730538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.730652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789323055202244:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-28T14:52:45.730686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789321839382692:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:45.730821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789322361770359:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43631
2025-11-28T14:52:45.730834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789322319168186:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.730671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789323055202157:2049] [node 2] ICP22 created new session: [5:7577789323055202246:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.730868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789323055202246:2049] [node 2] ICS09 handshake done sender: [5:7577789323055202235:2116] self: [5:7577789323055202238:0] peer: [2:7577789321839382675:0] socket: 148 qp: -1
2025-11-28T14:52:45.731378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789322361770360:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43631
2025-11-28T14:52:45.730901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789323055202246:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:45.731756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789321839382609:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.731766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557065:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.732038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789322319168104:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789322319168186:2119] held: no
2025-11-28T14:52:45.730937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:45.731800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.732323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557066:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
2025-11-28T14:52:45.731251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789323055202243:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-28T14:52:45.732000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789321839382609:2048] [node 1] ICP22 created new session: [2:7577789321839382707:2048]
2025-11-28T14:52:45.732278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789321839382707:2048] [node 1] ICS09 handshake done sender: [2:7577789321839382692:2119] self: [2:7577789321839382697:0] peer: [1:7577789322822324528:0] socket: 140 qp: -1
2025-11-28T14:52:45.732298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789321839382707:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:45.733058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789322361770357:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.732381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.734118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789322361770281:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.734212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789322319168104:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.734272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789322361770358:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.734238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557067:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
2025-11-28T14:52:45.734268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.734322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789322319168104:2049] [node 2] ICP22 created new session: [3:7577789322319168201:2049]
2025-11-28T14:52:45.734326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789321839382609:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:45.734319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789321284557065:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.734374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789322319168201:2049] [node 2] ICS09 handshake done sender: [3:7577789322319168186:2119] self: [3:7577789322319168192:0] peer: [2:7577789321839382676:0] socket: 145 qp: -1
2025-11-28T14:52:45.734378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789322361770360:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.734404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789322319168201:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:45.734401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789322361770279:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.734429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789321284556982:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.734461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789322361770280:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.734451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:45.734522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789323055202244:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.734551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324542:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.734584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789322822324542:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.734601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789322319168182:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.734601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789321839382685:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.734641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789321839382685:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.735714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789322361770359:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.734631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789322319168182:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.736448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789321839382687:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.736289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789323055202243:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.736469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789321839382687:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.736994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789323055202156:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.737119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789322319168182:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:45.737354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789321284557067:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.737489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789321284556983:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.737435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789322822324538:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.737841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789321839382687:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:45.737481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789322319168167:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.737741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789321284557066:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.737483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789322822324538:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.738054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789321839382685:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:45.737533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.737583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789322319168167:2107] [node 4] ICP22 created new session: [3:7577789322319168204:2107]
2025-11-28T14:52:45.737641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789322822324542:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:45.738036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789322319168204:2107] [node 4] ICS09 handshake done sender: [3:7577789322319168182:2116] self: [3:7577789322319168181:0] peer: [4:7577789322361770361:0] socket: 137 qp: -1
2025-11-28T14:52:45.738210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789321284557042:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.738656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789322361770365:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43631
2025-11-28T14:52:45.738062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789322319168204:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:45.738455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789322822324524:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.738588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789323055202234:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.738615Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789323055202234:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.739312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789321839382674:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.738200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:45.739479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:45.739593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:45.739912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789321839382673:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.739959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789322822324524:2107] [node 4] ICP22 created new session: [1:7577789322822324563:2107]
2025-11-28T14:52:45.739688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789323055202234:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:45.740017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789322822324538:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:45.740023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789321839382674:2107] [node 4] ICP22 created new session: [2:7577789321839382711:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.740054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.740058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789322822324563:2107] [node 4] ICS09 handshake done sender: [1:7577789322822324542:2117] self: [1:7577789322822324541:0] peer: [4:7577789322361770362:0] socket: 122 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.740264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789323055202220:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.740075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789321839382711:2107] [node 4] ICS09 handshake done sender: [2:7577789321839382687:2117] self: [2:7577789321839382686:0] peer: [4:7577789322361770363:0] socket: 164 qp: -1
2025-11-28T14:52:45.740247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789322319168184:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.740157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789322822324563:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.742025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557070:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
2025-11-28T14:52:45.740172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789321839382673:2109] [node 6] ICP22 created new session: [2:7577789321839382712:2109]
2025-11-28T14:52:45.740182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:45.740691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.740196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789321839382711:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:45.742619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789322822324525:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.740285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789322319168184:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.742269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789323055202250:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-28T14:52:45.740216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789321839382712:2109] [node 6] ICS09 handshake done sender: [2:7577789321839382685:2116] self: [2:7577789321839382683:0] peer: [6:7577789321284557068:0] socket: 163 qp: -1
2025-11-28T14:52:45.742652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789321284557065:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.742871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.743153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789322822324525:2108] [node 5] ICP22 created new session: [1:7577789322822324565:2108]
2025-11-28T14:52:45.742307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789323055202220:2109] [node 6] ICP22 created new session: [5:7577789323055202252:2109]
2025-11-28T14:52:45.743058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557072:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
2025-11-28T14:52:45.743138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789322319168184:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:45.740234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:45.743710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789322361770281:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8674) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789322361770357:2115] held: no
2025-11-28T14:52:45.744314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789322361770367:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43631
2025-11-28T14:52:45.744378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789322822324565:2108] [node 5] ICS09 handshake done sender: [1:7577789322822324538:2116] self: [1:7577789322822324536:0] peer: [5:7577789323055202249:0] socket: 109 qp: -1
2025-11-28T14:52:45.744390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789322361770366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43631
2025-11-28T14:52:45.744633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789322319168168:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.740252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789321839382712:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:45.744596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789321284556982:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.745607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.744426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789322822324565:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:45.742626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789323055202252:2109] [node 6] ICS09 handshake done sender: [5:7577789323055202234:2115] self: [5:7577789323055202232:0] peer: [6:7577789321284557071:0] socket: 166 qp: -1
2025-11-28T14:52:45.740286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.744674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:45.744734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.743108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789323055202252:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:45.745815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789322319168168:2109] [node 6] ICP22 created new session: [3:7577789322319168207:2109]
2025-11-28T14:52:45.745717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789321284556982:2049] [node 2] ICP22 created new session: [6:7577789321284557074:2049]
2025-11-28T14:52:45.743150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.745904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789321284557074:2049] [node 2] ICS09 handshake done sender: [6:7577789321284557065:2117] self: [6:7577789321284557068:0] peer: [2:7577789321839382683:0] socket: 128 qp: -1
2025-11-28T14:52:45.744423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789323055202243:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:45.746247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789322319168207:2109] [node 6] ICS09 handshake done sender: [3:7577789322319168184:2117] self: [3:7577789322319168183:0] peer: [6:7577789321284557069:0] socket: 143 qp: -1
2025-11-28T14:52:45.745938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789321284557074:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:45.745678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789323055202156:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.746568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789322361770279:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789322361770358:2116] held: no
2025-11-28T14:52:45.746612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789322319168207:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:45.746260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:45.746648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789322361770357:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:45.746803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.745718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.746539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789321284557042:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789321284557066:2118] held: no
2025-11-28T14:52:45.747916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789322361770281:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.746268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789323055202156:2048] [node 1] ICP22 created new session: [5:7577789323055202254:2048]
2025-11-28T14:52:45.746627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789323055202254:2048] [node 1] ICS09 handshake done sender: [5:7577789323055202243:2119] self: [5:7577789323055202249:0] peer: [1:7577789322822324536:0] socket: 131 qp: -1
2025-11-28T14:52:45.749494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.747062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789323055202254:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:45.747091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.749932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789322361770281:2050] [node 3] ICP22 created new session: [4:7577789322361770368:2050]
2025-11-28T14:52:45.750649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789322361770368:2050] [node 3] ICS09 handshake done sender: [4:7577789322361770357:2115] self: [4:7577789322361770361:0] peer: [3:7577789322319168181:0] socket: 113 qp: -1
2025-11-28T14:52:45.750708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789322361770368:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:45.750758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:45.750920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789321284557066:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:45.751179Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789322361770280:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789322361770360:2118] held: no
2025-11-28T14:52:45.751941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789321284556982:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:45.752042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789321284557042:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.752088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:45.752093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789322361770358:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:45.752132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789321284557042:2106] [node 5] ICP22 created new session: [6:7577789321284557076:2106]
2025-11-28T14:52:45.752134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789322361770360:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:45.752198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789321284557076:2106] [node 5] ICS09 handshake done sender: [6:7577789321284557066:2118] self: [6:7577789321284557071:0] peer: [5:7577789323055202232:0] socket: 144 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:45.752232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789321284557076:2106] [node 5] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:45.753022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:45.753263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557073:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
2025-11-28T14:52:45.753535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789322361770279:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.753584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:45.753637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789322361770279:2048] [node 1] ICP22 created new session: [4:7577789322361770372:2048]
2025-11-28T14:52:45.753648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789322361770280:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.753718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:45.753719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789322361770372:2048] [node 1] ICS09 handshake done sender: [4:7577789322361770358:2116] self: [4:7577789322361770362:0] peer: [1:7577789322822324541:0] socket: 126 qp: -1
2025-11-28T14:52:45.753796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789321284556983:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8674) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789321284557067:2119] held: no
2025-11-28T14:52:45.753757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789322361770372:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:45.753773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789322361770280:2049] [node 2] ICP22 created new session: [4:7577789322361770373:2049]
2025-11-28T14:52:45.753787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:45.753815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789322361770373:2049] [node 2] ICS09 handshake done sender: [4:7577789322361770360:2118] self: [4:7577789322361770363:0] peer: [2:7577789321839382686:0] socket: 165 qp: -1
2025-11-28T14:52:45.753841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789322361770373:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:45.753865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.754330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789323055202256:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-28T14:52:45.754468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789321284557067:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:45.754583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557079:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
2025-11-28T14:52:45.754758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789321284556983:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.754833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:45.754874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789321284556983:2050] [node 3] ICP22 created new session: [6:7577789321284557080:2050]
2025-11-28T14:52:45.754915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789321284557080:2050] [node 3] ICS09 handshake done sender: [6:7577789321284557067:2119] self: [6:7577789321284557069:0] peer: [3:7577789322319168183:0] socket: 167 qp: -1
2025-11-28T14:52:45.754944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789321284557080:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:45.754975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:45.755067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789322361770376:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43631
2025-11-28T14:52:45.756681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789323055202256:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.756696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789322361770376:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.756832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789323055202219:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.756822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789321284557079:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.756944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789321284557045:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:45.757034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789322361770354:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.757066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789322361770354:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.757130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789322361770356:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:45.757150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789322361770356:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:45.757781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789322361770354:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:45.757945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789322361770343:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.757995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:45.758048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789322361770343:2108] [node 5] ICP22 created new session: [4:7577789322361770380:2108]
2025-11-28T14:52:45.758049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789323055202258:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-11-28T14:52:45.758125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789322361770380:2108] [node 5] ICS09 handshake done sender: [4:7577789322361770354:2113] self: [4:7577789322361770353:0] peer: [5:7577789323055202257:0] socket: 108 qp: -1
2025-11-28T14:52:45.758164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789322361770356:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:45.758166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789322361770380:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:45.758193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.0
2025-11-28T14:52:45.758270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789323055202256:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:45.758491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789321284557083:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17243
2025-11-28T14:52:45.758372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789322361770342:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:45.758566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:45.758596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789322361770342:2109] [node 6] ICP22 created new session: [4:7577789322361770382:2109]
2025-11-28T14:52:45.758789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789323055202219:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28586) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789323055202256:2122] held: no
2025-11-28T14:52:45.758803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789322361770382:2109] [node 6] ICS09 handshake done sender: [4:7577789322361770356:2114] self: [4:7577789322361770355:0] peer: [6:7577789321284557082:0] socket: 110 qp: -1
2025-11-28T14:52:45.758823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789322361770382:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:45.758839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:45.759832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789321284557045:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28586) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789321284557079:2123] held: no
2025-11-28T14:52:45.760095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789321284557079:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:45.760672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789323055202219:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.760705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:45.760735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789323055202219:2108] [node 4] ICP22 created new session: [5:7577789323055202259:2108]
2025-11-28T14:52:45.760771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789321284557045:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:45.760765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789323055202259:2108] [node 4] ICS09 handshake done sender: [5:7577789323055202256:2122] self: [5:7577789323055202257:0] peer: [4:7577789322361770353:0] socket: 112 qp: -1
2025-11-28T14:52:45.760812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:45.760802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789323055202259:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:45.760847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789321284557045:2107] [node 4] ICP22 created new session: [6:7577789321284557084:2107]
2025-11-28T14:52:45.760848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:45.760873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789321284557084:2107] [node 4] ICS09 handshake done sender: [6:7577789321284557079:2123] self: [6:7577789321284557082:0] peer: [4:7577789322361770355:0] socket: 114 qp: -1
2025-11-28T14:52:45.760901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789321284557084:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:45.760934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:45.809738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:45.809987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789321839382707:2048] [node 1] ICS07 socket disconnect 140 reason# EndOfStream
2025-11-28T14:52:45.810015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789321839382707:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.810105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.810148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789321839382707:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:45.811140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:45.811382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789322319168198:2048] [node 1] ICS01 socket: 138 reason# EndOfStream
2025-11-28T14:52:45.811435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789322319168103:2048] [node 1] ICP30 unregister session Session# [3:7577789322319168198:2048] VirtualId# [3:7577789322319168190:0]
2025-11-28T14:52:45.811466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789322319168198:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.811559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.811643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.812314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789321839382609:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31876) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:45.812383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789321839382707:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:45.813431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789321839382609:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31876) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.813492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789321839382707:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.814095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789321839382609:2048] [node 1] ICP30 unregister session Session# [2:7577789321839382707:2048] VirtualId# [2:7577789321839382697:0]
2025-11-28T14:52:45.814212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.814248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789321839382609:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.814263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.814325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:45.814463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789322361770372:2048] [node 1] ICS01 socket: 126 reason# EndOfStream
2025-11-28T14:52:45.814495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789322361770279:2048] [node 1] ICP30 unregister session Session# [4:7577789322361770372:2048] VirtualId# [4:7577789322361770362:0]
2025-11-28T14:52:45.814517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789322361770372:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.814577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.814627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.815340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:45.815453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789323055202254:2048] [node 1] ICS01 socket: 131 reason# EndOfStream
2025-11-28T14:52:45.815479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789323055202156:2048] [node 1] ICP30 unregister session Session# [5:7577789323055202254:2048] VirtualId# [5:7577789323055202249:0]
2025-11-28T14:52:45.815496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789323055202254:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.815535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.815566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.816404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:45.816615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789321284557063:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-11-28T14:52:45.816662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789321284556981:2048] [node 1] ICP30 unregister session Session# [6:7577789321284557063:2048] VirtualId# [6:7577789321284557060:0]
2025-11-28T14:52:45.816690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789321284557063:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.816769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:45.816828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:45.859530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:45.859680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789322319168201:2049] [node 2] ICS01 socket: 145 reason# EndOfStream
2025-11-28T14:52:45.859708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789322319168104:2049] [node 2] ICP30 unregister session Session# [3:7577789322319168201:2049] VirtualId# [3:7577789322319168192:0]
2025-11-28T14:52:45.859748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789322319168201:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.859836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:45.859881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:45.862640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:45.862800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789322361770373:2049] [node 2] ICS01 socket: 165 reason# EndOfStream
2025-11-28T14:52:45.862853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789322361770280:2049] [node 2] ICP30 unregister session Session# [4:7577789322361770373:2049] VirtualId# [4:7577789322361770363:0]
2025-11-28T14:52:45.862880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789322361770373:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.862949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:45.862987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:45.863246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:45.863362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789323055202246:2049] [node 2] ICS01 socket: 148 reason# EndOfStream
2025-11-28T14:52:45.863386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789323055202157:2049] [node 2] ICP30 unregister session Session# [5:7577789323055202246:2049] VirtualId# [5:7577789323055202238:0]
2025-11-28T14:52:45.863405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789323055202246:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.863442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:45.863470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:45.864105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:45.864300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789321284557074:2049] [node 2] ICS07 socket disconnect 128 reason# EndOfStream
2025-11-28T14:52:45.864355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789321284557074:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.864459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:45.864509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789321284557074:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:45.865159Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789321284556982:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9805) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:45.865198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789321284557074:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:45.865269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789321284556982:2049] [node 2] ICP30 unregister session Session# [6:7577789321284557074:2049] VirtualId# [6:7577789321284557068:0]
2025-11-28T14:52:45.865359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:45.865379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789321284556982:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9805) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.865389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9805) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:45.874856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789321284556982:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:45.921069Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:45.921232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789322361770368:2050] [node 3] ICS01 socket: 113 reason# EndOfStream
2025-11-28T14:52:45.921278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789322361770281:2050] [node 3] ICP30 unregister session Session# [4:7577789322361770368:2050] VirtualId# [4:7577789322361770361:0]
2025-11-28T14:52:45.921312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789322361770368:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.921402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:45.921444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:45.921902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:45.922017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789323055202240:2050] [node 3] ICS01 socket: 136 reason# EndOfStream
2025-11-28T14:52:45.922047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789323055202158:2050] [node 3] ICP30 unregister session Session# [5:7577789323055202240:2050] VirtualId# [5:7577789323055202236:0]
2025-11-28T14:52:45.922067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789323055202240:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.922107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:45.922138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:45.922672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:45.922809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789321284557080:2050] [node 3] ICS01 socket: 167 reason# EndOfStream
2025-11-28T14:52:45.922838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789321284556983:2050] [node 3] ICP30 unregister session Session# [6:7577789321284557080:2050] VirtualId# [6:7577789321284557069:0]
2025-11-28T14:52:45.922858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789321284557080:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.922928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:45.922976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:45.978191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:45.978341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789323055202259:2108] [node 4] ICS01 socket: 112 reason# EndOfStream
2025-11-28T14:52:45.978371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789323055202219:2108] [node 4] ICP30 unregister session Session# [5:7577789323055202259:2108] VirtualId# [5:7577789323055202257:0]
2025-11-28T14:52:45.978397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789323055202259:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.978473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:45.978519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:45.979350Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:45.979487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789321284557084:2107] [node 4] ICS01 socket: 114 reason# EndOfStream
2025-11-28T14:52:45.979515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789321284557045:2107] [node 4] ICP30 unregister session Session# [6:7577789321284557084:2107] VirtualId# [6:7577789321284557082:0]
2025-11-28T14:52:45.979540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789321284557084:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:45.979599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:45.979636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:46.036642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:46.036797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789321284557076:2106] [node 5] ICS01 socket: 144 reason# EndOfStream
2025-11-28T14:52:46.036827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789321284557042:2106] [node 5] ICP30 unregister session Session# [6:7577789321284557076:2106] VirtualId# [6:7577789321284557071:0]
2025-11-28T14:52:46.036854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789321284557076:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.036912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:46.036955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:46.138598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789328877402766:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:46.138982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789328877402767:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:46.155933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789325242855801:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.155961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789325242855802:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:46.175030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789327842821546:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.175061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789327842821547:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:46.194256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789326945350196:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.194266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789326945350197:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:46.194279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789326945350198:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:46.213059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789328064913941:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.213080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789328064913942:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:46.213131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789328064913943:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:46.232958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789325602743535:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.233117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789325602743537:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:46.233150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789325602743536:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:46.309277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789328877402829:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:46.314969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789325242855864:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.318548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789327842821610:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:46.318990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789326945350259:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.319065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789328064914004:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.319249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789328064914005:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:46.319131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789328877402830:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.319079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789325242855865:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:46.318767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789327842821609:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:46.319464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789328877402831:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:46.318974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789327842821611:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.319463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789325242855866:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:46.319380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789325602743597:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:46.319530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789326945350260:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:46.319455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789325602743598:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:46.324173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789326945350198:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(37708554ns) {Configure:104(37702008ns) {ProcessConfigured:135(111654ns) {StartInitialHandshake:157(109036ns) {DropHandshakes:470(1595ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(617ns) PrepareNewSessionHandshake:477(1208ns) {SwitchToState:179(371ns)}}}}}
2025-11-28T14:52:46.325805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789327842821547:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(38981705ns) {Configure:104(38976073ns) {ProcessConfigured:135(176664ns) {StartInitialHandshake:157(174574ns) {DropHandshakes:470(37256ns) {DropIncomingHandshake:442(271ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(486ns) PrepareNewSessionHandshake:477(670ns) {SwitchToState:179(250ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.330057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789328877402840:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33853
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.330191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789325242855875:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19055
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.330688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789327842821620:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47646
2025-11-28T14:52:46.331185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789328877402841:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33853
No RDMA context found for GID: 2025-11-28T14:52:46.330981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789326945350274:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53328
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.334041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743611:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
2025-11-28T14:52:46.335146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789327842821624:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47646
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.338616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789328877402845:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33853
2025-11-28T14:52:46.339251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743612:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:46.341290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789328064914017:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.342345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789328877402840:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.342345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789325242855879:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19055
2025-11-28T14:52:46.342411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789325242855875:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.342593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789327842821626:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47646
2025-11-28T14:52:46.342566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789328877402841:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.342680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789328877402846:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33853
2025-11-28T14:52:46.342707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789327842821625:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47646
2025-11-28T14:52:46.342582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789325242855881:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19055
2025-11-28T14:52:46.343207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789328877402845:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.342960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789326945350274:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789325602743611:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.342942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789327842821620:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789325602743612:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789327842821627:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47646
2025-11-28T14:52:46.343553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789325602743535:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.343517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789328877402846:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.342905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789325242855880:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19055
2025-11-28T14:52:46.343248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789326945350198:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.343633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789325602743536:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.343772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789328877402835:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789325242855881:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789325242855882:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19055
2025-11-28T14:52:46.343876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789327842821625:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789328877402835:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.343892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789327842821624:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789325242855879:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.345433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789328877402837:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.345433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789328064914017:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789327842821614:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789325242855882:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.345555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789328064913943:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.344016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789327842821614:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.345462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789328877402837:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.343946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789325242855880:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789325242855872:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.346881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789328877402835:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:46.344359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789327842821627:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.343984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789325242855872:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.344388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789325242855801:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.344393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789327842821547:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.346283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789325242855871:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.346880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789327842821626:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.347427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789327842821618:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.346303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789325242855871:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.347901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789328877402830:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:46.346868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789325242855872:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:46.347443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789327842821618:2112] [node 5] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-28T14:52:46.348210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789325242855871:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:46.348208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789328877402837:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:46.347471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789327842821546:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.348516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789325242855864:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.348233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743617:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
2025-11-28T14:52:46.348530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789328877402838:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.348623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:46.349071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743618:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
2025-11-28T14:52:46.349119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789325242855864:2109] [node 6] ICP22 created new session: [2:7577789325242855894:2109]
2025-11-28T14:52:46.348572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789328877402838:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.349098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:46.349119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789327842821618:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:46.349576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789328877402830:2109] [node 6] ICP22 created new session: [1:7577789328877402855:2109]
2025-11-28T14:52:46.349120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789327842821614:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:46.349771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789325602743611:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:46.349737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789328877402766:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.349738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789328877402855:2109] [node 6] ICS09 handshake done sender: [1:7577789328877402835:2110] self: [1:7577789328877402832:0] peer: [6:7577789325602743616:0] socket: 134 qp: -1
2025-11-28T14:52:46.349851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789325602743612:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:46.349723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789325242855894:2109] [node 6] ICS09 handshake done sender: [2:7577789325242855872:2112] self: [2:7577789325242855870:0] peer: [6:7577789325602743615:0] socket: 122 qp: -1
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:46.349762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789328877402855:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:46.350012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789327842821610:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.349786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:46.349944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789328877402766:2048] [node 2] ICP22 created new session: [1:7577789328877402858:2048]
2025-11-28T14:52:46.350204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789327842821637:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47646
2025-11-28T14:52:46.349751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789325242855802:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.350603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789325602743536:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.350149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.350759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:46.349759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789325242855894:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:46.350861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789326945350276:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53328
2025-11-28T14:52:46.350721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789328877402858:2048] [node 2] ICS09 handshake done sender: [1:7577789328877402837:2111] self: [1:7577789328877402834:0] peer: [2:7577789325242855888:0] socket: 133 qp: -1
2025-11-28T14:52:46.351251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789325602743535:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.350896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789327842821609:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.352018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789328064914021:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-28T14:52:46.350880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789328877402858:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:46.351885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:46.349796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789325242855892:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19055
2025-11-28T14:52:46.351022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:46.351006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:46.352273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789328064914022:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-28T14:52:46.351071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789327842821609:2108] [node 5] ICP22 created new session: [3:7577789327842821640:2108]
2025-11-28T14:52:46.353063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789326945350198:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7866) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789326945350274:2115] held: no
2025-11-28T14:52:46.349803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.351253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789328877402838:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:46.352250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789325602743536:2049] [node 2] ICP22 created new session: [6:7577789325602743619:2049]
2025-11-28T14:52:46.353928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789326945350274:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:46.351197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789327842821610:2107] [node 4] ICP22 created new session: [3:7577789327842821638:2107]
2025-11-28T14:52:46.353233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789328877402767:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.352390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789325602743619:2049] [node 2] ICS09 handshake done sender: [6:7577789325602743611:2114] self: [6:7577789325602743615:0] peer: [2:7577789325242855870:0] socket: 132 qp: -1
2025-11-28T14:52:46.353282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:46.349845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789325242855802:2049] [node 3] ICP22 created new session: [2:7577789325242855895:2049]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:46.351254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789327842821640:2108] [node 5] ICS09 handshake done sender: [3:7577789327842821618:2112] self: [3:7577789327842821616:0] peer: [5:7577789328064914020:0] socket: 129 qp: -1
2025-11-28T14:52:46.353049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789325602743619:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:46.353902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789328877402767:2049] [node 3] ICP22 created new session: [1:7577789328877402862:2049]
2025-11-28T14:52:46.350162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:46.353245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:46.354236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789328877402862:2049] [node 3] ICS09 handshake done sender: [1:7577789328877402838:2112] self: [1:7577789328877402833:0] peer: [3:7577789327842821634:0] socket: 137 qp: -1
2025-11-28T14:52:46.357367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789328064914023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-28T14:52:46.351951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789327842821640:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:46.350478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789325242855895:2049] [node 3] ICS09 handshake done sender: [2:7577789325242855871:2111] self: [2:7577789325242855869:0] peer: [3:7577789327842821632:0] socket: 123 qp: -1
2025-11-28T14:52:46.358794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789326945350277:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53328
2025-11-28T14:52:46.356953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789328877402862:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:46.350894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789325242855895:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:46.357686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789328064913943:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7866) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789328064914017:2114] held: no
2025-11-28T14:52:46.353887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:46.352358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789327842821638:2107] [node 4] ICS09 handshake done sender: [3:7577789327842821614:2110] self: [3:7577789327842821613:0] peer: [4:7577789326945350275:0] socket: 126 qp: -1
2025-11-28T14:52:46.357612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:46.354196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789325602743535:2048] [node 1] ICP22 created new session: [6:7577789325602743622:2048]
2025-11-28T14:52:46.351237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:46.352402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789327842821638:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:46.359733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789326945350198:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.357048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789325242855880:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:46.358745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789328064914017:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:46.352405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:46.356940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789325602743622:2048] [node 1] ICS09 handshake done sender: [6:7577789325602743612:2115] self: [6:7577789325602743616:0] peer: [1:7577789328877402832:0] socket: 144 qp: -1
2025-11-28T14:52:46.357401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789325602743622:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.360853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789328064914021:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.353265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:46.360605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:46.361124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789328064914023:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.357427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:46.359659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789325242855801:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789325242855880:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.354556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789327842821624:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:46.361296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789328064914024:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-28T14:52:46.360802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789326945350198:2050] [node 3] ICP22 created new session: [4:7577789326945350279:2050]
2025-11-28T14:52:46.362184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743625:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:46.361687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789328064913943:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.357565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789327842821547:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.362001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789328064913942:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.357675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:46.361084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789326945350279:2050] [node 3] ICS09 handshake done sender: [4:7577789326945350274:2115] self: [4:7577789326945350275:0] peer: [3:7577789327842821613:0] socket: 127 qp: -1
2025-11-28T14:52:46.358425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789327842821547:2049] [node 2] ICP22 created new session: [3:7577789327842821645:2049]
2025-11-28T14:52:46.362124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789328064913941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.362342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:46.358678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789327842821645:2049] [node 2] ICS09 handshake done sender: [3:7577789327842821624:2115] self: [3:7577789327842821632:0] peer: [2:7577789325242855869:0] socket: 124 qp: -1
2025-11-28T14:52:46.361289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789326945350279:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:46.358709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789327842821645:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.362385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789328064913943:2050] [node 3] ICP22 created new session: [5:7577789328064914028:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.361299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789326945350277:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.359636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:46.363154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789328877402839:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.361332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:46.362752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789328064914028:2050] [node 3] ICS09 handshake done sender: [5:7577789328064914017:2114] self: [5:7577789328064914020:0] peer: [3:7577789327842821616:0] socket: 140 qp: -1
2025-11-28T14:52:46.362780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789327842821639:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47646
2025-11-28T14:52:46.363204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789325242855801:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.363186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789328877402839:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.362791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789328064914028:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:46.362329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789326945350196:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.366079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789327842821626:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:46.363715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789328877402848:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.363531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789325242855874:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.363178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789326945350280:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53328
2025-11-28T14:52:46.363743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789328877402848:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.363216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:46.363552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789325242855874:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.363733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:46.365136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789325602743625:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.363474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789326945350281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53328
2025-11-28T14:52:46.363470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789328064914024:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.365069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789325242855801:2048] [node 1] ICP22 created new session: [2:7577789325242855899:2048]
2025-11-28T14:52:46.366187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789328877402864:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33853
2025-11-28T14:52:46.366157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789325242855899:2048] [node 1] ICS09 handshake done sender: [2:7577789325242855880:2116] self: [2:7577789325242855888:0] peer: [1:7577789328877402834:0] socket: 145 qp: -1
2025-11-28T14:52:46.366078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789326945350280:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.366323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789325602743536:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:46.366299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789325242855899:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:46.366415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:46.366475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789325602743598:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.366462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789326945350281:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.366469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789327842821547:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:46.366508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743626:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
2025-11-28T14:52:46.366509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789325602743535:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.366721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789327842821546:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.366587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789326945350197:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.366637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789325242855874:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:46.366738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789326945350268:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.366784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:46.366807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743627:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
2025-11-28T14:52:46.366752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789326945350268:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.366797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789328877402848:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:46.366819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789327842821546:2048] [node 1] ICP22 created new session: [3:7577789327842821647:2048]
2025-11-28T14:52:46.367127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789325602743626:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.367309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789325602743597:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.367151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789327842821647:2048] [node 1] ICS09 handshake done sender: [3:7577789327842821626:2117] self: [3:7577789327842821634:0] peer: [1:7577789328877402833:0] socket: 146 qp: -1
2025-11-28T14:52:46.367629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789328877402831:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.367738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789328877402839:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:46.367173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789327842821647:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:46.368510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:46.368451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.368920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789326945350268:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:46.368575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789328877402831:2107] [node 4] ICP22 created new session: [1:7577789328877402867:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.368554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789328064914013:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.369024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789328877402867:2107] [node 4] ICS09 handshake done sender: [1:7577789328877402848:2118] self: [1:7577789328877402847:0] peer: [4:7577789326945350283:0] socket: 120 qp: -1
2025-11-28T14:52:46.368580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789328064914013:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.369191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789328877402867:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:46.370187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789327842821610:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16174348ns) {HandleHandshakeStatus:329(16172346ns) {DropIncomingHandshake:442(13940ns) DropOutgoingHandshake:457(361ns) SwitchToState:179(181ns) ProcessPendingSessionEvents:495(10939856ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10857368ns max# 2061612ns}}}
2025-11-28T14:52:46.370450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789328877402829:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.369163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789328064914031:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-28T14:52:46.370653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789325242855884:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.368954Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789326945350196:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789326945350277:2117] held: no
2025-11-28T14:52:46.370683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789325242855884:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.370718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:46.370105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789328064914032:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-28T14:52:46.371797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789325242855865:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.370693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789326945350259:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.371927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789328064914023:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:46.371934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:46.372268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:46.372081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789328877402829:2108] [node 5] ICP22 created new session: [1:7577789328877402869:2108]
2025-11-28T14:52:46.370695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789326945350286:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53328
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.372655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789325242855865:2108] [node 5] ICP22 created new session: [2:7577789325242855902:2108]
2025-11-28T14:52:46.371999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789328064914021:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:46.371979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:46.373105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789325242855902:2108] [node 5] ICS09 handshake done sender: [2:7577789325242855874:2113] self: [2:7577789325242855873:0] peer: [5:7577789328064914026:0] socket: 166 qp: -1
2025-11-28T14:52:46.374059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743630:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
2025-11-28T14:52:46.372797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789328064914013:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:46.372594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789328877402869:2108] [node 5] ICS09 handshake done sender: [1:7577789328877402839:2113] self: [1:7577789328877402836:0] peer: [5:7577789328064914027:0] socket: 150 qp: -1
2025-11-28T14:52:46.372793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789328877402869:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:46.372013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789326945350259:2109] [node 6] ICP22 created new session: [4:7577789326945350288:2109]
2025-11-28T14:52:46.373178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789328064913941:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.373264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:46.373267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789325242855902:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:46.372804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789326945350288:2109] [node 6] ICS09 handshake done sender: [4:7577789326945350268:2113] self: [4:7577789326945350267:0] peer: [6:7577789325602743628:0] socket: 116 qp: -1
2025-11-28T14:52:46.374833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789328877402767:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:46.373954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789328064913942:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.374154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:46.374004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:46.373108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789326945350288:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:46.374106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: 2025-11-28T14:52:46.375975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789325602743598:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20688) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789325602743625:2119] held: no
::ffff:127.1.0.0
2025-11-28T14:52:46.373306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:46.374151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789328064913941:2048] [node 1] ICP22 created new session: [5:7577789328064914034:2048]
2025-11-28T14:52:46.374489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789326945350277:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:46.376707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789325602743625:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:46.374274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789328064913942:2049] [node 2] ICP22 created new session: [5:7577789328064914035:2049]
2025-11-28T14:52:46.375012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789326945350196:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.374455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789328064914034:2048] [node 1] ICS09 handshake done sender: [5:7577789328064914023:2118] self: [5:7577789328064914027:0] peer: [1:7577789328877402836:0] socket: 170 qp: -1
2025-11-28T14:52:46.375045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:46.376771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743631:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
2025-11-28T14:52:46.374486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789328064914034:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:46.375832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789326945350196:2048] [node 1] ICP22 created new session: [4:7577789326945350291:2048]
2025-11-28T14:52:46.374730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789328064914035:2049] [node 2] ICS09 handshake done sender: [5:7577789328064914021:2116] self: [5:7577789328064914026:0] peer: [2:7577789325242855873:0] socket: 148 qp: -1
2025-11-28T14:52:46.376263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789326945350291:2048] [node 1] ICS09 handshake done sender: [4:7577789326945350277:2117] self: [4:7577789326945350283:0] peer: [1:7577789328877402847:0] socket: 130 qp: -1
2025-11-28T14:52:46.377062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789325602743627:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.374877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:46.376430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789326945350291:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:46.374878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789328064914035:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:46.376769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:46.375839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:46.378462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789325602743598:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.376797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789328064914004:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.378612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:46.377251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:46.378638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789325602743598:2107] [node 4] ICP22 created new session: [6:7577789325602743632:2107]
2025-11-28T14:52:46.377297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789328064914004:2109] [node 6] ICP22 created new session: [5:7577789328064914038:2109]
2025-11-28T14:52:46.378251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789328064914038:2109] [node 6] ICS09 handshake done sender: [5:7577789328064914013:2113] self: [5:7577789328064914009:0] peer: [6:7577789325602743629:0] socket: 172 qp: -1
2025-11-28T14:52:46.378741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789325602743626:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:46.378769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789325602743537:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.378288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789328064914038:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:46.378781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789325602743632:2107] [node 4] ICS09 handshake done sender: [6:7577789325602743625:2119] self: [6:7577789325602743628:0] peer: [4:7577789326945350267:0] socket: 138 qp: -1
2025-11-28T14:52:46.378438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:46.378900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789325602743632:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:46.378929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:46.378957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789325602743597:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.378945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789327842821629:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.379110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:46.378970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789327842821629:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.379303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789325602743597:2106] [node 5] ICP22 created new session: [6:7577789325602743635:2106]
2025-11-28T14:52:46.379488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789328064913941:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:46.379449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789325602743635:2106] [node 5] ICS09 handshake done sender: [6:7577789325602743626:2120] self: [6:7577789325602743629:0] peer: [5:7577789328064914009:0] socket: 176 qp: -1
2025-11-28T14:52:46.380920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789325242855884:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:46.379601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789325602743635:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:46.380605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:46.381485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789325242855866:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.381530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.381644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789325242855866:2107] [node 4] ICP22 created new session: [2:7577789325242855905:2107]
2025-11-28T14:52:46.381892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789327842821629:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:46.381788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789325242855905:2107] [node 4] ICS09 handshake done sender: [2:7577789325242855884:2119] self: [2:7577789325242855883:0] peer: [4:7577789326945350285:0] socket: 131 qp: -1
2025-11-28T14:52:46.381862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789325242855905:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:46.382112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789326945350293:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53328
2025-11-28T14:52:46.381885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:46.382830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789327842821611:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.383123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789326945350197:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789326945350281:2119] held: no
2025-11-28T14:52:46.383061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:46.383219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789327842821611:2109] [node 6] ICP22 created new session: [3:7577789327842821650:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.383267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789327842821650:2109] [node 6] ICS09 handshake done sender: [3:7577789327842821629:2119] self: [3:7577789327842821628:0] peer: [6:7577789325602743633:0] socket: 175 qp: -1
2025-11-28T14:52:46.384498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789325602743637:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11327
2025-11-28T14:52:46.383298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789327842821650:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:46.383530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: 2025-11-28T14:52:46.385558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789326945350281:2119] [node 2] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:46.385562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789325602743537:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7866) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789325602743627:2121] held: no
2025-11-28T14:52:46.385768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789326945350197:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.385981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:46.386013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789326945350197:2049] [node 2] ICP22 created new session: [4:7577789326945350295:2049]
2025-11-28T14:52:46.385996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789328064914040:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-28T14:52:46.386049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789326945350295:2049] [node 2] ICS09 handshake done sender: [4:7577789326945350281:2119] self: [4:7577789326945350285:0] peer: [2:7577789325242855883:0] socket: 128 qp: -1
2025-11-28T14:52:46.386070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789326945350295:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.386135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:46.386162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789326945350294:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53328
2025-11-28T14:52:46.386335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789328064914040:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.386326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789325602743627:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:46.386474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789325602743537:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.386478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789328064914005:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:46.386624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:46.386683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789325602743537:2050] [node 3] ICP22 created new session: [6:7577789325602743638:2050]
2025-11-28T14:52:46.386725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789325602743638:2050] [node 3] ICS09 handshake done sender: [6:7577789325602743627:2121] self: [6:7577789325602743633:0] peer: [3:7577789327842821628:0] socket: 177 qp: -1
2025-11-28T14:52:46.386751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789325602743638:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:46.386739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789326945350273:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.386783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:46.386771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789326945350273:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:46.388018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789326945350294:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:46.388296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789326945350273:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:46.388458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789326945350260:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:46.388417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789328064914043:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-28T14:52:46.388512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:46.388536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789326945350260:2108] [node 5] ICP22 created new session: [4:7577789326945350299:2108]
2025-11-28T14:52:46.388603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789326945350299:2108] [node 5] ICS09 handshake done sender: [4:7577789326945350273:2114] self: [4:7577789326945350272:0] peer: [5:7577789328064914042:0] socket: 115 qp: -1
2025-11-28T14:52:46.388609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789328064914040:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:46.388635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789326945350299:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:46.388667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:46.388705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789328064914005:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:46.388948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:46.388989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789328064914005:2108] [node 4] ICP22 created new session: [5:7577789328064914044:2108]
2025-11-28T14:52:46.389018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789328064914044:2108] [node 4] ICS09 handshake done sender: [5:7577789328064914040:2122] self: [5:7577789328064914042:0] peer: [4:7577789326945350272:0] socket: 121 qp: -1
2025-11-28T14:52:46.389040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789328064914044:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:46.389066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:46.389978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789328064914005:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:46.454992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:46.455194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789325242855899:2048] [node 1] ICS01 socket: 145 reason# EndOfStream
2025-11-28T14:52:46.455233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789325242855801:2048] [node 1] ICP30 unregister session Session# [2:7577789325242855899:2048] VirtualId# [2:7577789325242855888:0]
2025-11-28T14:52:46.455256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789325242855899:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.455350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:46.455400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:46.456495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:46.456749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789327842821647:2048] [node 1] ICS07 socket disconnect 146 reason# EndOfStream
2025-11-28T14:52:46.456783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789327842821647:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.456874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:46.456923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789327842821647:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:46.458701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789327842821546:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:46.458751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789327842821647:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:46.460618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:46.460708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789327842821546:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:46.460750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789327842821647:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:46.460780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789327842821546:2048] [node 1] ICP30 unregister session Session# [3:7577789327842821647:2048] VirtualId# [3:7577789327842821634:0]
2025-11-28T14:52:46.460795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789326945350291:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-28T14:52:46.460881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:46.460885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789326945350196:2048] [node 1] ICP30 unregister session Session# [4:7577789326945350291:2048] VirtualId# [4:7577789326945350283:0]
2025-11-28T14:52:46.460907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789327842821546:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.460910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789326945350291:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.460916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.461003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:46.461045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:46.461892Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:46.462051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789328064914034:2048] [node 1] ICS07 socket disconnect 170 reason# EndOfStream
2025-11-28T14:52:46.462082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789328064914034:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.462140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:46.462178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789328064914034:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:46.462718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789328064913941:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:46.462779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789328064914034:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:46.462807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789328064913941:2048] [node 1] ICP30 unregister session Session# [5:7577789328064914034:2048] VirtualId# [5:7577789328064914027:0]
2025-11-28T14:52:46.462888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:46.462913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789328064913941:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.462922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.463067Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:46.463384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789325602743622:2048] [node 1] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-28T14:52:46.463420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789325602743622:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.463508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:46.463568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789325602743622:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:46.464268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789325602743535:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:46.464340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789325602743622:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:46.464401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789325602743535:2048] [node 1] ICP30 unregister session Session# [6:7577789325602743622:2048] VirtualId# [6:7577789325602743616:0]
2025-11-28T14:52:46.464513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:46.464555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789325602743535:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.464586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.470818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789327842821546:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:46.472810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789328064913941:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:46.474983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789325602743535:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:46.521178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:46.521355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789327842821645:2049] [node 2] ICS07 socket disconnect 124 reason# EndOfStream
2025-11-28T14:52:46.521401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789327842821645:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.521476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:46.521526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789327842821645:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:46.523194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789327842821547:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:46.523226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789327842821645:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:46.524088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789327842821547:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:46.524127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789327842821645:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:46.524152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789327842821547:2049] [node 2] ICP30 unregister session Session# [3:7577789327842821645:2049] VirtualId# [3:7577789327842821632:0]
2025-11-28T14:52:46.524376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:46.524397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789327842821547:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.524406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.524630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:46.524932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789326945350295:2049] [node 2] ICS01 socket: 128 reason# EndOfStream
2025-11-28T14:52:46.524987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789326945350197:2049] [node 2] ICP30 unregister session Session# [4:7577789326945350295:2049] VirtualId# [4:7577789326945350285:0]
2025-11-28T14:52:46.525011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789326945350295:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.525092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:46.525140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:46.525467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:46.525695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789328064914035:2049] [node 2] ICS07 socket disconnect 148 reason# EndOfStream
2025-11-28T14:52:46.525731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789328064914035:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.525803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:46.525846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789328064914035:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:46.526082Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:46.526316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789325602743619:2049] [node 2] ICS07 socket disconnect 132 reason# EndOfStream
2025-11-28T14:52:46.526368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789325602743619:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.526415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789328064913942:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:46.526456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789328064914035:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:46.526472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:46.526574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789325602743619:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:46.526477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789328064913942:2049] [node 2] ICP30 unregister session Session# [5:7577789328064914035:2049] VirtualId# [5:7577789328064914026:0]
2025-11-28T14:52:46.526587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:46.526608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789328064913942:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.526627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.527352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789325602743536:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:46.527445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789325602743619:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:46.527496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789325602743536:2049] [node 2] ICP30 unregister session Session# [6:7577789325602743619:2049] VirtualId# [6:7577789325602743615:0]
2025-11-28T14:52:46.527689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:46.527760Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789325602743536:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.527813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.534305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789327842821547:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:46.536353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789328064913942:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:46.537378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789325602743536:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:46.588838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:46.589085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789326945350279:2050] [node 3] ICS01 socket: 127 reason# EndOfStream
2025-11-28T14:52:46.589137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789326945350198:2050] [node 3] ICP30 unregister session Session# [4:7577789326945350279:2050] VirtualId# [4:7577789326945350275:0]
2025-11-28T14:52:46.589217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789326945350279:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.589316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:46.589371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:46.589731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:46.589976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789328064914028:2050] [node 3] ICS01 socket: 140 reason# EndOfStream
2025-11-28T14:52:46.590029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789328064913943:2050] [node 3] ICP30 unregister session Session# [5:7577789328064914028:2050] VirtualId# [5:7577789328064914020:0]
2025-11-28T14:52:46.590068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789328064914028:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.590145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:46.590204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:46.590384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:46.590620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789325602743638:2050] [node 3] ICS01 socket: 177 reason# EndOfStream
2025-11-28T14:52:46.590671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789325602743537:2050] [node 3] ICP30 unregister session Session# [6:7577789325602743638:2050] VirtualId# [6:7577789325602743633:0]
2025-11-28T14:52:46.590709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789325602743638:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.590807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:46.590866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:46.646814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:46.647078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789328064914044:2108] [node 4] ICS07 socket disconnect 121 reason# EndOfStream
2025-11-28T14:52:46.647120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789328064914044:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.647197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:46.647245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789328064914044:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:46.647928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789328064914005:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20688) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:46.647970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789328064914044:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:46.648001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789328064914005:2108] [node 4] ICP30 unregister session Session# [5:7577789328064914044:2108] VirtualId# [5:7577789328064914042:0]
2025-11-28T14:52:46.648041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:46.648063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789328064914005:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20688) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.648073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20688) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.648117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:46.648329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789325602743632:2107] [node 4] ICS01 socket: 138 reason# EndOfStream
2025-11-28T14:52:46.648384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789325602743598:2107] [node 4] ICP30 unregister session Session# [6:7577789325602743632:2107] VirtualId# [6:7577789325602743628:0]
2025-11-28T14:52:46.648411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789325602743632:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.648498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:46.648537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:46.658220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789328064914005:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:46.709674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:46.709848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789325602743635:2106] [node 5] ICS07 socket disconnect 176 reason# EndOfStream
2025-11-28T14:52:46.709877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789325602743635:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:46.709945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:46.709977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789325602743635:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:46.710661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789325602743597:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15555) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:46.710706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789325602743635:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:46.710789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789325602743597:2106] [node 5] ICP30 unregister session Session# [6:7577789325602743635:2106] VirtualId# [6:7577789325602743629:0]
2025-11-28T14:52:46.710833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:46.710869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789325602743597:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:46.710884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:46.814817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789327881873772:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:46.814880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789327881873773:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:46.831591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789328097370812:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:46.831655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789328097370811:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.848094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789325331595607:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.848244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789325331595608:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:46.865921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789327508958548:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.865963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789327508958549:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:46.865985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789327508958550:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:46.884901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789327065351044:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.884937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789327065351046:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:46.884967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789327065351045:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:46.903003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789328363472039:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:46.903036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789328363472040:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:46.903057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789328363472041:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:46.991359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789327065351107:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:46.992556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789328097370874:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:46.993766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789327881873835:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.993634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789328363472102:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:46.993517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789325331595671:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:46.993759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789327065351108:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.994578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789327881873836:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:46.994538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789328097370875:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.993756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789325331595670:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.994574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789328363472103:2105] [node 4] ICP01 ready to work
2025-11-28T14:52:46.995423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789327508958611:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:46.994585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789327881873837:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:46.994505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789325331595672:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:46.994604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789328097370876:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:46.996109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789327508958612:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.006102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329626562979:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.008013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329626562979:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.008415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789332176841147:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10789
2025-11-28T14:52:47.009256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789325331595607:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.010221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789327881873840:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.010260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789327881873840:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
2025-11-28T14:52:47.011029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789332176841147:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.012149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789331360318416:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51038
2025-11-28T14:52:47.012842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329626562984:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
No RDMA context found for GID: 2025-11-28T14:52:47.012642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439411:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.012875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789331803925919:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43104
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.012867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789332176841148:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10789
2025-11-28T14:52:47.012430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789331360318415:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51038
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.014389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329626562985:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-11-28T14:52:47.013394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789331360318416:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.012968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789327881873840:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:47.015459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329626562977:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.013874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789332658439411:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.013747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789331360318415:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.015467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789331803925920:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43104
2025-11-28T14:52:47.013599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789332176841151:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10789
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.015487Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789329626562977:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.016258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439415:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.016322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789331803925919:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.013936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789327065351046:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.016577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789328363472041:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.020512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789332392338186:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5380
2025-11-28T14:52:47.016286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789327881873773:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.017355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329626562977:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:47.016600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789331803925923:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43104
2025-11-28T14:52:47.021108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789332392338187:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5380
2025-11-28T14:52:47.016345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.015333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789327065351044:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.018574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329626562975:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.016692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789327508958548:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.022836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789332392338186:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.015334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789331360318420:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51038
2025-11-28T14:52:47.016602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789327881873773:2049] [node 3] ICP22 created new session: [1:7577789332176841152:2049]
2025-11-28T14:52:47.018599Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789329626562975:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.021858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789331803925920:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.025646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789332392338181:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.017281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789332176841152:2049] [node 3] ICS09 handshake done sender: [1:7577789327881873840:2110] self: [1:7577789327881873838:0] peer: [3:7577789329626562983:0] socket: 120 qp: -1
2025-11-28T14:52:47.016236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789331360318421:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51038
2025-11-28T14:52:47.022238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329626562986:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.017886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789331360318420:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.017807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789332176841140:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.028153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789332658439415:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.025658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789332392338187:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.022816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789331360318421:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.022892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789331803925923:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.023879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329626562985:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.023867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789327065351045:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.017836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789332176841140:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.025689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789332392338181:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.027784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789332392338180:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.023875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789327508958549:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.029540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789328363472103:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.024396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329626562975:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:47.018657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789332176841152:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:47.024575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789331360318426:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51038
2025-11-28T14:52:47.027802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789332392338180:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.030000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789331803925927:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43104
2025-11-28T14:52:47.025606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329626562987:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-11-28T14:52:47.030354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439417:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
2025-11-28T14:52:47.027786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789331360318415:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:47.020543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.030034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789332392338181:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:47.030950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789331803925913:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.025710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789325331595671:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.031357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789328363472041:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789332658439411:2116] held: no
2025-11-28T14:52:47.030967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789331803925913:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.020828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789332176841141:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.029524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789327065351044:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789331360318415:2115] held: no
2025-11-28T14:52:47.030083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789332392338180:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.026508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329626562984:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.020864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789332176841141:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.032311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789332392338188:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5380
2025-11-28T14:52:47.029985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789331360318424:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51038
2025-11-28T14:52:47.032714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789331803925919:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:47.026588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:47.032912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789328097370874:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.034304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789331803925913:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:47.032467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789331360318416:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:47.021936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789332176841148:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.022810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789332176841140:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:47.027505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789325331595670:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:47.033888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789328097370876:2108] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.033858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789327065351046:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.034771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789327508958548:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.023030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789332176841151:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.034825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:47.037805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439419:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
2025-11-28T14:52:47.027524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789325331595607:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789329626562979:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.034285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.034276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:47.036153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789327508958548:2048] [node 1] ICP22 created new session: [4:7577789331803925932:2048]
2025-11-28T14:52:47.034494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789327065351046:2050] [node 3] ICP22 created new session: [5:7577789331360318430:2050]
2025-11-28T14:52:47.027820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789325331595671:2108] [node 5] ICP22 created new session: [3:7577789329626562991:2108]
2025-11-28T14:52:47.034307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789328097370876:2108] [node 5] ICP22 created new session: [2:7577789332392338193:2108]
2025-11-28T14:52:47.024055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789332176841141:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.037191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789331803925932:2048] [node 1] ICS09 handshake done sender: [4:7577789331803925919:2115] self: [4:7577789331803925925:0] peer: [1:7577789327881873839:0] socket: 132 qp: -1
2025-11-28T14:52:47.028835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329626562991:2108] [node 5] ICS09 handshake done sender: [3:7577789329626562977:2113] self: [3:7577789329626562972:0] peer: [5:7577789331360318422:0] socket: 125 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.025174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789327881873837:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:47.040101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789332658439411:2116] [node 3] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:47.034785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.034810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789331360318430:2050] [node 3] ICS09 handshake done sender: [5:7577789331360318416:2116] self: [5:7577789331360318422:0] peer: [3:7577789329626562972:0] socket: 131 qp: -1
2025-11-28T14:52:47.027485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:47.037247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789331803925932:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.040657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439421:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
2025-11-28T14:52:47.034801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789332392338188:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.028892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.035785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789331360318430:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.037763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:47.036177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789328097370874:2107] [node 4] ICP22 created new session: [2:7577789332392338194:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.027823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789327881873837:2108] [node 5] ICP22 created new session: [1:7577789332176841159:2108]
2025-11-28T14:52:47.029358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329626562991:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:47.040777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789332658439419:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.035819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.029284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789327881873836:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.036749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789327065351044:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.041437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789328363472039:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.039064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789327508958549:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1045) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789331803925920:2116] held: no
2025-11-28T14:52:47.036804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789332392338193:2108] [node 5] ICS09 handshake done sender: [2:7577789332392338181:2113] self: [2:7577789332392338177:0] peer: [5:7577789331360318427:0] socket: 140 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.029399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:47.037798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.041802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789328363472041:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.037171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789332392338193:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:47.029326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.029492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789325331595670:2109] [node 6] ICP22 created new session: [3:7577789329626562992:2109]
2025-11-28T14:52:47.039115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789327508958611:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.042597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789332658439415:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.037819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789331360318428:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51038
2025-11-28T14:52:47.037191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789332392338179:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.029454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789332176841159:2108] [node 5] ICS09 handshake done sender: [1:7577789332176841140:2111] self: [1:7577789327881873841:0] peer: [5:7577789331360318423:0] socket: 122 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.039739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789327065351044:2048] [node 1] ICP22 created new session: [5:7577789331360318432:2048]
2025-11-28T14:52:47.042674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.039762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.040074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789331360318432:2048] [node 1] ICS09 handshake done sender: [5:7577789331360318415:2115] self: [5:7577789331360318423:0] peer: [1:7577789327881873841:0] socket: 130 qp: -1
2025-11-28T14:52:47.042819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439422:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
2025-11-28T14:52:47.030931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329626562992:2109] [node 6] ICS09 handshake done sender: [3:7577789329626562975:2111] self: [3:7577789329626562971:0] peer: [6:7577789332658439416:0] socket: 128 qp: -1
2025-11-28T14:52:47.039816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789327508958611:2109] [node 6] ICP22 created new session: [4:7577789331803925934:2109]
2025-11-28T14:52:47.042850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789328363472041:2050] [node 3] ICP22 created new session: [6:7577789332658439425:2050]
2025-11-28T14:52:47.037226Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789332392338179:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.040102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789331360318432:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.029489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789332176841159:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:47.030993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329626562986:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.041364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789331803925934:2109] [node 6] ICS09 handshake done sender: [4:7577789331803925913:2110] self: [4:7577789331803925910:0] peer: [6:7577789332658439418:0] socket: 153 qp: -1
2025-11-28T14:52:47.038964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789332392338194:2107] [node 4] ICS09 handshake done sender: [2:7577789332392338180:2112] self: [2:7577789332392338176:0] peer: [4:7577789331803925926:0] socket: 137 qp: -1
2025-11-28T14:52:47.043867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789332658439425:2050] [node 3] ICS09 handshake done sender: [6:7577789332658439411:2116] self: [6:7577789332658439416:0] peer: [3:7577789329626562971:0] socket: 133 qp: -1
2025-11-28T14:52:47.044668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789332658439422:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.041671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789331803925934:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:47.031357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329626562992:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:47.044846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789332658439425:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:47.029533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:47.032401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.039448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789332392338194:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:47.041905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.029540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789327881873836:2107] [node 4] ICP22 created new session: [1:7577789332176841161:2107]
2025-11-28T14:52:47.033228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789325331595608:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.039523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:47.040743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789327065351045:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1045) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789331360318421:2118] held: no
2025-11-28T14:52:47.044853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789328363472103:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.034251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329626562979:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:47.030352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789332176841161:2107] [node 4] ICS09 handshake done sender: [1:7577789332176841141:2112] self: [1:7577789327881873839:0] peer: [4:7577789331803925925:0] socket: 121 qp: -1
2025-11-28T14:52:47.042682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789331803925929:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43104
2025-11-28T14:52:47.037990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789325331595607:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.039838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:47.031001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789332176841161:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:47.041683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:47.039732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:47.045572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789331803925931:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43104
2025-11-28T14:52:47.044885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.041171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789328097370811:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.032422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:47.047712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789331360318421:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:47.046571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789331803925920:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:47.039851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789325331595607:2048] [node 1] ICP22 created new session: [3:7577789329626562999:2048]
2025-11-28T14:52:47.047982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789327508958549:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.041615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329626562999:2048] [node 1] ICS09 handshake done sender: [3:7577789329626562979:2115] self: [3:7577789329626562983:0] peer: [1:7577789327881873838:0] socket: 124 qp: -1
2025-11-28T14:52:47.044188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789332392338179:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:47.048711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:47.041658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329626562999:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.044908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.048390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789331360318434:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51038
2025-11-28T14:52:47.042684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789332176841143:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.044612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789332392338195:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5380
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.041942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:47.048447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789331360318414:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.045631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789328363472103:2105] [node 4] ICP22 created new session: [6:7577789332658439427:2105]
2025-11-28T14:52:47.042712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789332176841143:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.049001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789327508958549:2049] [node 2] ICP22 created new session: [4:7577789331803925937:2049]
2025-11-28T14:52:47.048089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789328097370812:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.047173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789328363472102:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.048464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789331360318414:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.050620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329626563001:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-11-28T14:52:47.045547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789332176841142:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.048440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789332392338198:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5380
2025-11-28T14:52:47.049221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789331803925937:2049] [node 2] ICS09 handshake done sender: [4:7577789331803925920:2116] self: [4:7577789331803925926:0] peer: [2:7577789332392338176:0] socket: 142 qp: -1
2025-11-28T14:52:47.047619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789332658439427:2105] [node 4] ICS09 handshake done sender: [6:7577789332658439415:2117] self: [6:7577789332658439418:0] peer: [4:7577789331803925910:0] socket: 154 qp: -1
2025-11-28T14:52:47.049184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789327065351045:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.051246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329626562978:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.048142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789332658439427:2105] [node 4] ICS10 traffic start
2025-11-28T14:52:47.045572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789332176841142:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.048455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:47.051267Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789329626562978:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.050403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:47.047649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789332176841164:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10789
2025-11-28T14:52:47.048641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.051559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329626562984:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:47.048931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439424:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
2025-11-28T14:52:47.048190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789332176841143:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:47.050557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789327065351045:2049] [node 2] ICP22 created new session: [5:7577789331360318435:2049]
2025-11-28T14:52:47.049255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789331803925937:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:47.048990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789328097370812:2049] [node 3] ICP22 created new session: [2:7577789332392338203:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.050906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439428:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
2025-11-28T14:52:47.050272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789331803925931:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.053393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789325331595608:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.050928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789331360318435:2049] [node 2] ICS09 handshake done sender: [5:7577789331360318421:2118] self: [5:7577789331360318427:0] peer: [2:7577789332392338177:0] socket: 149 qp: -1
2025-11-28T14:52:47.048288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789332176841142:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:47.048994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789332392338195:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.050965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789331360318435:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:47.050368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789332392338203:2049] [node 3] ICS09 handshake done sender: [2:7577789332392338179:2111] self: [2:7577789332392338175:0] peer: [3:7577789329626562997:0] socket: 134 qp: -1
2025-11-28T14:52:47.053686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-28T14:52:47.050684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.051295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789332658439424:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.050357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789327881873772:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.050980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789332392338203:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:47.051001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789331360318414:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:47.050815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789327508958550:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.053717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329626562978:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.051563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789328363472040:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.050527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789327881873835:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.051055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.055275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789331803925916:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.051030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.050684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:47.051558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789327065351108:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.055289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789331803925916:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.054993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789325331595608:2049] [node 2] ICP22 created new session: [3:7577789329626563003:2049]
2025-11-28T14:52:47.051072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789332392338198:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.053364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789332658439419:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:47.052754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789331360318434:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.052771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789332392338204:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5380
2025-11-28T14:52:47.050733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789327881873772:2048] [node 2] ICP22 created new session: [1:7577789332176841167:2048]
2025-11-28T14:52:47.055335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789332392338182:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.052807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.059540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789331803925916:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:47.050759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.055361Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789332392338182:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.055368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329626563003:2049] [node 2] ICS09 handshake done sender: [3:7577789329626562984:2116] self: [3:7577789329626562997:0] peer: [2:7577789332392338175:0] socket: 135 qp: -1
2025-11-28T14:52:47.053148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789327065351108:2109] [node 6] ICP22 created new session: [5:7577789331360318438:2109]
2025-11-28T14:52:47.054520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789328363472039:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789332658439419:2119] held: no
2025-11-28T14:52:47.055786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789332392338188:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:47.059881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789331803925940:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43104
2025-11-28T14:52:47.050765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789327881873773:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(26736435ns) {HandleHandshakeStatus:329(26733972ns) {DropIncomingHandshake:442(11007ns) DropOutgoingHandshake:457(368ns) SwitchToState:179(448ns) ProcessPendingSessionEvents:495(21429884ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20032756ns max# 1610210ns}}}
2025-11-28T14:52:47.060273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789328097370811:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.061923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789327508958612:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.051091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789332176841167:2048] [node 2] ICS09 handshake done sender: [1:7577789332176841142:2113] self: [1:7577789332176841139:0] peer: [2:7577789332392338202:0] socket: 127 qp: -1
2025-11-28T14:52:47.055398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329626563003:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:47.054603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789327065351107:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.061039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:47.054989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789331360318438:2109] [node 6] ICS09 handshake done sender: [5:7577789331360318414:2114] self: [5:7577789331360318410:0] peer: [6:7577789332658439429:0] socket: 146 qp: -1
2025-11-28T14:52:47.051119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789327881873835:2109] [node 6] ICP22 created new session: [1:7577789332176841168:2109]
2025-11-28T14:52:47.055033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789331360318438:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:47.061328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789328097370811:2048] [node 1] ICP22 created new session: [2:7577789332392338207:2048]
2025-11-28T14:52:47.051330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789332176841167:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:47.055664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.055082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.063298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:47.060186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789328363472102:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789332658439422:2121] held: no
2025-11-28T14:52:47.059298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789325331595672:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.052776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.062406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789332392338207:2048] [node 1] ICS09 handshake done sender: [2:7577789332392338188:2117] self: [2:7577789332392338202:0] peer: [1:7577789332176841139:0] socket: 166 qp: -1
2025-11-28T14:52:47.063299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789331803925931:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:47.063341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789327508958612:2108] [node 5] ICP22 created new session: [4:7577789331803925942:2108]
2025-11-28T14:52:47.053109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789332176841168:2109] [node 6] ICS09 handshake done sender: [1:7577789332176841143:2114] self: [1:7577789327881873842:0] peer: [6:7577789332658439423:0] socket: 126 qp: -1
2025-11-28T14:52:47.060469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439431:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
2025-11-28T14:52:47.053653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789332176841168:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:47.063408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789327508958548:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17437068ns) {HandleHandshakeStatus:329(17435610ns) {DropIncomingHandshake:442(1596ns) DropOutgoingHandshake:457(5502ns) SwitchToState:179(310ns) ProcessPendingSessionEvents:495(11187117ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9706713ns max# 1165569ns}}}
2025-11-28T14:52:47.060225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789325331595671:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24615133ns) {HandleHandshakeStatus:329(24613550ns) {DropIncomingHandshake:442(1216ns) DropOutgoingHandshake:457(1741ns) SwitchToState:179(290ns) ProcessPendingSessionEvents:495(20565777ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20429272ns max# 1884044ns}}}
2025-11-28T14:52:47.063123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789332392338207:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.060926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789328363472039:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.064003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789327065351046:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17776923ns) {HandleHandshakeStatus:329(17775143ns) {DropIncomingHandshake:442(1434ns) DropOutgoingHandshake:457(5772ns) SwitchToState:179(207ns) ProcessPendingSessionEvents:495(11765826ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11688226ns max# 1131456ns}}}
2025-11-28T14:52:47.063416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789327508958550:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789331803925931:2120] held: no
2025-11-28T14:52:47.060967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:47.054532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789332176841164:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.064052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789331360318441:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51038
2025-11-28T14:52:47.064029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789331803925942:2108] [node 5] ICS09 handshake done sender: [4:7577789331803925916:2112] self: [4:7577789331803925909:0] peer: [5:7577789331360318439:0] socket: 186 qp: -1
2025-11-28T14:52:47.060258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.061001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789328363472039:2048] [node 1] ICP22 created new session: [6:7577789332658439433:2048]
2025-11-28T14:52:47.054729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.064076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789331803925942:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:47.063490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:47.060755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789325331595672:2107] [node 4] ICP22 created new session: [3:7577789329626563005:2107]
2025-11-28T14:52:47.061910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789327881873837:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(25181055ns) {HandleHandshakeStatus:329(25179533ns) {DropIncomingHandshake:442(9397ns) DropOutgoingHandshake:457(786ns) SwitchToState:179(178ns) ProcessPendingSessionEvents:495(17881472ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17785938ns max# 1335469ns}}}
2025-11-28T14:52:47.062845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789332658439433:2048] [node 1] ICS09 handshake done sender: [6:7577789332658439419:2119] self: [6:7577789332658439423:0] peer: [1:7577789327881873842:0] socket: 143 qp: -1
2025-11-28T14:52:47.063108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789332658439433:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.064450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789327065351107:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24744) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789331360318434:2122] held: no
2025-11-28T14:52:47.063268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:47.064039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789328097370812:2049] [node 3] ICP18 handshake (actor [2:7577789332392338195:2118]) is too old
2025-11-28T14:52:47.061893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329626563005:2107] [node 4] ICS09 handshake done sender: [3:7577789329626562978:2114] self: [3:7577789329626562974:0] peer: [4:7577789331803925938:0] socket: 176 qp: -1
2025-11-28T14:52:47.063791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789327881873836:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22896521ns) {HandleHandshakeStatus:329(21750410ns) {DropIncomingHandshake:442(122ns) DropOutgoingHandshake:457(237ns) SwitchToState:179(148ns) ProcessPendingSessionEvents:495(17167674ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15655680ns max# 1206104ns}}}
2025-11-28T14:52:47.064521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:47.064484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789328097370876:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17618638ns) {HandleHandshakeStatus:329(17431058ns) {DropIncomingHandshake:442(11002ns) DropOutgoingHandshake:457(269ns) SwitchToState:179(224ns) ProcessPendingSessionEvents:495(9606400ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9530527ns max# 1133872ns}}}
2025-11-28T14:52:47.064683Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789327508958611:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16253706ns) {HandleHandshakeStatus:329(16243806ns) {DropIncomingHandshake:442(727ns) DropOutgoingHandshake:457(482ns) SwitchToState:179(534ns) ProcessPendingSessionEvents:495(11104853ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11015518ns max# 1258481ns}}}
2025-11-28T14:52:47.065364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789332658439422:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:47.062411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329626563005:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.065516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789327508958550:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.066310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.066294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789328363472102:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.066612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789332658439435:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19052
2025-11-28T14:52:47.063120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:47.066446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789327508958550:2050] [node 3] ICP22 created new session: [4:7577789331803925944:2050]
2025-11-28T14:52:47.066643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789331803925944:2050] [node 3] ICS09 handshake done sender: [4:7577789331803925931:2120] self: [4:7577789331803925938:0] peer: [3:7577789329626562974:0] socket: 178 qp: -1
2025-11-28T14:52:47.066677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789331803925944:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:47.066688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:47.066709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.066844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789328097370874:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22454657ns) {HandleHandshakeStatus:329(22452826ns) {DropIncomingHandshake:442(15291ns) DropOutgoingHandshake:457(419ns) SwitchToState:179(285ns) ProcessPendingSessionEvents:495(12480394ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12396298ns max# 1762938ns}}}
2025-11-28T14:52:47.064141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789325331595670:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23157566ns) {HandleHandshakeStatus:329(23155722ns) {DropIncomingHandshake:442(7984ns) DropOutgoingHandshake:457(202ns) SwitchToState:179(118ns) ProcessPendingSessionEvents:495(17234396ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17142567ns max# 1656673ns}}}
2025-11-28T14:52:47.066874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789328363472102:2107] [node 5] ICP22 created new session: [6:7577789332658439436:2107]
2025-11-28T14:52:47.066999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789332392338182:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:47.067148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789332658439436:2107] [node 5] ICS09 handshake done sender: [6:7577789332658439422:2121] self: [6:7577789332658439429:0] peer: [5:7577789331360318410:0] socket: 155 qp: -1
2025-11-28T14:52:47.067295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789327881873772:2048] [node 2] ICP18 handshake (actor [1:7577789332176841164:2118]) is too old
2025-11-28T14:52:47.067767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789331360318434:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.067256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789332658439436:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:47.067344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:47.067724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789332658439424:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:47.068627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789328097370875:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.068306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789328363472040:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.068984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.069187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:47.069225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789328363472040:2049] [node 2] ICP22 created new session: [6:7577789332658439438:2049]
2025-11-28T14:52:47.069460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789328097370875:2109] [node 6] ICP22 created new session: [2:7577789332392338210:2109]
2025-11-28T14:52:47.069795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789332658439438:2049] [node 2] ICS09 handshake done sender: [6:7577789332658439424:2122] self: [6:7577789332658439432:0] peer: [2:7577789332392338174:0] socket: 183 qp: -1
2025-11-28T14:52:47.069825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789332658439438:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:47.070020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.069497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789332392338210:2109] [node 6] ICS09 handshake done sender: [2:7577789332392338182:2114] self: [2:7577789332392338174:0] peer: [6:7577789332658439432:0] socket: 150 qp: -1
2025-11-28T14:52:47.069639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789332392338210:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:47.070063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789327065351107:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.070025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.070711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.070764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789327881873835:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16988429ns) {HandleHandshakeStatus:329(16986937ns) {DropIncomingHandshake:442(14336ns) DropOutgoingHandshake:457(207ns) SwitchToState:179(198ns) ProcessPendingSessionEvents:495(10364579ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9546220ns max# 3211408ns}}}
2025-11-28T14:52:47.070775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789327065351107:2108] [node 4] ICP22 created new session: [5:7577789331360318442:2108]
2025-11-28T14:52:47.070923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789331360318442:2108] [node 4] ICS09 handshake done sender: [5:7577789331360318434:2122] self: [5:7577789331360318439:0] peer: [4:7577789331803925909:0] socket: 187 qp: -1
2025-11-28T14:52:47.070949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789331360318442:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:47.071039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:47.117047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:47.117214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789332392338207:2048] [node 1] ICS07 socket disconnect 166 reason# EndOfStream
2025-11-28T14:52:47.117247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789332392338207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.117343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.117395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789332392338207:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:47.118004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:47.118185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789329626562999:2048] [node 1] ICS01 socket: 124 reason# EndOfStream
2025-11-28T14:52:47.118228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789325331595607:2048] [node 1] ICP30 unregister session Session# [3:7577789329626562999:2048] VirtualId# [3:7577789329626562983:0]
2025-11-28T14:52:47.118273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789329626562999:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.118390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.118459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.119265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789328097370811:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:47.119299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789332392338207:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:47.120249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789328097370811:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.120331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789332392338207:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.120357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789328097370811:2048] [node 1] ICP30 unregister session Session# [2:7577789332392338207:2048] VirtualId# [2:7577789332392338202:0]
2025-11-28T14:52:47.120469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.120498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789328097370811:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.120481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:47.120507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.120628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789331803925932:2048] [node 1] ICS07 socket disconnect 132 reason# EndOfStream
2025-11-28T14:52:47.120657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789331803925932:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.120718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.120766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789331803925932:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:47.121260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:47.121372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789327508958548:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.121406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789331803925932:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.121428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789327508958548:2048] [node 1] ICP30 unregister session Session# [4:7577789331803925932:2048] VirtualId# [4:7577789331803925925:0]
2025-11-28T14:52:47.121511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.121440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789331360318432:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-28T14:52:47.121528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789327508958548:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.121543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789327065351044:2048] [node 1] ICP30 unregister session Session# [5:7577789331360318432:2048] VirtualId# [5:7577789331360318423:0]
2025-11-28T14:52:47.121536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9514) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.121568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789331360318432:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.121636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.121693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.122061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:47.122226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789332658439433:2048] [node 1] ICS01 socket: 143 reason# EndOfStream
2025-11-28T14:52:47.122270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789328363472039:2048] [node 1] ICP30 unregister session Session# [6:7577789332658439433:2048] VirtualId# [6:7577789332658439423:0]
2025-11-28T14:52:47.122305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789332658439433:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.122392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.122442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.130228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789328097370811:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:47.131246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789327508958548:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:47.179167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:47.179335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789329626563003:2049] [node 2] ICS07 socket disconnect 135 reason# EndOfStream
2025-11-28T14:52:47.179366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789329626563003:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.179459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:47.179546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789329626563003:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:47.181110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:47.181260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789331803925937:2049] [node 2] ICS01 socket: 142 reason# EndOfStream
2025-11-28T14:52:47.181350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789327508958549:2049] [node 2] ICP30 unregister session Session# [4:7577789331803925937:2049] VirtualId# [4:7577789331803925926:0]
2025-11-28T14:52:47.181375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789331803925937:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.181437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:47.181465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:47.181625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789325331595608:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1045) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:47.181653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789329626563003:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:47.182519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789325331595608:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1045) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.182560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789329626563003:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.182564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:47.182616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789325331595608:2049] [node 2] ICP30 unregister session Session# [3:7577789329626563003:2049] VirtualId# [3:7577789329626562997:0]
2025-11-28T14:52:47.182650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:47.182684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789325331595608:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.182694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.182759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789331360318435:2049] [node 2] ICS01 socket: 149 reason# EndOfStream
2025-11-28T14:52:47.182805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789327065351045:2049] [node 2] ICP30 unregister session Session# [5:7577789331360318435:2049] VirtualId# [5:7577789331360318427:0]
2025-11-28T14:52:47.182841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789331360318435:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.182940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:47.182986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:47.183391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:47.183623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789332658439438:2049] [node 2] ICS07 socket disconnect 183 reason# EndOfStream
2025-11-28T14:52:47.183659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789332658439438:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.183727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:47.183776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789332658439438:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:47.184451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789328363472040:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1045) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.184505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789332658439438:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.184577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789328363472040:2049] [node 2] ICP30 unregister session Session# [6:7577789332658439438:2049] VirtualId# [6:7577789332658439432:0]
2025-11-28T14:52:47.184640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:47.184668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789328363472040:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.184689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.194829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789328363472040:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:47.228905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:47.229071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789331803925944:2050] [node 3] ICS01 socket: 178 reason# EndOfStream
2025-11-28T14:52:47.229110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789327508958550:2050] [node 3] ICP30 unregister session Session# [4:7577789331803925944:2050] VirtualId# [4:7577789331803925938:0]
2025-11-28T14:52:47.229140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789331803925944:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.229213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:47.229249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:47.229838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:47.230006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789331360318430:2050] [node 3] ICS07 socket disconnect 131 reason# EndOfStream
2025-11-28T14:52:47.230037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789331360318430:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.230122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:47.230160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789331360318430:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:47.230538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:47.230750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789332658439425:2050] [node 3] ICS01 socket: 133 reason# EndOfStream
2025-11-28T14:52:47.230811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789328363472041:2050] [node 3] ICP30 unregister session Session# [6:7577789332658439425:2050] VirtualId# [6:7577789332658439416:0]
2025-11-28T14:52:47.230855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789327065351046:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30590) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.230895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789331360318430:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.230864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789332658439425:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.230948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:47.230921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789327065351046:2050] [node 3] ICP30 unregister session Session# [5:7577789331360318430:2050] VirtualId# [5:7577789331360318422:0]
2025-11-28T14:52:47.231013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:47.231010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:47.231030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789327065351046:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30590) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.231044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30590) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.240895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789327065351046:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:47.303167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:47.303337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789331360318442:2108] [node 4] ICS01 socket: 187 reason# EndOfStream
2025-11-28T14:52:47.303385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789327065351107:2108] [node 4] ICP30 unregister session Session# [5:7577789331360318442:2108] VirtualId# [5:7577789331360318439:0]
2025-11-28T14:52:47.303424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789331360318442:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.303521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:47.303564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:47.303938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:47.304083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789332658439427:2105] [node 4] ICS01 socket: 154 reason# EndOfStream
2025-11-28T14:52:47.304135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789328363472103:2105] [node 4] ICP30 unregister session Session# [6:7577789332658439427:2105] VirtualId# [6:7577789332658439418:0]
2025-11-28T14:52:47.304170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789332658439427:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.304234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:47.304271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:47.368101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:47.368262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789332658439436:2107] [node 5] ICS01 socket: 155 reason# EndOfStream
2025-11-28T14:52:47.368292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789328363472102:2107] [node 5] ICP30 unregister session Session# [6:7577789332658439436:2107] VirtualId# [6:7577789332658439429:0]
2025-11-28T14:52:47.368313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789332658439436:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.368404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:47.368442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:47.454994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789331062374021:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:47.455040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789331062374022:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:47.472262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789329566742923:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.472300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789329566742924:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:47.488462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789329258658295:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.488500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789329258658296:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:47.504692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789332066184548:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.504739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789332066184549:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:47.504771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789332066184550:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:47.520707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789330185917723:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.520756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789330185917724:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:47.520780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789330185917725:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:47.538287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789331863591297:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.538324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789331863591298:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:47.538353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789331863591299:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:47.620266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789331863591360:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:47.622558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789331062374084:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:47.622655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789332066184611:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:47.622651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789329566743004:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:47.622667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789329258658358:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:47.622854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789331062374085:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:47.622874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789330185917787:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:47.623018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789329258658359:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:47.623003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789329566743005:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:47.623155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789331863591361:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:47.623153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789332066184612:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:47.623376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789330185917788:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:47.623382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789331062374086:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:47.623635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789329258658360:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:47.626463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789329566743008:2115] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.631369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789330185917799:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11279
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.632168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789331062374097:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44608
2025-11-28T14:52:47.632228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789331062374098:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44608
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.632540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789331062374102:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44608
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.637302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789332066184626:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59154
2025-11-28T14:52:47.637571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789330185917799:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.638347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329258658371:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42088
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.638547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329258658372:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42088
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.638688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329258658373:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42088
2025-11-28T14:52:47.638648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789329566743013:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33880
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.638894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789329566743014:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33880
2025-11-28T14:52:47.638926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789332066184626:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.638922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374098:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.639038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374102:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.639051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789332066184548:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.639081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789329566743015:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33880
2025-11-28T14:52:47.639131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374097:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.639164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329258658371:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.639262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329258658372:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.639223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789329566743016:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33880
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.639467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329258658373:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.639514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789332066184628:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59154
2025-11-28T14:52:47.639502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374092:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.639532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789331062374092:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.640008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789329566743013:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.640163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789332066184629:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59154
2025-11-28T14:52:47.640295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789329258658295:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.640279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789329566743014:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.640448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789329566743015:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.640835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789329566743016:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.640962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789331062374103:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44608
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.641089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789331062374104:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44608
2025-11-28T14:52:47.641117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789332066184630:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59154
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.641229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789331062374092:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.641609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789329566743024:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33880
2025-11-28T14:52:47.641719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789329566742923:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.641986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789331062374084:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.642140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.642207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789331062374084:2107] [node 4] ICP22 created new session: [1:7577789331062374109:2107]
2025-11-28T14:52:47.642206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789332066184626:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:47.642267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789331062374109:2107] [node 4] ICS09 handshake done sender: [1:7577789331062374092:2112] self: [1:7577789331062374091:0] peer: [4:7577789332066184627:0] socket: 147 qp: -1
2025-11-28T14:52:47.642315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789331062374109:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:47.642390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:47.642442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789332066184548:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.642524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:47.642566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789332066184548:2048] [node 1] ICP22 created new session: [4:7577789332066184631:2048]
2025-11-28T14:52:47.642600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789332066184631:2048] [node 1] ICS09 handshake done sender: [4:7577789332066184626:2115] self: [4:7577789332066184627:0] peer: [1:7577789331062374091:0] socket: 149 qp: -1
2025-11-28T14:52:47.642621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789332066184631:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.642663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:47.645454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374090:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.645481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789331062374090:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.647024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329258658377:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42088
2025-11-28T14:52:47.647309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374088:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.647332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789331062374088:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.648589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789332066184628:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.649041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789332066184550:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.649047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789330185917804:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11279
2025-11-28T14:52:47.649150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329258658377:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.649245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789329258658296:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.649242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789329566743024:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.649268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374103:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.649506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789329566743010:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.649543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789329566743010:2129] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.649594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374104:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.649791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789331062374090:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:47.650072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329258658382:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42088
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.650537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789332066184629:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.650537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789330185917805:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11279
2025-11-28T14:52:47.650586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789331062374088:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:47.650765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789332066184549:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.650770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789330185917804:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.650911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329258658366:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.650961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789330185917787:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.650934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789329258658366:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.650997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789331062374022:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.651058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.651095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789329566743012:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.651112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789331062374022:2049] [node 3] ICP22 created new session: [1:7577789331062374115:2049]
2025-11-28T14:52:47.651126Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789329566743012:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.651145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789331062374115:2049] [node 3] ICS09 handshake done sender: [1:7577789331062374090:2111] self: [1:7577789331062374089:0] peer: [3:7577789329258658380:0] socket: 146 qp: -1
2025-11-28T14:52:47.651180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789331062374115:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:47.651160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789332066184620:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.651288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.651182Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789332066184620:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.651829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329258658373:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:47.651957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789332066184620:2113] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:47.652058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789329566743029:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33880
2025-11-28T14:52:47.652345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789332066184611:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.652424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:47.652463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789332066184611:2108] [node 5] ICP22 created new session: [4:7577789332066184637:2108]
2025-11-28T14:52:47.652494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789332066184637:2108] [node 5] ICS09 handshake done sender: [4:7577789332066184620:2113] self: [4:7577789332066184619:0] peer: [5:7577789330185917806:0] socket: 128 qp: -1
2025-11-28T14:52:47.652497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789330185917807:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11279
2025-11-28T14:52:47.652544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789329258658295:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.652521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789332066184637:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:47.652638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:47.652813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:47.652891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789329258658295:2048] [node 1] ICP22 created new session: [3:7577789329258658385:2048]
2025-11-28T14:52:47.652949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329258658385:2048] [node 1] ICS09 handshake done sender: [3:7577789329258658373:2117] self: [3:7577789329258658380:0] peer: [1:7577789331062374089:0] socket: 153 qp: -1
2025-11-28T14:52:47.653011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789329566743010:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:47.653051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329258658385:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.653114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789331062374021:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.653133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:47.653153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:47.653179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789331062374021:2048] [node 2] ICP22 created new session: [1:7577789331062374117:2048]
2025-11-28T14:52:47.653206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789330185917804:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.653207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789331062374117:2048] [node 2] ICS09 handshake done sender: [1:7577789331062374088:2110] self: [1:7577789331062374087:0] peer: [2:7577789329566743028:0] socket: 142 qp: -1
2025-11-28T14:52:47.653269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789331062374117:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:47.653424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.653622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789329566742924:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.653690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.653736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789330185917787:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.653748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789329566742924:2049] [node 3] ICP22 created new session: [2:7577789329566743033:2049]
2025-11-28T14:52:47.653815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789329566743033:2049] [node 3] ICS09 handshake done sender: [2:7577789329566743010:2129] self: [2:7577789329566743009:0] peer: [3:7577789329258658381:0] socket: 160 qp: -1
2025-11-28T14:52:47.653816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.653847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789329566743033:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:47.653878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.653873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789330185917787:2108] [node 4] ICP22 created new session: [5:7577789330185917808:2108]
2025-11-28T14:52:47.653924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789330185917808:2108] [node 4] ICS09 handshake done sender: [5:7577789330185917804:2117] self: [5:7577789330185917806:0] peer: [4:7577789332066184619:0] socket: 136 qp: -1
2025-11-28T14:52:47.653962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789330185917808:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:47.654060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:47.654053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789331062374021:2048] [node 2] ICP18 handshake (actor [1:7577789331062374104:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.654551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789332066184639:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59154
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.654932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591375:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.655328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789329566743016:2134] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.657099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329258658383:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42088
2025-11-28T14:52:47.657937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329258658366:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.657939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789329566742923:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.657996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:47.658043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789329566742923:2048] [node 1] ICP22 created new session: [2:7577789329566743035:2048]
2025-11-28T14:52:47.658088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789329566743035:2048] [node 1] ICS09 handshake done sender: [2:7577789329566743016:2134] self: [2:7577789329566743028:0] peer: [1:7577789331062374087:0] socket: 145 qp: -1
2025-11-28T14:52:47.658124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789329566743035:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.658226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:47.658264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789331863591375:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.658394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789331863591361:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.658741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789330185917805:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.658859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789329566742923:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:47.658888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591377:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.658990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591378:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.659397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789332066184622:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.659432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789332066184622:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.660296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789332066184628:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.660577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789329258658296:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22660) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789329258658377:2118] held: no
2025-11-28T14:52:47.660720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789330185917723:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.660747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789332066184641:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59154
2025-11-28T14:52:47.660890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789332066184550:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.660991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.661027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789332066184550:2050] [node 3] ICP22 created new session: [4:7577789332066184643:2050]
2025-11-28T14:52:47.661072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789332066184643:2050] [node 3] ICS09 handshake done sender: [4:7577789332066184628:2116] self: [4:7577789332066184634:0] peer: [3:7577789329258658365:0] socket: 158 qp: -1
2025-11-28T14:52:47.661100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789332066184643:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:47.661127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.661182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789331863591378:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.661296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591379:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.661333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789329258658358:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.661390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789331863591360:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.661425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.661415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789329566743012:2130] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.661478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789329258658358:2107] [node 4] ICP22 created new session: [3:7577789329258658387:2107]
2025-11-28T14:52:47.661562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329258658387:2107] [node 4] ICS09 handshake done sender: [3:7577789329258658366:2112] self: [3:7577789329258658365:0] peer: [4:7577789332066184634:0] socket: 156 qp: -1
2025-11-28T14:52:47.661610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329258658387:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:47.661655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:47.661655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789330185917798:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.661663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789331863591377:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.661689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789330185917798:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.661678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374094:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.661788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789331863591297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.661706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789331062374094:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.662293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789329566743004:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.662354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.662400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789329566743004:2107] [node 4] ICP22 created new session: [2:7577789329566743038:2107]
2025-11-28T14:52:47.662531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789330185917798:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:47.662425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789329566743038:2107] [node 4] ICS09 handshake done sender: [2:7577789329566743012:2130] self: [2:7577789329566743011:0] peer: [4:7577789332066184635:0] socket: 161 qp: -1
2025-11-28T14:52:47.662564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789329566743038:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:47.662587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:47.662631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789331062374096:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.662652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789331062374096:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.662776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591382:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.663291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789330185917788:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.663813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.663885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789330185917788:2109] [node 6] ICP22 created new session: [5:7577789330185917813:2109]
2025-11-28T14:52:47.663934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789330185917813:2109] [node 6] ICS09 handshake done sender: [5:7577789330185917798:2115] self: [5:7577789330185917797:0] peer: [6:7577789331863591380:0] socket: 164 qp: -1
2025-11-28T14:52:47.663977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789330185917813:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:47.664037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.664086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789331062374094:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:47.664242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789331863591378:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:47.664306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789332066184622:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:47.664793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789331062374096:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:47.665047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789331062374085:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.665091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:47.665094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789331863591360:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.665124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789331062374085:2108] [node 5] ICP22 created new session: [1:7577789331062374122:2108]
2025-11-28T14:52:47.665160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789331062374122:2108] [node 5] ICS09 handshake done sender: [1:7577789331062374094:2113] self: [1:7577789331062374093:0] peer: [5:7577789330185917811:0] socket: 173 qp: -1
2025-11-28T14:52:47.665157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:47.665182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789331062374122:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:47.665190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789331863591360:2107] [node 5] ICP22 created new session: [6:7577789331863591384:2107]
2025-11-28T14:52:47.665231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789331863591384:2107] [node 5] ICS09 handshake done sender: [6:7577789331863591378:2117] self: [6:7577789331863591380:0] peer: [5:7577789330185917797:0] socket: 184 qp: -1
2025-11-28T14:52:47.665231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:47.665263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789331863591384:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:47.665310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:47.665427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789332066184612:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.665518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.665546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789332066184612:2109] [node 6] ICP22 created new session: [4:7577789332066184645:2109]
2025-11-28T14:52:47.665577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789332066184645:2109] [node 6] ICS09 handshake done sender: [4:7577789332066184622:2114] self: [4:7577789332066184621:0] peer: [6:7577789331863591376:0] socket: 182 qp: -1
2025-11-28T14:52:47.665604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789332066184645:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:47.665622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.665958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789331062374086:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.665947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329258658377:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:47.665992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.666019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789331062374086:2109] [node 6] ICP22 created new session: [1:7577789331062374124:2109]
2025-11-28T14:52:47.666096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789331062374124:2109] [node 6] ICS09 handshake done sender: [1:7577789331062374096:2114] self: [1:7577789331062374095:0] peer: [6:7577789331863591381:0] socket: 159 qp: -1
2025-11-28T14:52:47.666118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789331062374124:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:47.666303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.667912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789329258658296:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.667986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:47.668539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789329258658296:2049] [node 2] ICP22 created new session: [3:7577789329258658391:2049]
2025-11-28T14:52:47.668682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329258658391:2049] [node 2] ICS09 handshake done sender: [3:7577789329258658377:2118] self: [3:7577789329258658381:0] peer: [2:7577789329566743009:0] socket: 162 qp: -1
2025-11-28T14:52:47.668723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329258658391:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:47.668766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.668818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789332066184629:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.669192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591383:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.669547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789331863591375:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:47.669544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789332066184549:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22660) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789332066184629:2117] held: no
2025-11-28T14:52:47.670320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789332066184549:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.670695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:47.670744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789332066184549:2049] [node 2] ICP22 created new session: [4:7577789332066184647:2049]
2025-11-28T14:52:47.671159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789332066184647:2049] [node 2] ICS09 handshake done sender: [4:7577789332066184629:2117] self: [4:7577789332066184635:0] peer: [2:7577789329566743011:0] socket: 163 qp: -1
2025-11-28T14:52:47.671220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789332066184647:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:47.671252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.671353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789330185917723:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789330185917805:2118] held: no
2025-11-28T14:52:47.671395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789331863591297:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789331863591377:2116] held: no
2025-11-28T14:52:47.671783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789331863591361:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.671864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:47.671894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789331863591361:2109] [node 4] ICP22 created new session: [6:7577789331863591387:2109]
2025-11-28T14:52:47.671930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789331863591387:2109] [node 4] ICS09 handshake done sender: [6:7577789331863591375:2115] self: [6:7577789331863591376:0] peer: [4:7577789332066184621:0] socket: 157 qp: -1
2025-11-28T14:52:47.671979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789331863591387:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:47.672014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.672131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789330185917815:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11279
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.672573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789330185917816:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11279
2025-11-28T14:52:47.673429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789330185917816:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.673509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789330185917805:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.673711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789331863591377:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:47.673776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789330185917817:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11279
2025-11-28T14:52:47.673920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789330185917723:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.673971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:47.674124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789330185917723:2048] [node 1] ICP22 created new session: [5:7577789330185917818:2048]
2025-11-28T14:52:47.674291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789330185917818:2048] [node 1] ICS09 handshake done sender: [5:7577789330185917805:2118] self: [5:7577789330185917811:0] peer: [1:7577789331062374093:0] socket: 174 qp: -1
2025-11-28T14:52:47.674361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789330185917818:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.674422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.674686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591386:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.674882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789331863591297:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.674924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:47.674954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789331863591297:2048] [node 1] ICP22 created new session: [6:7577789331863591391:2048]
2025-11-28T14:52:47.674991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789331863591391:2048] [node 1] ICS09 handshake done sender: [6:7577789331863591377:2116] self: [6:7577789331863591381:0] peer: [1:7577789331062374095:0] socket: 183 qp: -1
2025-11-28T14:52:47.675013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789331863591391:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:47.675035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:47.675429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789330185917817:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.675774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789331863591386:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.675822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789330185917724:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.675989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789329258658393:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42088
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.676082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591389:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.676089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789330185917725:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.676106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789329566743018:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.676138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789329566743018:2135] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.676254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789331863591298:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.676381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329258658368:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.676415Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789329258658368:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.676720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789329566743018:2135] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:47.677502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789329566743005:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.677560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:47.677599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789329566743005:2108] [node 5] ICP22 created new session: [2:7577789329566743042:2108]
2025-11-28T14:52:47.677594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789331863591389:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.677629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789329566743042:2108] [node 5] ICS09 handshake done sender: [2:7577789329566743018:2135] self: [2:7577789329566743017:0] peer: [5:7577789330185917820:0] socket: 190 qp: -1
2025-11-28T14:52:47.677655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789329566743042:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:47.677697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.677706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329258658368:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:47.677764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789331863591299:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:47.677794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789330185917822:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11279
2025-11-28T14:52:47.678051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329258658393:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.678141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789329258658359:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.678201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.678246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789329258658359:2108] [node 5] ICP22 created new session: [3:7577789329258658395:2108]
2025-11-28T14:52:47.678290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329258658395:2108] [node 5] ICS09 handshake done sender: [3:7577789329258658368:2113] self: [3:7577789329258658367:0] peer: [5:7577789330185917821:0] socket: 130 qp: -1
2025-11-28T14:52:47.678323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329258658395:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:47.678286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789330185917823:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11279
2025-11-28T14:52:47.678382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:47.678568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789329566743020:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.678594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789329566743020:2136] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.678628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789330185917816:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:47.679209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789330185917724:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.679281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:47.679328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789330185917724:2049] [node 2] ICP22 created new session: [5:7577789330185917824:2049]
2025-11-28T14:52:47.679414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789330185917824:2049] [node 2] ICS09 handshake done sender: [5:7577789330185917816:2121] self: [5:7577789330185917820:0] peer: [2:7577789329566743017:0] socket: 191 qp: -1
2025-11-28T14:52:47.679455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789330185917824:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:47.679519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.679819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789329566743020:2136] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.679900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591395:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.680005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789329258658370:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:47.680022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789329566743008:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.680038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789329258658370:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:47.680061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.680113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789329566743008:2115] [node 6] ICP22 created new session: [2:7577789329566743045:2115]
2025-11-28T14:52:47.680143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789329566743045:2115] [node 6] ICS09 handshake done sender: [2:7577789329566743020:2136] self: [2:7577789329566743019:0] peer: [6:7577789331863591393:0] socket: 143 qp: -1
2025-11-28T14:52:47.680161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789329566743045:2115] [node 6] ICS10 traffic start
2025-11-28T14:52:47.680346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.680872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789331863591386:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:47.681100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789331863591298:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.681190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:47.681247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789331863591298:2049] [node 2] ICP22 created new session: [6:7577789331863591396:2049]
2025-11-28T14:52:47.681295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789331863591396:2049] [node 2] ICS09 handshake done sender: [6:7577789331863591386:2121] self: [6:7577789331863591393:0] peer: [2:7577789329566743019:0] socket: 126 qp: -1
2025-11-28T14:52:47.681325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789331863591396:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:47.681363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:47.681620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789329258658370:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:47.681937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789329258658360:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:47.681982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:47.682028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789329258658360:2109] [node 6] ICP22 created new session: [3:7577789329258658399:2109]
2025-11-28T14:52:47.682065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789329258658399:2109] [node 6] ICS09 handshake done sender: [3:7577789329258658370:2114] self: [3:7577789329258658369:0] peer: [6:7577789331863591394:0] socket: 135 qp: -1
2025-11-28T14:52:47.682061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789330185917817:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:47.682094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789329258658399:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:47.682147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:47.685394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789330185917725:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.685494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.685545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789330185917725:2050] [node 3] ICP22 created new session: [5:7577789330185917826:2050]
2025-11-28T14:52:47.685592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789330185917826:2050] [node 3] ICS09 handshake done sender: [5:7577789330185917817:2122] self: [5:7577789330185917821:0] peer: [3:7577789329258658367:0] socket: 127 qp: -1
2025-11-28T14:52:47.685631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789330185917826:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:47.685696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.686155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789331863591299:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789331863591389:2122] held: no
2025-11-28T14:52:47.686312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789330185917725:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:47.686795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789331863591398:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:590
2025-11-28T14:52:47.687109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789331863591389:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:47.687268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789331863591299:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:47.687309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:47.687338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789331863591299:2050] [node 3] ICP22 created new session: [6:7577789331863591399:2050]
2025-11-28T14:52:47.687360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789331863591399:2050] [node 3] ICS09 handshake done sender: [6:7577789331863591389:2122] self: [6:7577789331863591394:0] peer: [3:7577789329258658369:0] socket: 137 qp: -1
2025-11-28T14:52:47.687391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789331863591399:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:47.687416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:47.701470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:47.701636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789329566743035:2048] [node 1] ICS07 socket disconnect 145 reason# EndOfStream
2025-11-28T14:52:47.701660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789329566743035:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.701727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.701761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789329566743035:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:47.702620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:47.702781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789329258658385:2048] [node 1] ICS01 socket: 153 reason# EndOfStream
2025-11-28T14:52:47.702816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789329258658295:2048] [node 1] ICP30 unregister session Session# [3:7577789329258658385:2048] VirtualId# [3:7577789329258658380:0]
2025-11-28T14:52:47.702837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789329258658385:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.702899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.702933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.704171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789329566742923:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16558) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:47.704196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789329566743035:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:47.705147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:47.705305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789332066184631:2048] [node 1] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-28T14:52:47.705330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789332066184631:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.705389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.705438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789332066184631:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:47.705474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789329566742923:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.705512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789329566743035:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.705567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789329566742923:2048] [node 1] ICP30 unregister session Session# [2:7577789329566743035:2048] VirtualId# [2:7577789329566743028:0]
2025-11-28T14:52:47.705656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.705681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789329566742923:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.705698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.706070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789332066184548:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.706108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789332066184631:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.706126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789332066184548:2048] [node 1] ICP30 unregister session Session# [4:7577789332066184631:2048] VirtualId# [4:7577789332066184627:0]
2025-11-28T14:52:47.706157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.706178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789332066184548:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.706186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.706172Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:47.706429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789330185917818:2048] [node 1] ICS01 socket: 174 reason# EndOfStream
2025-11-28T14:52:47.706533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789330185917723:2048] [node 1] ICP30 unregister session Session# [5:7577789330185917818:2048] VirtualId# [5:7577789330185917811:0]
2025-11-28T14:52:47.706578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789330185917818:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.706682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.706740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.706914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:47.707040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789331863591391:2048] [node 1] ICS01 socket: 183 reason# EndOfStream
2025-11-28T14:52:47.707071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789331863591297:2048] [node 1] ICP30 unregister session Session# [6:7577789331863591391:2048] VirtualId# [6:7577789331863591381:0]
2025-11-28T14:52:47.707089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789331863591391:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.707145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:47.707172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:47.715968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789329566742923:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:47.715962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789332066184548:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:47.751516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:47.751684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789329258658391:2049] [node 2] ICS01 socket: 162 reason# EndOfStream
2025-11-28T14:52:47.751722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789329258658296:2049] [node 2] ICP30 unregister session Session# [3:7577789329258658391:2049] VirtualId# [3:7577789329258658381:0]
2025-11-28T14:52:47.751744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789329258658391:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.751805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:47.751840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:47.753538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:47.753693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789332066184647:2049] [node 2] ICS01 socket: 163 reason# EndOfStream
2025-11-28T14:52:47.753727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789332066184549:2049] [node 2] ICP30 unregister session Session# [4:7577789332066184647:2049] VirtualId# [4:7577789332066184635:0]
2025-11-28T14:52:47.753748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789332066184647:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.753804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:47.753836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:47.754364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:47.754625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789330185917824:2049] [node 2] ICS07 socket disconnect 191 reason# EndOfStream
2025-11-28T14:52:47.754668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789330185917824:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.754775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:47.754840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789330185917824:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:47.754945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:47.755105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789331863591396:2049] [node 2] ICS07 socket disconnect 126 reason# EndOfStream
2025-11-28T14:52:47.755127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789331863591396:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.755184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:47.755213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789331863591396:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:47.755697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789330185917724:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22660) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.755756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789330185917824:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.755795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789330185917724:2049] [node 2] ICP30 unregister session Session# [5:7577789330185917824:2049] VirtualId# [5:7577789330185917820:0]
2025-11-28T14:52:47.755914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:47.755926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789331863591298:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22660) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.755957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789330185917724:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.755980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.755959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789331863591396:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.755981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789331863591298:2049] [node 2] ICP30 unregister session Session# [6:7577789331863591396:2049] VirtualId# [6:7577789331863591393:0]
2025-11-28T14:52:47.756009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:47.756027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789331863591298:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.756035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.766163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789331863591298:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:47.766181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789330185917724:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:47.777915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:47.778099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789332066184643:2050] [node 3] ICS07 socket disconnect 158 reason# EndOfStream
2025-11-28T14:52:47.778124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789332066184643:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.778184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:47.778219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789332066184643:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:47.778787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789332066184550:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26788) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.778829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789332066184643:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.778848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789332066184550:2050] [node 3] ICP30 unregister session Session# [4:7577789332066184643:2050] VirtualId# [4:7577789332066184634:0]
2025-11-28T14:52:47.778829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:47.778878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:47.778904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789332066184550:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.778914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.778985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789330185917826:2050] [node 3] ICS07 socket disconnect 127 reason# EndOfStream
2025-11-28T14:52:47.779020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789330185917826:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.779107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:47.779150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789330185917826:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:47.779438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:47.779567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789331863591399:2050] [node 3] ICS01 socket: 137 reason# EndOfStream
2025-11-28T14:52:47.779598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789331863591299:2050] [node 3] ICP30 unregister session Session# [6:7577789331863591399:2050] VirtualId# [6:7577789331863591394:0]
2025-11-28T14:52:47.779618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789331863591399:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.779667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:47.779694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:47.779852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789330185917725:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26788) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.779901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789330185917826:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.779928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789330185917725:2050] [node 3] ICP30 unregister session Session# [5:7577789330185917826:2050] VirtualId# [5:7577789330185917821:0]
2025-11-28T14:52:47.779998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:47.780022Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789330185917725:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.780033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.788685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789332066184550:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:47.789701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789330185917725:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:47.810282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:47.810446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789330185917808:2108] [node 4] ICS07 socket disconnect 136 reason# EndOfStream
2025-11-28T14:52:47.810475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789330185917808:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.810564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:47.810616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789330185917808:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:47.811407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:47.811548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789330185917787:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4839) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.811565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789331863591387:2109] [node 4] ICS07 socket disconnect 157 reason# EndOfStream
2025-11-28T14:52:47.811594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789330185917808:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.811588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789331863591387:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.811625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789330185917787:2108] [node 4] ICP30 unregister session Session# [5:7577789330185917808:2108] VirtualId# [5:7577789330185917806:0]
2025-11-28T14:52:47.811644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:47.811672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789331863591387:2109] [node 4] ICS15 start handshake
2025-11-28T14:52:47.811686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:47.811718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789330185917787:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.811745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.812281Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789331863591361:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4839) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.812315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789331863591387:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.812341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789331863591361:2109] [node 4] ICP30 unregister session Session# [6:7577789331863591387:2109] VirtualId# [6:7577789331863591376:0]
2025-11-28T14:52:47.812441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:47.812463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789331863591361:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.812471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.822498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789331863591361:2109] [node 4] ICP33 wake up from error state
2025-11-28T14:52:47.831492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:47.831639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789331863591384:2107] [node 5] ICS07 socket disconnect 184 reason# EndOfStream
2025-11-28T14:52:47.831663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789331863591384:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:47.831726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:47.831760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789331863591384:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:47.832321Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789331863591360:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3884) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:47.832360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789331863591384:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:47.832404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789331863591360:2107] [node 5] ICP30 unregister session Session# [6:7577789331863591384:2107] VirtualId# [6:7577789331863591380:0]
2025-11-28T14:52:47.832484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:47.832518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789331863591360:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3884) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.832529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3884) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:47.841918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789331863591360:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:47.910848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789331708257785:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:47.910881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789331708257786:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:47.927730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789329745553880:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.927751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789329745553881:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:47.945656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789329794818045:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.945700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789329794818046:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:47.963101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789331756020614:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.963135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789331756020616:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:47.963152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789331756020615:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:47.981402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789333023698300:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.981445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789333023698301:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:47.981539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789333023698302:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:47.998664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789329448575165:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:47.998707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789329448575167:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:47.998745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789329448575166:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:48.074110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789334089785404:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:48.075858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789336003225144:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:48.076574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789336003225145:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:48.076656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789336050987974:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:48.076778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789334040521240:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:48.076849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789337318665668:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:48.077042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789334089785405:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:48.077121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789336050987975:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:48.077217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789334040521241:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:48.076945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789333743542524:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:48.077167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789336003225146:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:48.077211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789334089785406:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:48.077054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789333743542523:2104] [node 4] ICP01 ready to work
2025-11-28T14:52:48.077706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789334040521242:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:48.079988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789337318665671:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.089785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789336003225155:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6479
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.089911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789336050987986:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:48.090174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789336050987987:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:48.090147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789336003225156:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6479
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.090625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789334040521251:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1107
2025-11-28T14:52:48.090909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542539:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
2025-11-28T14:52:48.091687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789334040521252:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1107
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.092106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789336003225162:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6479
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.092377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789336003225163:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6479
2025-11-28T14:52:48.092489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789334040521251:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.092726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225155:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:48.092705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789334089785421:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29764
::ffff:127.1.0.0
2025-11-28T14:52:48.092797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789334040521257:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1107
2025-11-28T14:52:48.092890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225156:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.092779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789334089785420:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29764
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.093025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789334040521252:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.093050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789334040521258:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1107
2025-11-28T14:52:48.094106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789337318665683:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33149
2025-11-28T14:52:48.094322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789334089785422:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29764
2025-11-28T14:52:48.094114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789334040521260:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1107
2025-11-28T14:52:48.094522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789336003225164:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6479
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.094909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789336050987991:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:48.095027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789336050987986:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.095111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789336050987987:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.096105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789331756020616:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.095477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789334040521258:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.095985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789334040521257:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.096966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789334089785420:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.096974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333743542539:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.097119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789334089785412:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.097194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789333743542523:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.097145Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789334089785412:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.098366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789334089785421:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.098614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789336050987981:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.098684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789329794818045:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.098642Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789336050987981:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.098889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789334089785412:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:48.098633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225163:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.098643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225162:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.099116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225148:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.099486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789336050987991:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.099137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789336003225148:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.100239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789334040521260:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.099966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789334089785422:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.100666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789329745553880:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.100447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789331756020614:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.100452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789329794818046:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.100194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225164:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.100498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789334089785404:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.100831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789336050987981:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:48.100526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.100661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789334089785404:2107] [node 4] ICP22 created new session: [3:7577789334089785427:2107]
2025-11-28T14:52:48.100941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789336003225148:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.100952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225152:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.100883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789334089785427:2107] [node 4] ICS09 handshake done sender: [3:7577789334089785412:2112] self: [3:7577789334089785411:0] peer: [4:7577789336050987993:0] socket: 137 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.100975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789336003225152:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.101117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789336050987994:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:48.100927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789334089785427:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:48.101141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789337318665683:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.101555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789334040521246:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.100963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:48.101574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789333023698301:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.102073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542541:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
2025-11-28T14:52:48.101577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789334040521246:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.102755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789334040521249:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.102903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789336050987974:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.102813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789331708257786:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.102779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789334040521249:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.102969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.102852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789334040521246:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.102970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.103021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789336050987974:2109] [node 6] ICP22 created new session: [4:7577789336050987997:2109]
2025-11-28T14:52:48.103066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789331708257786:2049] [node 3] ICP22 created new session: [1:7577789336003225171:2049]
2025-11-28T14:52:48.103043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789336050987987:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.103437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333743542539:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:48.103082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789336050987997:2109] [node 6] ICS09 handshake done sender: [4:7577789336050987981:2113] self: [4:7577789336050987980:0] peer: [6:7577789333743542540:0] socket: 153 qp: -1
2025-11-28T14:52:48.103073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225150:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.103121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789336050987997:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:48.103469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.103984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789329745553881:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.103091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789336003225150:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.103988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789331756020616:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.104059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789333743542523:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.103103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789336003225171:2049] [node 3] ICS09 handshake done sender: [1:7577789336003225148:2110] self: [1:7577789336003225147:0] peer: [3:7577789334089785425:0] socket: 158 qp: -1
2025-11-28T14:52:48.104053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.104053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.104137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789331756020616:2050] [node 3] ICP22 created new session: [4:7577789336050987999:2050]
2025-11-28T14:52:48.104095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789329745553881:2049] [node 3] ICP22 created new session: [2:7577789334040521268:2049]
2025-11-28T14:52:48.104163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:48.103961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789336003225171:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:48.104418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789334040521268:2049] [node 3] ICS09 handshake done sender: [2:7577789334040521246:2112] self: [2:7577789334040521245:0] peer: [3:7577789334089785426:0] socket: 163 qp: -1
2025-11-28T14:52:48.104253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789336050987999:2050] [node 3] ICS09 handshake done sender: [4:7577789336050987987:2117] self: [4:7577789336050987993:0] peer: [3:7577789334089785411:0] socket: 145 qp: -1
2025-11-28T14:52:48.104002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.104284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789333743542523:2104] [node 4] ICP22 created new session: [6:7577789333743542542:2104]
2025-11-28T14:52:48.104439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789334040521268:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:48.104336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789336050987999:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:48.104425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333743542542:2104] [node 4] ICS09 handshake done sender: [6:7577789333743542539:2115] self: [6:7577789333743542540:0] peer: [4:7577789336050987980:0] socket: 156 qp: -1
2025-11-28T14:52:48.104459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:48.104463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333743542542:2104] [node 4] ICS10 traffic start
2025-11-28T14:52:48.104805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:48.104681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:48.105346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789334089785429:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29764
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.105736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789334040521249:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.105751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789337318665685:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33149
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.106078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789337318665686:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33149
2025-11-28T14:52:48.106120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789337318665687:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33149
2025-11-28T14:52:48.106224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789334040521241:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.106278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.106322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789334040521241:2108] [node 5] ICP22 created new session: [2:7577789334040521272:2108]
2025-11-28T14:52:48.106329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789336003225152:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:48.106368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789334040521272:2108] [node 5] ICS09 handshake done sender: [2:7577789334040521249:2113] self: [2:7577789334040521248:0] peer: [5:7577789337318665684:0] socket: 170 qp: -1
2025-11-28T14:52:48.106400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789334040521272:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:48.107150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.107820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789337318665683:2124] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.107826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789336003225150:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.107844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789336003225144:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.107878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-28T14:52:48.108114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789336003225144:2107] [node 4] ICP22 created new session: [1:7577789336003225176:2107]
::ffff:127.1.0.0
2025-11-28T14:52:48.108102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789329794818045:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20249) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789334089785421:2116] held: no
2025-11-28T14:52:48.108118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789333023698301:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.108226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:48.108230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789336003225176:2107] [node 4] ICS09 handshake done sender: [1:7577789336003225152:2112] self: [1:7577789336003225151:0] peer: [4:7577789336050987996:0] socket: 164 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.108260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789336003225176:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:48.108259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789333023698301:2049] [node 2] ICP22 created new session: [5:7577789337318665689:2049]
2025-11-28T14:52:48.108252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789334089785431:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29764
2025-11-28T14:52:48.108397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789337318665688:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33149
2025-11-28T14:52:48.108318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:48.108408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789337318665689:2049] [node 2] ICS09 handshake done sender: [5:7577789337318665683:2124] self: [5:7577789337318665684:0] peer: [2:7577789334040521248:0] socket: 171 qp: -1
2025-11-28T14:52:48.108461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789337318665689:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:48.108482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:48.108519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789337318665687:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.110023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789329794818046:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21252) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789334089785422:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.110326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789331708257785:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.110493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:48.110539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789331708257785:2048] [node 2] ICP22 created new session: [1:7577789336003225178:2048]
2025-11-28T14:52:48.110586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789336003225178:2048] [node 2] ICS09 handshake done sender: [1:7577789336003225150:2111] self: [1:7577789336003225149:0] peer: [2:7577789334040521265:0] socket: 160 qp: -1
2025-11-28T14:52:48.110551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789334040521271:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1107
2025-11-28T14:52:48.110625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789336003225178:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:48.110692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:48.110882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789334089785421:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:48.111761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789329794818045:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.111796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:48.111797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789334089785422:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.111830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789329794818045:2048] [node 1] ICP22 created new session: [3:7577789334089785432:2048]
2025-11-28T14:52:48.111865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789334089785432:2048] [node 1] ICS09 handshake done sender: [3:7577789334089785421:2116] self: [3:7577789334089785425:0] peer: [1:7577789336003225147:0] socket: 161 qp: -1
2025-11-28T14:52:48.111887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789334089785432:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:48.111907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.113253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789331756020614:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20249) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789336050987991:2118] held: no
2025-11-28T14:52:48.113403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542545:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
2025-11-28T14:52:48.113604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789334040521260:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.113809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789329794818046:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.114379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:48.114444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542546:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.114487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789329794818046:2049] [node 2] ICP22 created new session: [3:7577789334089785436:2049]
2025-11-28T14:52:48.114664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789329745553880:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20249) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789334040521260:2120] held: no
2025-11-28T14:52:48.114870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789334089785436:2049] [node 2] ICS09 handshake done sender: [3:7577789334089785422:2117] self: [3:7577789334089785426:0] peer: [2:7577789334040521245:0] socket: 168 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.114901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789334089785436:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:48.115101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:48.115030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789336050988003:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:48.115084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789336050988002:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:48.115341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789336050987991:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:48.115349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789337318665688:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.115481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789333023698300:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.115631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333743542546:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.115652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789329745553880:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.115642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789331756020614:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.115871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789329448575165:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.115689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:48.115848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:48.115877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789329745553880:2048] [node 1] ICP22 created new session: [2:7577789334040521275:2048]
2025-11-28T14:52:48.115881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225154:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.115909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789336003225154:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.116020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789337318665693:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33149
2025-11-28T14:52:48.115929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789334040521275:2048] [node 1] ICS09 handshake done sender: [2:7577789334040521260:2120] self: [2:7577789334040521265:0] peer: [1:7577789336003225149:0] socket: 169 qp: -1
2025-11-28T14:52:48.116043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789337318665685:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.116002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789334040521275:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:48.115892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789331756020614:2048] [node 1] ICP22 created new session: [4:7577789336050988005:2048]
2025-11-28T14:52:48.116160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789333023698302:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.116028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:48.116045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789336050988005:2048] [node 1] ICS09 handshake done sender: [4:7577789336050987991:2118] self: [4:7577789336050987996:0] peer: [1:7577789336003225151:0] socket: 166 qp: -1
2025-11-28T14:52:48.116076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789336050988005:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.116142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:48.116223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789336003225158:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.116257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789336003225158:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.116288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542548:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.117089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789334089785434:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29764
2025-11-28T14:52:48.117189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789336050988002:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.117231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789334089785414:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.117251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789334089785414:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.117460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789336003225154:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:48.117291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333743542545:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.117890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789329448575166:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.117970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789334089785434:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.118615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789336003225158:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:48.118751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789331756020615:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.118608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789337318665693:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.118810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789337318665668:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.118870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789334040521254:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.119466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789337318665696:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33149
2025-11-28T14:52:48.119380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789334089785414:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:48.119328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789336003225145:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.118898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789334040521254:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.119465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789336050987985:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.119488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789336003225146:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.120228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789334089785405:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.119483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789336050987985:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.119717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.119726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.120186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789334040521250:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.120739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.119983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789336003225145:2108] [node 5] ICP22 created new session: [1:7577789336003225184:2108]
2025-11-28T14:52:48.120209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789334040521250:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.120208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789336050987985:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:48.120193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789336003225146:2109] [node 6] ICP22 created new session: [1:7577789336003225183:2109]
2025-11-28T14:52:48.120900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789334089785405:2108] [node 5] ICP22 created new session: [3:7577789334089785440:2108]
2025-11-28T14:52:48.120890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542550:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
2025-11-28T14:52:48.120910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789337318665688:2128] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:48.120225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789336003225184:2108] [node 5] ICS09 handshake done sender: [1:7577789336003225154:2113] self: [1:7577789336003225153:0] peer: [5:7577789337318665694:0] socket: 189 qp: -1
2025-11-28T14:52:48.121073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789334040521254:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:48.120753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789336003225184:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:48.121059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789334089785440:2108] [node 5] ICS09 handshake done sender: [3:7577789334089785414:2113] self: [3:7577789334089785413:0] peer: [5:7577789337318665695:0] socket: 182 qp: -1
2025-11-28T14:52:48.121271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789334040521250:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:48.121085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789334089785440:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:48.120794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.121311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789336050987975:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.120824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789336003225183:2109] [node 6] ICS09 handshake done sender: [1:7577789336003225158:2116] self: [1:7577789336003225157:0] peer: [6:7577789333743542547:0] socket: 167 qp: -1
2025-11-28T14:52:48.121118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.120851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789336003225183:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:48.122503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789333023698300:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.121753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.122508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333743542548:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.122480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789336050987975:2108] [node 5] ICP22 created new session: [4:7577789336050988009:2108]
2025-11-28T14:52:48.122699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:48.120884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.122582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333743542546:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:48.122728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789333023698300:2048] [node 1] ICP22 created new session: [5:7577789337318665700:2048]
2025-11-28T14:52:48.122751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789336050988009:2108] [node 5] ICS09 handshake done sender: [4:7577789336050987985:2115] self: [4:7577789336050987984:0] peer: [5:7577789337318665697:0] socket: 162 qp: -1
2025-11-28T14:52:48.122859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789337318665700:2048] [node 1] ICS09 handshake done sender: [5:7577789337318665688:2128] self: [5:7577789337318665694:0] peer: [1:7577789336003225153:0] socket: 191 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.122897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789336050988009:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:48.122895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789337318665700:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:48.122918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:48.122889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789334040521242:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.122932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.122930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.123328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542551:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.122987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789334040521240:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.123129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789334040521242:2110] [node 6] ICP22 created new session: [2:7577789334040521279:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.123200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:48.123691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789329448575165:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.123692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789337318665698:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33149
2025-11-28T14:52:48.123735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:48.123767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789329448575165:2048] [node 1] ICP22 created new session: [6:7577789333743542552:2048]
2025-11-28T14:52:48.123205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789334040521279:2110] [node 6] ICS09 handshake done sender: [2:7577789334040521254:2117] self: [2:7577789334040521253:0] peer: [6:7577789333743542549:0] socket: 150 qp: -1
2025-11-28T14:52:48.123316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789334040521240:2107] [node 4] ICP22 created new session: [2:7577789334040521280:2107]
2025-11-28T14:52:48.123876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333743542552:2048] [node 1] ICS09 handshake done sender: [6:7577789333743542546:2118] self: [6:7577789333743542547:0] peer: [1:7577789336003225157:0] socket: 192 qp: -1
2025-11-28T14:52:48.123817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789336050988010:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5439
2025-11-28T14:52:48.123939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333743542552:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:48.123354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789334040521280:2107] [node 4] ICS09 handshake done sender: [2:7577789334040521250:2114] self: [2:7577789334040521247:0] peer: [4:7577789336050988007:0] socket: 143 qp: -1
2025-11-28T14:52:48.123967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:48.123340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789334040521279:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:48.123380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789334040521280:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:48.124150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789329448575167:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.123448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.123595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:48.124229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789336050988002:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.124430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789331756020615:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.124463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:48.124489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789331756020615:2049] [node 2] ICP22 created new session: [4:7577789336050988012:2049]
2025-11-28T14:52:48.124522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789336050988012:2049] [node 2] ICS09 handshake done sender: [4:7577789336050988002:2120] self: [4:7577789336050988007:0] peer: [2:7577789334040521247:0] socket: 133 qp: -1
2025-11-28T14:52:48.124591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789333023698302:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17524) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789337318665685:2125] held: no
2025-11-28T14:52:48.124539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789336050988012:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:48.124571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:48.125690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333743542545:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.125699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789334089785416:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.125723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789334089785416:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.128492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789334089785416:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:48.129365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789334089785406:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.129461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.129645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789337318665685:2125] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.129495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789334089785406:2109] [node 6] ICP22 created new session: [3:7577789334089785443:2109]
2025-11-28T14:52:48.130350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789334089785443:2109] [node 6] ICS09 handshake done sender: [3:7577789334089785416:2114] self: [3:7577789334089785415:0] peer: [6:7577789333743542554:0] socket: 194 qp: -1
2025-11-28T14:52:48.130493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789329448575166:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.130578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789337318665668:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16149) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789337318665693:2129] held: no
2025-11-28T14:52:48.130385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789334089785443:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:48.130559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.130536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:48.130768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789329448575166:2049] [node 2] ICP22 created new session: [6:7577789333743542555:2049]
2025-11-28T14:52:48.130926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333743542555:2049] [node 2] ICS09 handshake done sender: [6:7577789333743542545:2117] self: [6:7577789333743542549:0] peer: [2:7577789334040521253:0] socket: 152 qp: -1
2025-11-28T14:52:48.130962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333743542555:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:48.131044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.131324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789337318665699:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33149
2025-11-28T14:52:48.131512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789329448575166:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:48.131664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789333023698302:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.131724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.131754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789333023698302:2050] [node 3] ICP22 created new session: [5:7577789337318665703:2050]
2025-11-28T14:52:48.131788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789337318665703:2050] [node 3] ICS09 handshake done sender: [5:7577789337318665685:2125] self: [5:7577789337318665695:0] peer: [3:7577789334089785413:0] socket: 184 qp: -1
2025-11-28T14:52:48.131812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789337318665703:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:48.131835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:48.132554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789337318665693:2129] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:48.133019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789337318665668:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.133071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:48.133106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789337318665668:2108] [node 4] ICP22 created new session: [5:7577789337318665705:2108]
2025-11-28T14:52:48.133090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789329448575167:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17524) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789333743542548:2119] held: no
2025-11-28T14:52:48.133137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789337318665705:2108] [node 4] ICS09 handshake done sender: [5:7577789337318665693:2129] self: [5:7577789337318665697:0] peer: [4:7577789336050987984:0] socket: 193 qp: -1
2025-11-28T14:52:48.133162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789337318665705:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:48.133181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.133895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542557:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
2025-11-28T14:52:48.133956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542558:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
2025-11-28T14:52:48.134121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333743542548:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.134235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333743542558:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.134284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789329448575167:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.134305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.134328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789329448575167:2050] [node 3] ICP22 created new session: [6:7577789333743542559:2050]
2025-11-28T14:52:48.134354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333743542559:2050] [node 3] ICS09 handshake done sender: [6:7577789333743542548:2119] self: [6:7577789333743542554:0] peer: [3:7577789334089785415:0] socket: 195 qp: -1
2025-11-28T14:52:48.134374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333743542559:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:48.134392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:48.134393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789333743542524:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.134580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789337318665682:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.134600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789337318665682:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.135151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789337318665682:2123] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.135305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789337318665671:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.135320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333743542562:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54895
2025-11-28T14:52:48.135389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.135425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789337318665671:2109] [node 6] ICP22 created new session: [5:7577789337318665708:2109]
2025-11-28T14:52:48.135451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789337318665708:2109] [node 6] ICS09 handshake done sender: [5:7577789337318665682:2123] self: [5:7577789337318665681:0] peer: [6:7577789333743542560:0] socket: 134 qp: -1
2025-11-28T14:52:48.135475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789337318665708:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:48.135492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.135506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333743542558:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:48.135778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789333743542524:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.135840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.135892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789333743542524:2105] [node 5] ICP22 created new session: [6:7577789333743542563:2105]
2025-11-28T14:52:48.135943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333743542563:2105] [node 5] ICS09 handshake done sender: [6:7577789333743542558:2123] self: [6:7577789333743542560:0] peer: [5:7577789337318665681:0] socket: 142 qp: -1
2025-11-28T14:52:48.136009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333743542563:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:48.136045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.136896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789333743542524:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:48.176429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:48.176623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789334040521275:2048] [node 1] ICS01 socket: 169 reason# EndOfStream
2025-11-28T14:52:48.176661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789329745553880:2048] [node 1] ICP30 unregister session Session# [2:7577789334040521275:2048] VirtualId# [2:7577789334040521265:0]
2025-11-28T14:52:48.176685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789334040521275:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.176761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.176822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.177390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:48.177535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789334089785432:2048] [node 1] ICS01 socket: 161 reason# EndOfStream
2025-11-28T14:52:48.177566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789329794818045:2048] [node 1] ICP30 unregister session Session# [3:7577789334089785432:2048] VirtualId# [3:7577789334089785425:0]
2025-11-28T14:52:48.177589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789334089785432:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.177655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.177705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.179521Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:48.179766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789336050988005:2048] [node 1] ICS01 socket: 166 reason# EndOfStream
2025-11-28T14:52:48.179832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789331756020614:2048] [node 1] ICP30 unregister session Session# [4:7577789336050988005:2048] VirtualId# [4:7577789336050987996:0]
2025-11-28T14:52:48.179858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789336050988005:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.179923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.179959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.180295Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:48.180535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789337318665700:2048] [node 1] ICS07 socket disconnect 191 reason# EndOfStream
2025-11-28T14:52:48.180583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789337318665700:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.180650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.180691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789337318665700:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:48.181501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:48.181713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789333743542552:2048] [node 1] ICS07 socket disconnect 192 reason# EndOfStream
2025-11-28T14:52:48.181665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789333023698300:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20249) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.181752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333743542552:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.181775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789337318665700:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.181826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.181840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789333023698300:2048] [node 1] ICP30 unregister session Session# [5:7577789337318665700:2048] VirtualId# [5:7577789337318665694:0]
2025-11-28T14:52:48.181869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789333743542552:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:48.181896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.181947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789333023698300:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20249) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.182022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20249) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.182575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789329448575165:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20249) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.182701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333743542552:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.182741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789329448575165:2048] [node 1] ICP30 unregister session Session# [6:7577789333743542552:2048] VirtualId# [6:7577789333743542547:0]
2025-11-28T14:52:48.182832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.182868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789329448575165:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20249) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.182902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20249) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.192160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789333023698300:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:48.193173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789329448575165:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:48.219798Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:48.219964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789334089785436:2049] [node 2] ICS01 socket: 168 reason# EndOfStream
2025-11-28T14:52:48.219999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789329794818046:2049] [node 2] ICP30 unregister session Session# [3:7577789334089785436:2049] VirtualId# [3:7577789334089785426:0]
2025-11-28T14:52:48.220031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789334089785436:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.220115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:48.220170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:48.221753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:48.221882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789336050988012:2049] [node 2] ICS07 socket disconnect 133 reason# EndOfStream
2025-11-28T14:52:48.221899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789336050988012:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.221952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:48.221978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789336050988012:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:48.222444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:48.222569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789337318665689:2049] [node 2] ICS01 socket: 171 reason# EndOfStream
2025-11-28T14:52:48.222599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789333023698301:2049] [node 2] ICP30 unregister session Session# [5:7577789337318665689:2049] VirtualId# [5:7577789337318665684:0]
2025-11-28T14:52:48.222617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789337318665689:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.222642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789331756020615:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21252) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.222670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:48.222680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789336050988012:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.222711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:48.222723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789331756020615:2049] [node 2] ICP30 unregister session Session# [4:7577789336050988012:2049] VirtualId# [4:7577789336050988007:0]
2025-11-28T14:52:48.222797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:48.222816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789331756020615:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21252) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.222824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21252) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.223089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:48.223225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789333743542555:2049] [node 2] ICS07 socket disconnect 152 reason# EndOfStream
2025-11-28T14:52:48.223240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333743542555:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.223290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:48.223312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789333743542555:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:48.223802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789329448575166:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21252) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.223881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333743542555:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.223905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789329448575166:2049] [node 2] ICP30 unregister session Session# [6:7577789333743542555:2049] VirtualId# [6:7577789333743542549:0]
2025-11-28T14:52:48.224002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:48.224038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789329448575166:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21252) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.224059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21252) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.233111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789331756020615:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:48.234136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789329448575166:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:48.261255Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:48.261458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789336050987999:2050] [node 3] ICS07 socket disconnect 145 reason# EndOfStream
2025-11-28T14:52:48.261487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789336050987999:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.261542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:48.261577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789336050987999:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:48.262031Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:48.262172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789337318665703:2050] [node 3] ICS01 socket: 184 reason# EndOfStream
2025-11-28T14:52:48.262199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789333023698302:2050] [node 3] ICP30 unregister session Session# [5:7577789337318665703:2050] VirtualId# [5:7577789337318665695:0]
2025-11-28T14:52:48.262219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789337318665703:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.262275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:48.262291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789331756020616:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17524) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.262331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789336050987999:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.262331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:48.262362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789331756020616:2050] [node 3] ICP30 unregister session Session# [4:7577789336050987999:2050] VirtualId# [4:7577789336050987993:0]
2025-11-28T14:52:48.262479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:48.262518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789331756020616:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17524) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.262528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17524) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.262706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:48.262870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333743542559:2050] [node 3] ICS01 socket: 195 reason# EndOfStream
2025-11-28T14:52:48.262901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789329448575167:2050] [node 3] ICP30 unregister session Session# [6:7577789333743542559:2050] VirtualId# [6:7577789333743542554:0]
2025-11-28T14:52:48.262917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333743542559:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.262966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:48.263005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:48.271999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789331756020616:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:48.321673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:48.321850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789337318665705:2108] [node 4] ICS01 socket: 193 reason# EndOfStream
2025-11-28T14:52:48.321881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789337318665668:2108] [node 4] ICP30 unregister session Session# [5:7577789337318665705:2108] VirtualId# [5:7577789337318665697:0]
2025-11-28T14:52:48.321915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789337318665705:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.321978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:48.322007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:48.322491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:48.322624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789333743542542:2104] [node 4] ICS07 socket disconnect 156 reason# EndOfStream
2025-11-28T14:52:48.322644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333743542542:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.322700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:48.322737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789333743542542:2104] [node 4] ICS15 start handshake
2025-11-28T14:52:48.323231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789333743542523:2104] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16149) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.323316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333743542542:2104] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.323345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789333743542523:2104] [node 4] ICP30 unregister session Session# [6:7577789333743542542:2104] VirtualId# [6:7577789333743542540:0]
2025-11-28T14:52:48.323429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:48.323453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789333743542523:2104] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.323462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.333459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789333743542523:2104] [node 4] ICP33 wake up from error state
2025-11-28T14:52:48.361982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:48.362154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789333743542563:2105] [node 5] ICS07 socket disconnect 142 reason# EndOfStream
2025-11-28T14:52:48.362182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333743542563:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.362270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:48.362313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789333743542563:2105] [node 5] ICS15 start handshake
2025-11-28T14:52:48.362854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789333743542524:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32129) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.362925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333743542563:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.362960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789333743542524:2105] [node 5] ICP30 unregister session Session# [6:7577789333743542563:2105] VirtualId# [6:7577789333743542560:0]
2025-11-28T14:52:48.363047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:48.363098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789333743542524:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32129) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.363114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32129) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.373425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789333743542524:2105] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:48.432202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789334025776606:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:48.432210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789334025776607:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:48.453794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789337603908103:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:48.453834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789337603908104:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:48.476781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789335900152090:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:48.476791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789335900152089:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:48.497505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789337207749254:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:48.497557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789337207749255:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:48.497819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789337207749256:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:48.520243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789334299512853:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:48.520248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789334299512854:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:48.520263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789334299512852:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:48.542799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789333625973569:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:48.542826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789333625973570:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:48.542891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789333625973571:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:48.634335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789335900152152:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:48.636169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789335900152153:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:48.635449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789337207749318:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:48.636269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789334025776670:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:48.636427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789337603908166:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:48.636247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789334299512915:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:48.636760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789337207749319:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:48.636512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789334025776669:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:48.636798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789337603908167:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:48.636939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789337603908168:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:48.637263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789333625973631:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:48.637288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789334025776671:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:48.637308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789334299512916:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:48.637446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789335900152154:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:48.637421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789333625973632:2106] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.650147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789334025776678:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13375
2025-11-28T14:52:48.651532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776678:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.653031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789335900152168:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.653502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789335900152169:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
2025-11-28T14:52:48.653564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789334025776687:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13375
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.654385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789335900152170:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.655282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789337603908175:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11272
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.656040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973640:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
2025-11-28T14:52:48.655330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789337603908176:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11272
2025-11-28T14:52:48.655571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789334025776688:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13375
2025-11-28T14:52:48.656095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789337603908179:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11272
2025-11-28T14:52:48.656111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789337603908180:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11272
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.657761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789335900152168:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.656251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789337603908175:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.658322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333625973640:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.657755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789337603908176:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.659531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789333625973571:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.658671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789335900152089:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.658733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776673:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.658271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789337603908103:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.659530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973643:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
2025-11-28T14:52:48.658638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789337603908179:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.658760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789334025776673:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.659596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789335900152169:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.660603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776677:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:48.660633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789335900152171:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
2025-11-28T14:52:48.660629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789334025776677:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.660936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789337207749333:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-28T14:52:48.660934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789335900152160:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.660975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333625973643:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.661218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789337603908180:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.660990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776688:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.662568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789333625973569:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.660962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789335900152170:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.662583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789334025776690:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13375
2025-11-28T14:52:48.660971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789335900152160:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.663232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789334025776689:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13375
2025-11-28T14:52:48.661618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789335900152172:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
2025-11-28T14:52:48.663295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789337603908188:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11272
2025-11-28T14:52:48.663340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776687:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.663401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776675:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.663432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789334025776675:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.663502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789335900152171:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.663457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789334025776673:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.663889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789337603908190:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11272
2025-11-28T14:52:48.663922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776689:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.663695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789335900152160:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:48.664303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789335900152090:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.664672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789334025776677:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.664329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789337207749333:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.666169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789337603908174:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.666280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789335900152152:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.666009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789337207749255:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.666209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789337603908174:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.666327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.667750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789335900152152:2109] [node 6] ICP22 created new session: [3:7577789335900152179:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.666757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789334025776606:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.667835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789335900152179:2109] [node 6] ICS09 handshake done sender: [3:7577789335900152160:2112] self: [3:7577789335900152156:0] peer: [6:7577789333625973649:0] socket: 140 qp: -1
2025-11-28T14:52:48.666820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:48.668354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789335900152179:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:48.669046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789337603908174:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.669552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973651:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
2025-11-28T14:52:48.666966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789334025776606:2048] [node 2] ICP22 created new session: [1:7577789334025776697:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.669642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789337603908171:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.668419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.669017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789335900152172:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.667044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789334025776607:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.669662Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789337603908171:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:52:48.671164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789335900152178:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.667618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.670901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789337603908188:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.671859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333625973640:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.667793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789334025776607:2049] [node 3] ICP22 created new session: [1:7577789334025776698:2049]
2025-11-28T14:52:48.673236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973652:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
2025-11-28T14:52:48.673677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789335900152182:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
2025-11-28T14:52:48.670940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789337603908104:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.667813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789334025776697:2048] [node 2] ICS09 handshake done sender: [1:7577789334025776673:2110] self: [1:7577789334025776672:0] peer: [2:7577789337603908187:0] socket: 145 qp: -1
2025-11-28T14:52:48.674004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789337207749335:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-28T14:52:48.671001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789337603908176:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:48.667858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789334025776698:2049] [node 3] ICS09 handshake done sender: [1:7577789334025776677:2112] self: [1:7577789334025776676:0] peer: [3:7577789335900152173:0] socket: 148 qp: -1
2025-11-28T14:52:48.673426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789333625973571:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.673779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789335900152168:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:48.673469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.671074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.674901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789335900152089:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.674833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789337207749335:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.667861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789334025776697:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:48.675043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:48.671265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789337603908104:2049] [node 3] ICP22 created new session: [2:7577789337603908193:2049]
2025-11-28T14:52:48.673519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789333625973571:2050] [node 3] ICP22 created new session: [6:7577789333625973653:2050]
2025-11-28T14:52:48.676096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789335900152171:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.668403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:48.676119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789337207749256:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.671818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789337603908193:2049] [node 3] ICS09 handshake done sender: [2:7577789337603908174:2112] self: [2:7577789337603908173:0] peer: [3:7577789335900152177:0] socket: 167 qp: -1
2025-11-28T14:52:48.672163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789337603908193:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:48.673745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333625973653:2050] [node 3] ICS09 handshake done sender: [6:7577789333625973640:2113] self: [6:7577789333625973649:0] peer: [3:7577789335900152156:0] socket: 143 qp: -1
2025-11-28T14:52:48.676358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789335900152089:2048] [node 1] ICP22 created new session: [3:7577789335900152183:2048]
2025-11-28T14:52:48.672630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:48.676409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789335900152183:2048] [node 1] ICS09 handshake done sender: [3:7577789335900152168:2115] self: [3:7577789335900152173:0] peer: [1:7577789334025776676:0] socket: 149 qp: -1
2025-11-28T14:52:48.668578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789334025776698:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:48.673767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333625973653:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.675026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789337603908103:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16180) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789337603908176:2114] held: no
2025-11-28T14:52:48.676442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789335900152183:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:48.668622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:48.676475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:48.673964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:48.672973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789334025776675:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:48.676419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333625973643:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:48.677267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776690:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.677895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789334025776670:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.677996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789337207749337:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-28T14:52:48.677569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789333625973569:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.677941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.677616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:48.678124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789337603908171:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:48.678005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789334025776670:2109] [node 6] ICP22 created new session: [1:7577789334025776703:2109]
2025-11-28T14:52:48.677837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789333625973569:2048] [node 1] ICP22 created new session: [6:7577789333625973656:2048]
2025-11-28T14:52:48.678636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789337603908103:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.678079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789334025776703:2109] [node 6] ICS09 handshake done sender: [1:7577789334025776675:2111] self: [1:7577789334025776674:0] peer: [6:7577789333625973650:0] socket: 147 qp: -1
2025-11-28T14:52:48.678000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333625973656:2048] [node 1] ICS09 handshake done sender: [6:7577789333625973643:2115] self: [6:7577789333625973650:0] peer: [1:7577789334025776674:0] socket: 165 qp: -1
2025-11-28T14:52:48.681317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789335900152090:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.678674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:48.678133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789334025776703:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:48.681409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789337207749333:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.678037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333625973656:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:48.681364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.680144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789337603908103:2048] [node 1] ICP22 created new session: [2:7577789337603908198:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.678319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.682823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789337207749255:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.681380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789335900152166:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.678071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:48.685061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789334299512930:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11829
2025-11-28T14:52:48.680983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789337603908198:2048] [node 1] ICS09 handshake done sender: [2:7577789337603908176:2114] self: [2:7577789337603908187:0] peer: [1:7577789334025776672:0] socket: 146 qp: -1
2025-11-28T14:52:48.684458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:48.681399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789335900152166:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.681584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789337603908198:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:48.685283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789334299512931:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11829
2025-11-28T14:52:48.684843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789337207749255:2049] [node 2] ICP22 created new session: [4:7577789337207749338:2049]
2025-11-28T14:52:48.682295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789335900152090:2049] [node 2] ICP22 created new session: [3:7577789335900152185:2049]
2025-11-28T14:52:48.681621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:48.682927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789337603908166:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.682704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789335900152185:2049] [node 2] ICS09 handshake done sender: [3:7577789335900152171:2118] self: [3:7577789335900152177:0] peer: [2:7577789337603908173:0] socket: 169 qp: -1
2025-11-28T14:52:48.685076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789337207749338:2049] [node 2] ICS09 handshake done sender: [4:7577789337207749333:2116] self: [4:7577789337207749334:0] peer: [2:7577789337603908170:0] socket: 176 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.683169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:48.687642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789334299512930:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.683151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789335900152185:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:48.684763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789337603908166:2107] [node 4] ICP22 created new session: [2:7577789337603908201:2107]
2025-11-28T14:52:48.687673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789334299512932:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11829
2025-11-28T14:52:48.683182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:48.685329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789337207749338:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:48.685411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789337603908201:2107] [node 4] ICS09 handshake done sender: [2:7577789337603908171:2110] self: [2:7577789337603908170:0] peer: [4:7577789337207749334:0] socket: 170 qp: -1
2025-11-28T14:52:48.685361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:48.685441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789337603908201:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:48.687760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789337207749339:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-28T14:52:48.685484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:48.690415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789335900152152:2109] [node 6] ICP18 handshake (actor [3:7577789335900152172:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.691188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789334025776670:2109] [node 6] ICP18 handshake (actor [1:7577789334025776690:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.691270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973658:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
2025-11-28T14:52:48.691376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973659:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.691859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789337207749342:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-28T14:52:48.692113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789337207749343:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-28T14:52:48.694233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789335900152090:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:48.694395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333625973659:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.694613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789333625973632:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.694620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789335900152166:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:48.694646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789334299512931:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.694652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789337207749342:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.694858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333625973658:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.694948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789334299512853:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.695156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789335900152153:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.695163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789334299512934:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11829
2025-11-28T14:52:48.695323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789337207749330:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.695332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789333625973570:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.695209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:48.695353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789337207749330:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.695330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789337603908182:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.695335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789335900152153:2107] [node 4] ICP22 created new session: [3:7577789335900152189:2107]
2025-11-28T14:52:48.695433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789337207749335:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.695367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789337603908182:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.695878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789337603908178:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.695377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789335900152189:2107] [node 4] ICS09 handshake done sender: [3:7577789335900152166:2113] self: [3:7577789335900152164:0] peer: [4:7577789337207749336:0] socket: 188 qp: -1
2025-11-28T14:52:48.695404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789335900152189:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:48.695901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789337603908178:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.695494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:48.697333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789334299512932:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.697428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789337207749339:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.697493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789337207749256:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.697541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.697588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789337207749256:2050] [node 3] ICP22 created new session: [4:7577789337207749346:2050]
2025-11-28T14:52:48.697655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789334299512852:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.697700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789337207749346:2050] [node 3] ICS09 handshake done sender: [4:7577789337207749335:2117] self: [4:7577789337207749336:0] peer: [3:7577789335900152164:0] socket: 190 qp: -1
2025-11-28T14:52:48.697841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789337603908178:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:48.697750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789337207749346:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:48.698481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789337603908182:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:48.697772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.698882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776683:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.698960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789337603908168:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.699463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789337207749330:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:48.698927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789334025776683:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.699611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.700741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973663:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
2025-11-28T14:52:48.700789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789337207749254:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.700750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789337603908168:2108] [node 5] ICP22 created new session: [2:7577789337603908205:2108]
2025-11-28T14:52:48.700848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789334299512937:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11829
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.700961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789337603908167:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.701168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789334025776681:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.701754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789337207749318:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.701008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789337603908205:2108] [node 5] ICS09 handshake done sender: [2:7577789337603908182:2118] self: [2:7577789337603908181:0] peer: [5:7577789334299512935:0] socket: 168 qp: -1
2025-11-28T14:52:48.701803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333625973658:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.701197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789334025776681:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.701817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.702377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973664:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
2025-11-28T14:52:48.701246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789334025776683:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:48.701033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789337603908205:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:48.702400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789337207749318:2109] [node 6] ICP22 created new session: [4:7577789337207749350:2109]
2025-11-28T14:52:48.701073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.702336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789334025776671:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.703012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789333625973570:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.703438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:48.701808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.703097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789337207749350:2109] [node 6] ICS09 handshake done sender: [4:7577789337207749330:2114] self: [4:7577789337207749329:0] peer: [6:7577789333625973661:0] socket: 158 qp: -1
2025-11-28T14:52:48.702557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789334025776681:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:48.702228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789337603908167:2109] [node 6] ICP22 created new session: [2:7577789337603908207:2109]
2025-11-28T14:52:48.703184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789337207749350:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:48.704196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789333625973570:2049] [node 2] ICP22 created new session: [6:7577789333625973665:2049]
2025-11-28T14:52:48.702419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789337603908207:2109] [node 6] ICS09 handshake done sender: [2:7577789337603908178:2115] self: [2:7577789337603908177:0] peer: [6:7577789333625973662:0] socket: 194 qp: -1
2025-11-28T14:52:48.705680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333625973659:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:48.703160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.706241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789334299512931:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:48.702727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789337603908207:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:48.703397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.703704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789334025776671:2108] [node 5] ICP22 created new session: [1:7577789334025776707:2108]
2025-11-28T14:52:48.707574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789334299512853:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2092) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789334299512931:2116] held: no
2025-11-28T14:52:48.705785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333625973665:2049] [node 2] ICS09 handshake done sender: [6:7577789333625973658:2119] self: [6:7577789333625973662:0] peer: [2:7577789337603908177:0] socket: 152 qp: -1
2025-11-28T14:52:48.703102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.705829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333625973665:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:48.703885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789334025776707:2108] [node 5] ICS09 handshake done sender: [1:7577789334025776683:2115] self: [1:7577789334025776679:0] peer: [5:7577789334299512936:0] socket: 139 qp: -1
2025-11-28T14:52:48.707745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789334299512934:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.706259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:48.704083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789334025776707:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:48.704121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.707859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789334299512852:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16180) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789334299512932:2117] held: no
2025-11-28T14:52:48.704138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789334025776669:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.706268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789333625973632:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789333625973659:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.705298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:48.705779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789334025776669:2107] [node 4] ICP22 created new session: [1:7577789334025776709:2107]
2025-11-28T14:52:48.705922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789334025776709:2107] [node 4] ICS09 handshake done sender: [1:7577789334025776681:2114] self: [1:7577789334025776680:0] peer: [4:7577789337207749348:0] socket: 138 qp: -1
2025-11-28T14:52:48.709010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789337207749352:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-28T14:52:48.705956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789334025776709:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:48.706227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
::ffff:127.1.0.0
2025-11-28T14:52:48.709479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789337207749254:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16180) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789337207749339:2119] held: no
2025-11-28T14:52:48.709332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789334299512938:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11829
2025-11-28T14:52:48.709343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789333625973632:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.709396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:48.709617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789334299512853:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.709446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789333625973632:2106] [node 4] ICP22 created new session: [6:7577789333625973667:2106]
No RDMA context found for GID: 2025-11-28T14:52:48.709619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333625973667:2106] [node 4] ICS09 handshake done sender: [6:7577789333625973659:2120] self: [6:7577789333625973661:0] peer: [4:7577789337207749329:0] socket: 195 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:48.709936Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789337207749256:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:48.709807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:48.709691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333625973667:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:48.709937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:48.709864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789334299512853:2049] [node 2] ICP22 created new session: [5:7577789334299512940:2049]
2025-11-28T14:52:48.709923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789334299512940:2049] [node 2] ICS09 handshake done sender: [5:7577789334299512931:2116] self: [5:7577789334299512935:0] peer: [2:7577789337603908181:0] socket: 192 qp: -1
2025-11-28T14:52:48.709993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789334299512940:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:48.710094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789334299512939:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11829
2025-11-28T14:52:48.710100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:48.710540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789334299512932:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:48.710602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789334299512854:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.710934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789334299512852:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.711020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:48.711063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789334299512852:2048] [node 1] ICP22 created new session: [5:7577789334299512945:2048]
2025-11-28T14:52:48.711060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789337207749339:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:48.711100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789334299512945:2048] [node 1] ICS09 handshake done sender: [5:7577789334299512932:2117] self: [5:7577789334299512936:0] peer: [1:7577789334025776679:0] socket: 171 qp: -1
2025-11-28T14:52:48.711129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789334299512945:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:48.711118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789335900152167:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.711153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:48.711150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789335900152167:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.711945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789337207749254:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:48.712143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
::ffff:127.1.0.0
2025-11-28T14:52:48.712254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789335900152167:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:48.712291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789337207749254:2048] [node 1] ICP22 created new session: [4:7577789337207749354:2048]
2025-11-28T14:52:48.712333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789337207749354:2048] [node 1] ICS09 handshake done sender: [4:7577789337207749339:2119] self: [4:7577789337207749348:0] peer: [1:7577789334025776680:0] socket: 193 qp: -1
2025-11-28T14:52:48.712338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973669:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
2025-11-28T14:52:48.712355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789337207749354:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:48.712397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:48.712448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789335900152154:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.712627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.712680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789335900152154:2108] [node 5] ICP22 created new session: [3:7577789335900152192:2108]
2025-11-28T14:52:48.712720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789335900152192:2108] [node 5] ICS09 handshake done sender: [3:7577789335900152167:2114] self: [3:7577789335900152165:0] peer: [5:7577789334299512944:0] socket: 175 qp: -1
2025-11-28T14:52:48.713456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789337207749353:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51246
2025-11-28T14:52:48.712959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789335900152192:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:48.712982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.714063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789337207749353:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.714888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789334299512939:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.715074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789334299512854:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24268) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789334299512934:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.715335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789334299512947:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11829
2025-11-28T14:52:48.715390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789334299512916:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.715620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789333625973669:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.715672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789334299512934:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:48.715746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789337207749332:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.715777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789337207749332:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.715808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789334299512854:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.715830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789333625973631:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:48.715905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:48.716213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789334299512854:2050] [node 3] ICP22 created new session: [5:7577789334299512949:2050]
2025-11-28T14:52:48.716218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789334299512929:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:48.716238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789334299512929:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:48.716395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789334299512949:2050] [node 3] ICS09 handshake done sender: [5:7577789334299512934:2118] self: [5:7577789334299512944:0] peer: [3:7577789335900152165:0] socket: 200 qp: -1
2025-11-28T14:52:48.717082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789334299512949:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:48.717366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789337207749332:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:48.717130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.717593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789337207749319:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.717615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789334299512951:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11829
2025-11-28T14:52:48.717677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.717730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789337207749319:2108] [node 5] ICP22 created new session: [4:7577789337207749358:2108]
2025-11-28T14:52:48.717769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789337207749358:2108] [node 5] ICS09 handshake done sender: [4:7577789337207749332:2115] self: [4:7577789337207749331:0] peer: [5:7577789334299512948:0] socket: 207 qp: -1
2025-11-28T14:52:48.717802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789337207749358:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:48.717847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.718168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789334299512939:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:48.718703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789334299512929:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:48.718699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789333625973671:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26379
2025-11-28T14:52:48.718929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789333625973669:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:48.719146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789333625973631:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.719191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:48.719312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789334299512916:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789334299512939:2121] held: no
2025-11-28T14:52:48.719304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789333625973631:2107] [node 5] ICP22 created new session: [6:7577789333625973672:2107]
2025-11-28T14:52:48.719417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789333625973672:2107] [node 5] ICS09 handshake done sender: [6:7577789333625973669:2123] self: [6:7577789333625973670:0] peer: [5:7577789334299512928:0] socket: 155 qp: -1
2025-11-28T14:52:48.719408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789334299512915:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:48.719459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789333625973672:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:48.719489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:48.719498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:48.719542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789334299512915:2109] [node 6] ICP22 created new session: [5:7577789334299512953:2109]
2025-11-28T14:52:48.719663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789334299512953:2109] [node 6] ICS09 handshake done sender: [5:7577789334299512929:2114] self: [5:7577789334299512928:0] peer: [6:7577789333625973670:0] socket: 154 qp: -1
2025-11-28T14:52:48.719723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789334299512953:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:48.720117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:48.720119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789334299512916:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:48.720302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:48.720329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789334299512916:2108] [node 4] ICP22 created new session: [5:7577789334299512955:2108]
2025-11-28T14:52:48.720391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789334299512955:2108] [node 4] ICS09 handshake done sender: [5:7577789334299512939:2121] self: [5:7577789334299512948:0] peer: [4:7577789337207749331:0] socket: 208 qp: -1
2025-11-28T14:52:48.720440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789334299512955:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:48.720465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:48.761734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:48.761936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789337603908198:2048] [node 1] ICS01 socket: 146 reason# EndOfStream
2025-11-28T14:52:48.761987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789337603908103:2048] [node 1] ICP30 unregister session Session# [2:7577789337603908198:2048] VirtualId# [2:7577789337603908187:0]
2025-11-28T14:52:48.762025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789337603908198:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.762150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.762233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.762750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:48.762986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789335900152183:2048] [node 1] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-28T14:52:48.763032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789335900152183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.763133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.763188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789335900152183:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:48.763983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789335900152089:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16180) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.764022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789335900152183:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.764042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789335900152089:2048] [node 1] ICP30 unregister session Session# [3:7577789335900152183:2048] VirtualId# [3:7577789335900152173:0]
2025-11-28T14:52:48.764691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.764719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789335900152089:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16180) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.764733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16180) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.764865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:48.765033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789337207749354:2048] [node 1] ICS01 socket: 193 reason# EndOfStream
2025-11-28T14:52:48.765083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789337207749254:2048] [node 1] ICP30 unregister session Session# [4:7577789337207749354:2048] VirtualId# [4:7577789337207749348:0]
2025-11-28T14:52:48.765104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789337207749354:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.765161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.765217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.765660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:48.765797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789334299512945:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-11-28T14:52:48.765828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789334299512852:2048] [node 1] ICP30 unregister session Session# [5:7577789334299512945:2048] VirtualId# [5:7577789334299512936:0]
2025-11-28T14:52:48.765845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789334299512945:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.765904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.765986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.766567Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:48.766728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333625973656:2048] [node 1] ICS01 socket: 165 reason# EndOfStream
2025-11-28T14:52:48.766763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789333625973569:2048] [node 1] ICP30 unregister session Session# [6:7577789333625973656:2048] VirtualId# [6:7577789333625973650:0]
2025-11-28T14:52:48.766780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333625973656:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.766832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:48.766867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:48.774847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789335900152089:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:48.813854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:48.814097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789335900152185:2049] [node 2] ICS07 socket disconnect 169 reason# EndOfStream
2025-11-28T14:52:48.814137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789335900152185:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.814263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:48.814324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789335900152185:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:48.816338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789335900152090:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2092) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:48.816394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789335900152185:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:48.817326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:48.817560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789337207749338:2049] [node 2] ICS07 socket disconnect 176 reason# EndOfStream
2025-11-28T14:52:48.817597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789337207749338:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.817702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:48.817738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789335900152090:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2092) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.817751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789337207749338:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:48.817846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789335900152185:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.817882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789335900152090:2049] [node 2] ICP30 unregister session Session# [3:7577789335900152185:2049] VirtualId# [3:7577789335900152177:0]
2025-11-28T14:52:48.818084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:48.818153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789335900152090:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2092) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.818174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2092) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.818686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789337207749255:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2092) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.818724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789337207749338:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.818805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789337207749255:2049] [node 2] ICP30 unregister session Session# [4:7577789337207749338:2049] VirtualId# [4:7577789337207749334:0]
2025-11-28T14:52:48.818758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:48.818909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:48.818907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789334299512940:2049] [node 2] ICS01 socket: 192 reason# EndOfStream
2025-11-28T14:52:48.818930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789337207749255:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2092) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.818935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789334299512853:2049] [node 2] ICP30 unregister session Session# [5:7577789334299512940:2049] VirtualId# [5:7577789334299512935:0]
2025-11-28T14:52:48.818937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2092) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.818962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789334299512940:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.819016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:48.819047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:48.819978Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:48.820108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333625973665:2049] [node 2] ICS01 socket: 152 reason# EndOfStream
2025-11-28T14:52:48.820139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789333625973570:2049] [node 2] ICP30 unregister session Session# [6:7577789333625973665:2049] VirtualId# [6:7577789333625973662:0]
2025-11-28T14:52:48.820159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333625973665:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.820198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:48.820224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:48.828068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789335900152090:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:48.829066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789337207749255:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:48.871795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:48.872019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789337207749346:2050] [node 3] ICS07 socket disconnect 190 reason# EndOfStream
2025-11-28T14:52:48.872056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789337207749346:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.872160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:48.872236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789337207749346:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:48.872844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:48.873005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789334299512949:2050] [node 3] ICS01 socket: 200 reason# EndOfStream
2025-11-28T14:52:48.872994Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789337207749256:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24268) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.873042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789337207749346:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.873067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789337207749256:2050] [node 3] ICP30 unregister session Session# [4:7577789337207749346:2050] VirtualId# [4:7577789337207749336:0]
2025-11-28T14:52:48.873144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:48.873046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789334299512854:2050] [node 3] ICP30 unregister session Session# [5:7577789334299512949:2050] VirtualId# [5:7577789334299512944:0]
2025-11-28T14:52:48.873190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789337207749256:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24268) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.873198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24268) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.873066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789334299512949:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.873125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:48.873152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:48.873949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:48.874061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789333625973653:2050] [node 3] ICS07 socket disconnect 143 reason# EndOfStream
2025-11-28T14:52:48.874081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333625973653:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.874118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:48.874151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789333625973653:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:48.874739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789333625973571:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24268) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.874769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333625973653:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.874827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789333625973571:2050] [node 3] ICP30 unregister session Session# [6:7577789333625973653:2050] VirtualId# [6:7577789333625973649:0]
2025-11-28T14:52:48.874897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:48.874914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789333625973571:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24268) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.874922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24268) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.883395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789337207749256:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:48.884452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789333625973571:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:48.935183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:48.935337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789334299512955:2108] [node 4] ICS01 socket: 208 reason# EndOfStream
2025-11-28T14:52:48.935369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789334299512916:2108] [node 4] ICP30 unregister session Session# [5:7577789334299512955:2108] VirtualId# [5:7577789334299512948:0]
2025-11-28T14:52:48.935391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789334299512955:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.935450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:48.935478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:48.935895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:48.936010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333625973667:2106] [node 4] ICS01 socket: 195 reason# EndOfStream
2025-11-28T14:52:48.936044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789333625973632:2106] [node 4] ICP30 unregister session Session# [6:7577789333625973667:2106] VirtualId# [6:7577789333625973661:0]
2025-11-28T14:52:48.936059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333625973667:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.936098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:48.936125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:48.993387Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:48.993543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789333625973672:2107] [node 5] ICS07 socket disconnect 155 reason# EndOfStream
2025-11-28T14:52:48.993572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789333625973672:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:48.993654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:48.993692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789333625973672:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:48.994323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789333625973631:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13614) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:48.994371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789333625973672:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:48.994432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789333625973631:2107] [node 5] ICP30 unregister session Session# [6:7577789333625973672:2107] VirtualId# [6:7577789333625973670:0]
2025-11-28T14:52:48.994472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:48.994501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789333625973631:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:48.994510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.004241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789333625973631:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:49.089531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789338974101703:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:49.089586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789338974101704:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:49.106094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789341124333153:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.106094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789341124333154:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:49.123605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789337979406125:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.123675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789337979406126:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:49.140908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338175844943:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.140946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338175844944:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:49.141039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338175844945:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:49.159275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339500268215:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.159305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339500268216:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:49.159377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339500268217:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:49.176439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338908493601:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.176473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338908493602:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:49.176704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338908493603:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:49.245500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789338974101766:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:49.251971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789337979406188:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:49.259689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789337979406189:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:49.245607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789338974101767:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:49.259707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338908493663:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:49.259706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789341124333216:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:49.259757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789337979406190:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:49.260209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789341124333217:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:49.260249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338175845006:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:49.260248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789341124333218:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:49.260241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789338974101768:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:49.260772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339500268278:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:49.260545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338908493665:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:49.260951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338175845007:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:49.260792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339500268279:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.271008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338175845016:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37441
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.272525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789338974101777:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31836
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.272956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789341124333230:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8185
No RDMA context found for GID: 2025-11-28T14:52:49.273004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789337979406202:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2148
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.273172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339500268288:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60238
2025-11-28T14:52:49.273360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338175845016:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.273350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789338974101778:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31836
2025-11-28T14:52:49.273625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339500268291:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60238
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.273754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789341124333232:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8185
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.275799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789337979406203:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2148
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.276163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789338175844944:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.276517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789337979406204:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2148
2025-11-28T14:52:49.276899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101777:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.276741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789337979406202:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.278636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339500268288:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.278035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338175845020:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37441
2025-11-28T14:52:49.278573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789341124333224:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.277585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789338974101782:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31836
2025-11-28T14:52:49.278579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789341124333234:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8185
2025-11-28T14:52:49.278031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789337979406206:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2148
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.278604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789341124333224:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.282506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338175845021:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37441
2025-11-28T14:52:49.282851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339500268291:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.279938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789337979406126:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.282893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101782:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.278994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789341124333230:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.280655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789337979406206:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.279962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789341124333235:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8185
2025-11-28T14:52:49.282890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789339500268217:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.283420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101778:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.282528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789341124333220:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.283852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789338974101783:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31836
2025-11-28T14:52:49.282558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789341124333220:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.283595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789337979406204:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.284699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338175845020:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.284400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789337979406203:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.285241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789338175844943:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.283455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789341124333235:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.284076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789338974101784:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31836
2025-11-28T14:52:49.283533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789341124333232:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.284408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789337979406195:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.284695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101783:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.284268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789341124333224:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:49.285507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338175845021:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.284271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789341124333220:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:49.284444Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789337979406195:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.285507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101784:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.285291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789337979406125:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.285593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101775:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.284272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789341124333234:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.290371Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789337979406195:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:49.284392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789341124333153:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.285614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789338974101775:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.286965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789341124333216:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.286724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101772:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.291112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338175845025:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37441
2025-11-28T14:52:49.289184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:49.286754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789338974101772:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:52:49.289188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789341124333154:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.286767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101771:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.291382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789337979406188:2108] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:49.290354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789341124333216:2107] [node 4] ICP22 created new session: [2:7577789341124333242:2107]
2025-11-28T14:52:49.291530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789337979406211:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2148
2025-11-28T14:52:49.286790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789338974101771:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.289406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789338974101772:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:49.290646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.291593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:49.291659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789337979406188:2108] [node 5] ICP22 created new session: [3:7577789337979406215:2108]
2025-11-28T14:52:49.290692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789341124333154:2049] [node 3] ICP22 created new session: [2:7577789341124333243:2049]
2025-11-28T14:52:49.291390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789338974101775:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:49.292257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789337979406215:2108] [node 5] ICS09 handshake done sender: [3:7577789337979406195:2111] self: [3:7577789337979406193:0] peer: [5:7577789339500268295:0] socket: 152 qp: -1
2025-11-28T14:52:49.292131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789338175844944:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63775) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789338175845016:2114] held: no
2025-11-28T14:52:49.292147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789338974101771:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:49.293343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789337979406215:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:49.293006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338175845028:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37441
2025-11-28T14:52:49.290765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789341124333242:2107] [node 4] ICS09 handshake done sender: [2:7577789341124333224:2112] self: [2:7577789341124333223:0] peer: [4:7577789338175845022:0] socket: 146 qp: -1
2025-11-28T14:52:49.292157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789338974101767:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.293400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:49.290787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789341124333243:2049] [node 3] ICS09 handshake done sender: [2:7577789341124333220:2110] self: [2:7577789341124333219:0] peer: [3:7577789337979406208:0] socket: 144 qp: -1
2025-11-28T14:52:49.293690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339500268297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60238
2025-11-28T14:52:49.293040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:49.291057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789341124333243:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:49.293706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789337979406126:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63775) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789337979406202:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
2025-11-28T14:52:49.291061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789341124333242:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:49.294866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338175845020:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.293348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789338974101767:2107] [node 4] ICP22 created new session: [1:7577789338974101795:2107]
2025-11-28T14:52:49.293738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789338974101795:2107] [node 4] ICS09 handshake done sender: [1:7577789338974101772:2111] self: [1:7577789338974101770:0] peer: [4:7577789338175845026:0] socket: 167 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.291094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.295819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493679:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
2025-11-28T14:52:49.293919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789338974101795:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:49.294011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789338974101703:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.291096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.295834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789338175844943:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.295818Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789339500268217:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789339500268288:2114] held: no
2025-11-28T14:52:49.294292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789338974101704:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.297911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338908493679:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.296262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.297847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789337979406214:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2148
2025-11-28T14:52:49.298117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789341124333248:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8185
2025-11-28T14:52:49.296213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339500268288:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:49.296300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338175844943:2048] [node 1] ICP22 created new session: [4:7577789338175845030:2048]
2025-11-28T14:52:49.294298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.298853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493680:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
2025-11-28T14:52:49.299098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789337979406202:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:49.296720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339500268298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60238
2025-11-28T14:52:49.298996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338908493602:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.294485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789338974101703:2048] [node 2] ICP22 created new session: [1:7577789338974101797:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.300241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789341124333226:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.297867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338175845030:2048] [node 1] ICS09 handshake done sender: [4:7577789338175845020:2115] self: [4:7577789338175845026:0] peer: [1:7577789338974101770:0] socket: 172 qp: -1
2025-11-28T14:52:49.299100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789337979406203:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.298892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339500268298:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.298130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338175845030:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.302620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493682:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
2025-11-28T14:52:49.300274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789341124333226:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.294286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.302714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789337979406125:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.299769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789339500268216:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.298177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338175845016:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:49.304179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789337979406126:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.304438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338908493680:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.303771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789339500268217:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.304370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789341124333233:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.294645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789338974101797:2048] [node 2] ICS09 handshake done sender: [1:7577789338974101775:2112] self: [1:7577789338974101774:0] peer: [2:7577789341124333241:0] socket: 166 qp: -1
2025-11-28T14:52:49.298618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:49.300128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789338175844944:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.294815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.304226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.300286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.294870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789338974101797:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:49.304403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789341124333233:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.304273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789337979406126:2049] [node 2] ICP22 created new session: [3:7577789337979406219:2049]
2025-11-28T14:52:49.303795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.295487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789338974101704:2049] [node 3] ICP22 created new session: [1:7577789338974101798:2049]
2025-11-28T14:52:49.304572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789341124333234:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.300349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338175844944:2049] [node 2] ICP22 created new session: [4:7577789338175845033:2049]
2025-11-28T14:52:49.304274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.295588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.305761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789341124333153:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789341124333234:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.305991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338908493665:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.303228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338175845033:2049] [node 2] ICS09 handshake done sender: [4:7577789338175845016:2114] self: [4:7577789338175845022:0] peer: [2:7577789341124333223:0] socket: 151 qp: -1
2025-11-28T14:52:49.295603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789338974101798:2049] [node 3] ICS09 handshake done sender: [1:7577789338974101771:2110] self: [1:7577789338974101769:0] peer: [3:7577789337979406212:0] socket: 160 qp: -1
2025-11-28T14:52:49.304410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789337979406125:2048] [node 1] ICP22 created new session: [3:7577789337979406220:2048]
2025-11-28T14:52:49.304158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339500268217:2050] [node 3] ICP22 created new session: [5:7577789339500268301:2050]
2025-11-28T14:52:49.304785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339500268301:2050] [node 3] ICS09 handshake done sender: [5:7577789339500268288:2114] self: [5:7577789339500268295:0] peer: [3:7577789337979406193:0] socket: 161 qp: -1
2025-11-28T14:52:49.304798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789337979406220:2048] [node 1] ICS09 handshake done sender: [3:7577789337979406203:2115] self: [3:7577789337979406212:0] peer: [1:7577789338974101769:0] socket: 163 qp: -1
2025-11-28T14:52:49.303269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338175845033:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:49.295835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789338974101798:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:49.303303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.305274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339500268301:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:49.304832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789337979406219:2049] [node 2] ICS09 handshake done sender: [3:7577789337979406202:2114] self: [3:7577789337979406208:0] peer: [2:7577789341124333219:0] socket: 149 qp: -1
2025-11-28T14:52:49.295858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:49.304322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338175845032:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37441
2025-11-28T14:52:49.305524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.305392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789337979406220:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.305964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338175845015:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.308094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339500268303:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60238
2025-11-28T14:52:49.305980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789338175845015:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.305417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789337979406219:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:49.309707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339500268303:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.305427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:49.305822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.309332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338175845032:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.309935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339500268304:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60238
2025-11-28T14:52:49.309557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338175845015:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:49.309949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789341124333153:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.309923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493684:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.309583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789338175844945:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.310003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.310117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338908493682:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.310477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789337979406198:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.310316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789339500268278:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.310119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789338175844943:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:49.310119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789341124333153:2048] [node 1] ICP22 created new session: [2:7577789341124333252:2048]
2025-11-28T14:52:49.310512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789337979406198:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.310282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789338175845006:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.310480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493685:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
2025-11-28T14:52:49.312054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789337979406207:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.310511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:49.310529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338908493603:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.310386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789341124333154:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(18790288ns) {HandleHandshakeStatus:329(16973458ns) {DropIncomingHandshake:442(11913ns) DropOutgoingHandshake:457(265ns) SwitchToState:179(148ns) ProcessPendingSessionEvents:495(14720772ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14636673ns max# 1811122ns}}}
2025-11-28T14:52:49.312089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789337979406207:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.311216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338175845006:2109] [node 6] ICP22 created new session: [4:7577789338175845038:2109]
2025-11-28T14:52:49.312031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493686:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
2025-11-28T14:52:49.310642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789341124333252:2048] [node 1] ICS09 handshake done sender: [2:7577789341124333234:2117] self: [2:7577789341124333241:0] peer: [1:7577789338974101774:0] socket: 176 qp: -1
2025-11-28T14:52:49.312646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789337979406198:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:49.311439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338175845024:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.313514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338908493680:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:49.311216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789341124333252:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.313544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789337979406207:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:49.314253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339500268304:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.311463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789338175845024:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.311291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789341124333226:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:49.314167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789337979406189:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.314552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338908493684:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.314475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339500268306:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60238
2025-11-28T14:52:49.314562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338908493679:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:49.312595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338175845038:2109] [node 6] ICS09 handshake done sender: [4:7577789338175845015:2113] self: [4:7577789338175845011:0] peer: [6:7577789338908493683:0] socket: 198 qp: -1
2025-11-28T14:52:49.314724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789339500268215:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.314217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:49.311460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:49.312619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338175845038:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:49.315368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339500268298:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.314590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338908493665:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.315866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101776:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.314645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789337979406189:2107] [node 4] ICP22 created new session: [3:7577789337979406225:2107]
2025-11-28T14:52:49.311611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789341124333249:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8185
2025-11-28T14:52:49.315372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:49.315910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789338974101776:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.316382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789339500268216:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.312865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.313398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789341124333233:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:49.315012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789337979406188:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18362122ns) {HandleHandshakeStatus:329(18360106ns) {DropIncomingHandshake:442(13035ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(312ns) ProcessPendingSessionEvents:495(14945388ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14813716ns max# 967765ns}}}
2025-11-28T14:52:49.315487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338908493601:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.312996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338175845024:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:49.315194Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789341124333216:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21929880ns) {HandleHandshakeStatus:329(21927673ns) {DropIncomingHandshake:442(18334ns) DropOutgoingHandshake:457(2609ns) SwitchToState:179(241ns) ProcessPendingSessionEvents:495(16445462ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16253892ns max# 1505513ns}}}
::ffff:127.1.0.0
2025-11-28T14:52:49.315073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789337979406190:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.314452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789338175845007:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.318562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789338974101767:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17764112ns) {HandleHandshakeStatus:329(17762194ns) {DropIncomingHandshake:442(10345ns) DropOutgoingHandshake:457(264ns) SwitchToState:179(218ns) ProcessPendingSessionEvents:495(14368369ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14296389ns max# 1240910ns}}}
2025-11-28T14:52:49.314638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:49.315357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789341124333249:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.317530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.314675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338175845007:2108] [node 5] ICP22 created new session: [4:7577789338175845042:2108]
2025-11-28T14:52:49.315528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338908493665:2108] [node 4] ICP22 created new session: [6:7577789338908493689:2108]
2025-11-28T14:52:49.318568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789338974101786:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.315495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789337979406225:2107] [node 4] ICS09 handshake done sender: [3:7577789337979406198:2113] self: [3:7577789337979406197:0] peer: [4:7577789338175845037:0] socket: 173 qp: -1
2025-11-28T14:52:49.315524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338175845042:2108] [node 5] ICS09 handshake done sender: [4:7577789338175845024:2117] self: [4:7577789338175845023:0] peer: [5:7577789339500268305:0] socket: 145 qp: -1
2025-11-28T14:52:49.317561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339500268216:2049] [node 2] ICP22 created new session: [5:7577789339500268309:2049]
2025-11-28T14:52:49.315390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789341124333217:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.315627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338908493602:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.315527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:49.318590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789338974101786:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.315534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789337979406225:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:49.318513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339500268309:2049] [node 2] ICS09 handshake done sender: [5:7577789339500268298:2118] self: [5:7577789339500268300:0] peer: [2:7577789341124333231:0] socket: 197 qp: -1
2025-11-28T14:52:49.320150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789338974101776:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:49.315675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.315543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:49.315847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493688:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
2025-11-28T14:52:49.320820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789338974101766:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.318804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339500268307:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60238
2025-11-28T14:52:49.315891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338175845042:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:49.315912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789337979406190:2109] [node 6] ICP22 created new session: [3:7577789337979406226:2109]
2025-11-28T14:52:49.315694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789341124333217:2109] [node 6] ICP22 created new session: [2:7577789341124333254:2109]
2025-11-28T14:52:49.316357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338908493689:2108] [node 4] ICS09 handshake done sender: [6:7577789338908493680:2116] self: [6:7577789338908493683:0] peer: [4:7577789338175845011:0] socket: 199 qp: -1
2025-11-28T14:52:49.315923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:49.318866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339500268309:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:49.320853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:49.316471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789337979406226:2109] [node 6] ICS09 handshake done sender: [3:7577789337979406207:2118] self: [3:7577789337979406205:0] peer: [6:7577789338908493687:0] socket: 177 qp: -1
2025-11-28T14:52:49.315791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789341124333218:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.317491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789341124333254:2109] [node 6] ICS09 handshake done sender: [2:7577789341124333226:2113] self: [2:7577789341124333225:0] peer: [6:7577789338908493681:0] socket: 148 qp: -1
2025-11-28T14:52:49.319662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.316753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338908493689:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:49.321956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789338974101766:2108] [node 5] ICP22 created new session: [1:7577789338974101804:2108]
2025-11-28T14:52:49.320631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789339500268278:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16786) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789339500268303:2119] held: no
2025-11-28T14:52:49.317533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:49.322361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789338974101804:2108] [node 5] ICS09 handshake done sender: [1:7577789338974101776:2113] self: [1:7577789338974101773:0] peer: [5:7577789339500268308:0] socket: 164 qp: -1
2025-11-28T14:52:49.317511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.316806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789337979406226:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:49.317978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789341124333254:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:49.317526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.320639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789338175844945:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789338175845032:2120] held: no
2025-11-28T14:52:49.322568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789338974101804:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:49.316840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.318005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.318514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338908493602:2049] [node 2] ICP22 created new session: [6:7577789338908493692:2049]
2025-11-28T14:52:49.318518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789341124333218:2108] [node 5] ICP22 created new session: [2:7577789341124333255:2108]
2025-11-28T14:52:49.319664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338908493692:2049] [node 2] ICS09 handshake done sender: [6:7577789338908493679:2115] self: [6:7577789338908493681:0] peer: [2:7577789341124333225:0] socket: 171 qp: -1
2025-11-28T14:52:49.322627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789338974101786:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:49.319799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338908493692:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:49.320073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789341124333255:2108] [node 5] ICS09 handshake done sender: [2:7577789341124333233:2116] self: [2:7577789341124333231:0] peer: [5:7577789339500268300:0] socket: 180 qp: -1
2025-11-28T14:52:49.320687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338175845041:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37441
2025-11-28T14:52:49.322827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:49.319829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.320099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789341124333255:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:49.320550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338908493682:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:49.324148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789339500268215:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789339500268304:2120] held: no
2025-11-28T14:52:49.320953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338908493603:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.322009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.320124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:49.324247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789338974101768:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.322218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338908493603:2050] [node 3] ICP22 created new session: [6:7577789338908493695:2050]
2025-11-28T14:52:49.324307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:49.324917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789338974101768:2109] [node 6] ICP22 created new session: [1:7577789338974101807:2109]
2025-11-28T14:52:49.324973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789338974101807:2109] [node 6] ICS09 handshake done sender: [1:7577789338974101786:2119] self: [1:7577789338974101785:0] peer: [6:7577789338908493690:0] socket: 202 qp: -1
2025-11-28T14:52:49.322258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338908493695:2050] [node 3] ICS09 handshake done sender: [6:7577789338908493682:2117] self: [6:7577789338908493687:0] peer: [3:7577789337979406205:0] socket: 200 qp: -1
2025-11-28T14:52:49.325008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789338974101807:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:49.325157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.322293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338908493695:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:49.322323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:49.325863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339500268303:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:49.325933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338175845032:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.326489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493697:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
2025-11-28T14:52:49.326491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339500268311:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60238
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.327043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789339500268278:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.327259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789338175844945:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.327265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493698:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
2025-11-28T14:52:49.327401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338908493684:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.327305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.327712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:49.327771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339500268278:2108] [node 4] ICP22 created new session: [5:7577789339500268313:2108]
2025-11-28T14:52:49.327343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338175844945:2050] [node 3] ICP22 created new session: [4:7577789338175845044:2050]
2025-11-28T14:52:49.328146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338908493601:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789338908493684:2118] held: no
2025-11-28T14:52:49.328228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339500268313:2108] [node 4] ICS09 handshake done sender: [5:7577789339500268303:2119] self: [5:7577789339500268305:0] peer: [4:7577789338175845023:0] socket: 154 qp: -1
2025-11-28T14:52:49.328406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339500268313:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:49.327579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338175845044:2050] [node 3] ICS09 handshake done sender: [4:7577789338175845032:2120] self: [4:7577789338175845037:0] peer: [3:7577789337979406197:0] socket: 182 qp: -1
2025-11-28T14:52:49.328445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.329234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339500268304:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.327733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338175845044:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.327777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:49.329795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789337979406229:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2148
2025-11-28T14:52:49.330557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338908493601:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.330744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.330784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338908493601:2048] [node 1] ICP22 created new session: [6:7577789338908493699:2048]
2025-11-28T14:52:49.330832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338908493699:2048] [node 1] ICS09 handshake done sender: [6:7577789338908493684:2118] self: [6:7577789338908493690:0] peer: [1:7577789338974101785:0] socket: 204 qp: -1
2025-11-28T14:52:49.330817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789339500268215:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.330855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338908493699:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.330819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789341124333218:2108] [node 5] ICP18 handshake (actor [2:7577789341124333249:2120]) is too old
2025-11-28T14:52:49.330888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:49.330932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.330974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339500268215:2048] [node 1] ICP22 created new session: [5:7577789339500268315:2048]
2025-11-28T14:52:49.331020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339500268315:2048] [node 1] ICS09 handshake done sender: [5:7577789339500268304:2120] self: [5:7577789339500268308:0] peer: [1:7577789338974101773:0] socket: 203 qp: -1
2025-11-28T14:52:49.331056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339500268315:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.331095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:49.331526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338908493698:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.331659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338908493663:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.331924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339500268294:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.331958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789339500268294:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.332820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339500268294:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.332970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789339500268279:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.332947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338908493702:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39524
2025-11-28T14:52:49.333061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:49.333099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339500268279:2109] [node 6] ICP22 created new session: [5:7577789339500268318:2109]
2025-11-28T14:52:49.333162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339500268318:2109] [node 6] ICS09 handshake done sender: [5:7577789339500268294:2116] self: [5:7577789339500268293:0] peer: [6:7577789338908493701:0] socket: 175 qp: -1
2025-11-28T14:52:49.333171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338908493698:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:49.333197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339500268318:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:49.333233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.333302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338908493663:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.333640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:49.333681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338908493663:2107] [node 5] ICP22 created new session: [6:7577789338908493703:2107]
2025-11-28T14:52:49.333724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338908493703:2107] [node 5] ICS09 handshake done sender: [6:7577789338908493698:2123] self: [6:7577789338908493701:0] peer: [5:7577789339500268293:0] socket: 174 qp: -1
2025-11-28T14:52:49.333762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338908493703:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:49.333796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:49.334483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789338908493663:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:49.380507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:49.380782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789341124333252:2048] [node 1] ICS01 socket: 176 reason# EndOfStream
2025-11-28T14:52:49.380831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789341124333153:2048] [node 1] ICP30 unregister session Session# [2:7577789341124333252:2048] VirtualId# [2:7577789341124333241:0]
2025-11-28T14:52:49.380865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789341124333252:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.380964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:49.381023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:49.381710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:49.381865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789337979406220:2048] [node 1] ICS07 socket disconnect 163 reason# EndOfStream
2025-11-28T14:52:49.381892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789337979406220:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.381947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:49.381989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789337979406220:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:49.383047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789337979406125:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23676) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:49.383142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789337979406220:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:49.383180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789337979406125:2048] [node 1] ICP30 unregister session Session# [3:7577789337979406220:2048] VirtualId# [3:7577789337979406212:0]
2025-11-28T14:52:49.383287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:49.383325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789337979406125:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23676) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.383335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23676) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.383935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:49.384149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789338175845030:2048] [node 1] ICS01 socket: 172 reason# EndOfStream
2025-11-28T14:52:49.384212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789338175844943:2048] [node 1] ICP30 unregister session Session# [4:7577789338175845030:2048] VirtualId# [4:7577789338175845026:0]
2025-11-28T14:52:49.384247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789338175845030:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.384312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:49.384359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:49.384656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:49.384855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789339500268315:2048] [node 1] ICS01 socket: 203 reason# EndOfStream
2025-11-28T14:52:49.384899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789339500268215:2048] [node 1] ICP30 unregister session Session# [5:7577789339500268315:2048] VirtualId# [5:7577789339500268308:0]
2025-11-28T14:52:49.384929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789339500268315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.385014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:49.385062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:49.385317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:49.385441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338908493699:2048] [node 1] ICS01 socket: 204 reason# EndOfStream
2025-11-28T14:52:49.385470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338908493601:2048] [node 1] ICP30 unregister session Session# [6:7577789338908493699:2048] VirtualId# [6:7577789338908493690:0]
2025-11-28T14:52:49.385487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338908493699:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.385533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:49.385573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:49.393276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789337979406125:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:49.435987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:49.436145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789337979406219:2049] [node 2] ICS01 socket: 149 reason# EndOfStream
2025-11-28T14:52:49.436178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789337979406126:2049] [node 2] ICP30 unregister session Session# [3:7577789337979406219:2049] VirtualId# [3:7577789337979406208:0]
2025-11-28T14:52:49.436198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789337979406219:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.436262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:49.436298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:49.438123Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:49.438245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789338175845033:2049] [node 2] ICS01 socket: 151 reason# EndOfStream
2025-11-28T14:52:49.438275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789338175844944:2049] [node 2] ICP30 unregister session Session# [4:7577789338175845033:2049] VirtualId# [4:7577789338175845022:0]
2025-11-28T14:52:49.438293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789338175845033:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.438346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:49.438378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:49.438800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:49.438923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789339500268309:2049] [node 2] ICS01 socket: 197 reason# EndOfStream
2025-11-28T14:52:49.438948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789339500268216:2049] [node 2] ICP30 unregister session Session# [5:7577789339500268309:2049] VirtualId# [5:7577789339500268300:0]
2025-11-28T14:52:49.438966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789339500268309:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.439022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:49.439070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:49.439431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:49.439627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789338908493692:2049] [node 2] ICS07 socket disconnect 171 reason# EndOfStream
2025-11-28T14:52:49.439660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338908493692:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.439727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:49.439771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789338908493692:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:49.440352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338908493602:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63775) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:49.440467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338908493692:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:49.440515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338908493602:2049] [node 2] ICP30 unregister session Session# [6:7577789338908493692:2049] VirtualId# [6:7577789338908493681:0]
2025-11-28T14:52:49.440618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:49.440659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789338908493602:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63775) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.440673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63775) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.450667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789338908493602:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:49.504027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:49.504193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789338175845044:2050] [node 3] ICS01 socket: 182 reason# EndOfStream
2025-11-28T14:52:49.504227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789338175844945:2050] [node 3] ICP30 unregister session Session# [4:7577789338175845044:2050] VirtualId# [4:7577789338175845037:0]
2025-11-28T14:52:49.504253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789338175845044:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.504335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:49.504403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:49.504827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:49.504974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789339500268301:2050] [node 3] ICS01 socket: 161 reason# EndOfStream
2025-11-28T14:52:49.505010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789339500268217:2050] [node 3] ICP30 unregister session Session# [5:7577789339500268301:2050] VirtualId# [5:7577789339500268295:0]
2025-11-28T14:52:49.505036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789339500268301:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.505097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:49.505131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:49.505468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:49.505658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789338908493695:2050] [node 3] ICS07 socket disconnect 200 reason# EndOfStream
2025-11-28T14:52:49.505698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338908493695:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.505765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:49.505812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789338908493695:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:49.506327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338908493603:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25608) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:49.506396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338908493695:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:49.506421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338908493603:2050] [node 3] ICP30 unregister session Session# [6:7577789338908493695:2050] VirtualId# [6:7577789338908493687:0]
2025-11-28T14:52:49.506451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:49.506482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789338908493603:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.506497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.516180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789338908493603:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:49.561517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:49.561663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789339500268313:2108] [node 4] ICS01 socket: 154 reason# EndOfStream
2025-11-28T14:52:49.561694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789339500268278:2108] [node 4] ICP30 unregister session Session# [5:7577789339500268313:2108] VirtualId# [5:7577789339500268305:0]
2025-11-28T14:52:49.561716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789339500268313:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.561775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:49.561807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:49.562294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:49.562482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789338908493689:2108] [node 4] ICS07 socket disconnect 199 reason# EndOfStream
2025-11-28T14:52:49.562527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338908493689:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.562606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:49.562660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789338908493689:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:49.563220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338908493665:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16786) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:49.563258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338908493689:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:49.563330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338908493665:2108] [node 4] ICP30 unregister session Session# [6:7577789338908493689:2108] VirtualId# [6:7577789338908493683:0]
2025-11-28T14:52:49.563375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:49.563402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789338908493665:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16786) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.563419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16786) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.573565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789338908493665:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:49.632145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:49.632357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789338908493703:2107] [node 5] ICS07 socket disconnect 174 reason# EndOfStream
2025-11-28T14:52:49.632412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338908493703:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:49.632522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:49.632603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789338908493703:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:49.633220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338908493663:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20203) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:49.633302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338908493703:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:49.633346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338908493663:2107] [node 5] ICP30 unregister session Session# [6:7577789338908493703:2107] VirtualId# [6:7577789338908493701:0]
2025-11-28T14:52:49.633424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:49.633448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789338908493663:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:49.633459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:49.741705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789337933416273:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:49.741716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789337933416274:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:49.757991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789339360772618:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.758024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789339360772619:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:49.773588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789339465722545:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.773809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789339465722546:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:49.790615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338924966123:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.790683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338924966124:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:49.790683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338924966125:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:49.806009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339852765985:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.806040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339852765986:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:49.806067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339852765987:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:49.824116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338239013542:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:49.824151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338239013543:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:49.824206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338239013544:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:49.874494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338239013605:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:49.910061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789337933416336:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:49.910059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789337933416337:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:49.910684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338924966186:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:49.910667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789339465722611:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:49.910859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339852766048:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:49.910455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789339360772690:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:49.910714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789338239013607:2110] [node 4] ICP01 ready to work
2025-11-28T14:52:49.910864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789339852766049:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:49.910679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789339465722610:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:49.910802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789339360772692:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:49.910959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789339465722612:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:49.910698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789339360772691:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:49.911019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789337933416338:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:49.911078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789338924966187:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:49.918038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789338924966123:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42254229ns) {Configure:104(42246667ns) {ProcessConfigured:135(390340ns) {StartInitialHandshake:157(388983ns) {DropHandshakes:470(943ns) {DropIncomingHandshake:442(305ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(669ns) PrepareNewSessionHandshake:477(1462ns) {SwitchToState:179(88ns)}}}}}
2025-11-28T14:52:49.920514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789337933416274:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51828296ns) {Configure:104(51823621ns) {ProcessConfigured:135(660477ns) {StartInitialHandshake:157(658446ns) {DropHandshakes:470(1197ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(486ns) PrepareNewSessionHandshake:477(981ns) {SwitchToState:179(391ns)}}}}}
2025-11-28T14:52:49.921086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789337933416273:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45324553ns) {Configure:104(45320164ns) {ProcessConfigured:135(619578ns) {StartInitialHandshake:157(447026ns) {DropHandshakes:470(1459ns) {DropIncomingHandshake:442(309ns) DropOutgoingHandshake:457(183ns)} GenerateSessionVirtualId:988(689ns) PrepareNewSessionHandshake:477(891ns) {SwitchToState:179(351ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.924438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339852766063:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26100
2025-11-28T14:52:49.924290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789339465722623:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-28T14:52:49.924443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789339360772702:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46894
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.924780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789339465722624:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.927187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789337933416349:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52782
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.926305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339852766063:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.927116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789339465722624:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.927909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789337933416350:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52782
2025-11-28T14:52:49.927875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338924966202:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54568
2025-11-28T14:52:49.928284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789339852766048:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.928207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789339465722623:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.928173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789337933416349:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.927898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338924966201:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54568
2025-11-28T14:52:49.928493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339852766064:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26100
2025-11-28T14:52:49.928266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789337933416351:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52782
2025-11-28T14:52:49.928240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789339465722628:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-28T14:52:49.929935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339852766065:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26100
2025-11-28T14:52:49.928428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789337933416352:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52782
2025-11-28T14:52:49.927909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338924966203:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54568
2025-11-28T14:52:49.929999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789339360772702:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.930818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789337933416350:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.930792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339852766064:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.930745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338924966192:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.931590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789337933416352:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.930771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789338924966192:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.931588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789339852765986:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.931829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789337933416351:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.932014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789339360772707:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46894
2025-11-28T14:52:49.932142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338924966201:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.932248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339852766065:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.932128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789339360772697:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.932539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339852766067:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26100
2025-11-28T14:52:49.932389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338924966192:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:49.932159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789339360772697:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.934424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789339465722620:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.932832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338924966203:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.932808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339852766066:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26100
2025-11-28T14:52:49.932802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789339360772708:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46894
2025-11-28T14:52:49.934470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789339465722620:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.933993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789339852765987:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.935358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789339360772699:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.935768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789339465722618:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.933018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789338924966124:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.937152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339852766066:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:49.935796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789339465722620:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:49.935380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789339360772699:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.933135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789338924966125:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.937478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339852766067:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.935412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789339360772697:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:49.935801Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789339465722618:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.937452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789338924966186:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.940588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789337933416343:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.938225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789339852765985:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.938422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789339360772699:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:49.940264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339852766070:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26100
2025-11-28T14:52:49.938536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789339465722618:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:49.938169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:49.940629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789337933416343:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.938507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789339360772690:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.938573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789339465722628:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.940464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339852766072:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26100
2025-11-28T14:52:49.938218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338924966186:2108] [node 5] ICP22 created new session: [4:7577789338924966207:2108]
2025-11-28T14:52:49.940041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:49.938520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338924966202:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.941922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789337933416342:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.938696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789339465722545:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.944350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339852766073:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26100
2025-11-28T14:52:49.940264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789339360772708:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.939988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789339465722610:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.939932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338924966207:2108] [node 5] ICS09 handshake done sender: [4:7577789338924966192:2110] self: [4:7577789338924966188:0] peer: [5:7577789339852766068:0] socket: 151 qp: -1
2025-11-28T14:52:49.941942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789337933416342:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.940526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789339360772690:2108] [node 5] ICP22 created new session: [2:7577789339360772712:2108]
2025-11-28T14:52:49.940975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:49.944646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789339852765986:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11959) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789339852766064:2116] held: no
2025-11-28T14:52:49.941980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789337933416348:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.940251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338924966207:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:49.941036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789339360772707:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.941670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789339465722612:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.945448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789339852765987:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789339852766065:2117] held: no
2025-11-28T14:52:49.941719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:49.942003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789337933416348:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.949520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013621:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.941282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789339360772618:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:52:49.946500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338924966208:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54568
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:49.946606Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339852766064:2116] [node 2] ICH04 handshake succeeded
::ffff:127.1.0.02025-11-28T14:52:49.941667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789339360772712:2108] [node 5] ICS09 handshake done sender: [2:7577789339360772697:2119] self: [2:7577789339360772693:0] peer: [5:7577789339852766069:0] socket: 169 qp: -1
2025-11-28T14:52:49.943389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789337933416343:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:49.941959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789339465722610:2108] [node 5] ICP22 created new session: [3:7577789339465722634:2108]
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:49.946500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338924966209:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54568
2025-11-28T14:52:49.941934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789339360772712:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:49.948396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339852766063:2115] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:49.946523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789338924966125:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789338924966203:2117] held: no
2025-11-28T14:52:49.948937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789337933416348:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.941941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789339360772691:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.941994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:49.950753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338239013621:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.949541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789337933416337:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.950785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338924966201:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:49.942212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:49.942203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789339465722634:2108] [node 5] ICS09 handshake done sender: [3:7577789339465722620:2114] self: [3:7577789339465722615:0] peer: [5:7577789339852766071:0] socket: 172 qp: -1
2025-11-28T14:52:49.950843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013622:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.951173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789339852765986:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.950256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789337933416342:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:49.950921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789338924966124:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.944419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789339360772691:2107] [node 4] ICP22 created new session: [2:7577789339360772718:2107]
2025-11-28T14:52:49.943376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789339465722634:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:49.951475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789339852766048:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.951426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013623:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.950975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.946519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789339360772718:2107] [node 4] ICS09 handshake done sender: [2:7577789339360772699:2121] self: [2:7577789339360772696:0] peer: [4:7577789338924966205:0] socket: 166 qp: -1
2025-11-28T14:52:49.943430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:49.951489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338924966124:2049] [node 2] ICP22 created new session: [4:7577789338924966213:2049]
2025-11-28T14:52:49.951437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338239013543:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.951564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.950741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:49.945403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:49.947486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789339360772718:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:49.951755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338924966213:2049] [node 2] ICS09 handshake done sender: [4:7577789338924966201:2115] self: [4:7577789338924966205:0] peer: [2:7577789339360772696:0] socket: 167 qp: -1
2025-11-28T14:52:49.951891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.950762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789337933416273:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.947521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.946521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789339465722612:2107] [node 4] ICP22 created new session: [3:7577789339465722639:2107]
2025-11-28T14:52:49.951940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338924966213:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:49.951929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339852766048:2108] [node 4] ICP22 created new session: [5:7577789339852766078:2108]
2025-11-28T14:52:49.950821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.952333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338239013622:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.948343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789339465722639:2107] [node 4] ICS09 handshake done sender: [3:7577789339465722618:2113] self: [3:7577789339465722616:0] peer: [4:7577789338924966206:0] socket: 174 qp: -1
2025-11-28T14:52:49.951973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.950821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789339360772719:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46894
2025-11-28T14:52:49.952515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338239013544:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.952301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339852766065:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:49.950879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789337933416337:2108] [node 5] ICP22 created new session: [1:7577789337933416364:2108]
2025-11-28T14:52:49.952308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338924966203:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:49.952663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339852765986:2049] [node 2] ICP22 created new session: [5:7577789339852766077:2049]
2025-11-28T14:52:49.951442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789339360772720:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46894
2025-11-28T14:52:49.951129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789337933416364:2108] [node 5] ICS09 handshake done sender: [1:7577789337933416343:2111] self: [1:7577789337933416340:0] peer: [5:7577789339852766075:0] socket: 185 qp: -1
2025-11-28T14:52:49.952762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338239013623:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.948387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789339465722639:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:49.954057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013624:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.954106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789338924966125:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.954976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338239013607:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.951947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789339360772700:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.949515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.954719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.951168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789337933416364:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:49.951062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789339465722638:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-28T14:52:49.951964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789339360772700:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.952685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339852766078:2108] [node 4] ICS09 handshake done sender: [5:7577789339852766063:2115] self: [5:7577789339852766068:0] peer: [4:7577789338924966188:0] socket: 159 qp: -1
2025-11-28T14:52:49.954770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338924966125:2050] [node 3] ICP22 created new session: [4:7577789338924966216:2050]
2025-11-28T14:52:49.951493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789339465722640:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-28T14:52:49.952776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789339360772719:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.951448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789337933416273:2048] [node 2] ICP22 created new session: [1:7577789337933416365:2048]
No RDMA context found for GID: 2025-11-28T14:52:49.955502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338924966216:2050] [node 3] ICS09 handshake done sender: [4:7577789338924966203:2117] self: [4:7577789338924966206:0] peer: [3:7577789339465722616:0] socket: 175 qp: -1
2025-11-28T14:52:49.953436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339852766078:2108] [node 4] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:49.952815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789339465722638:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.953517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789339360772708:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.955528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338924966216:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:49.951507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789337933416365:2048] [node 2] ICS09 handshake done sender: [1:7577789337933416348:2114] self: [1:7577789337933416344:0] peer: [2:7577789339360772714:0] socket: 188 qp: -1
2025-11-28T14:52:49.955596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:49.954991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789339360772618:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789339360772708:2126] held: no
2025-11-28T14:52:49.953471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.953509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789339465722628:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.957923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789339360772703:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.956095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338924966193:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.954040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339852766077:2049] [node 2] ICS09 handshake done sender: [5:7577789339852766064:2116] self: [5:7577789339852766069:0] peer: [2:7577789339360772693:0] socket: 171 qp: -1
2025-11-28T14:52:49.953525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789339465722546:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.951553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789337933416274:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.956120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789338924966193:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.957947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789339360772703:2124] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.955514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789339465722622:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.954867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339852766077:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:49.959900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789339360772700:2122] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:49.959984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338924966193:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.960212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013628:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.951766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789337933416365:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:49.960993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789338924966187:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.955539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789339465722622:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.955154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.960575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338239013624:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.951824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.960608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789339360772703:2124] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:49.961192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:49.957946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789339465722545:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.961423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338239013542:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.956114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789339852766076:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26100
2025-11-28T14:52:49.961228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338924966187:2109] [node 6] ICP22 created new session: [4:7577789338924966220:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.961840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789339360772692:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.961897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013629:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.957979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.951856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789337933416274:2049] [node 3] ICP22 created new session: [1:7577789337933416366:2049]
2025-11-28T14:52:49.958920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789339465722622:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:49.962069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338924966218:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54568
2025-11-28T14:52:49.958923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789339465722545:2048] [node 1] ICP22 created new session: [3:7577789339465722644:2048]
2025-11-28T14:52:49.951895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789337933416366:2049] [node 3] ICS09 handshake done sender: [1:7577789337933416342:2110] self: [1:7577789337933416339:0] peer: [3:7577789339465722633:0] socket: 184 qp: -1
2025-11-28T14:52:49.962070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338924966220:2109] [node 6] ICS09 handshake done sender: [4:7577789338924966193:2111] self: [4:7577789338924966189:0] peer: [6:7577789338239013627:0] socket: 205 qp: -1
2025-11-28T14:52:49.962386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338239013607:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10453) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789338239013623:2118] held: no
2025-11-28T14:52:49.956704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789339852765985:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789339852766067:2119] held: no
2025-11-28T14:52:49.959802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789339465722644:2048] [node 1] ICS09 handshake done sender: [3:7577789339465722628:2119] self: [3:7577789339465722633:0] peer: [1:7577789337933416339:0] socket: 186 qp: -1
2025-11-28T14:52:49.951923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789337933416366:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:49.962672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338924966220:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.962049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:49.951939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:49.957667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789339852765987:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.962791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789338924966186:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(20180506ns) {HandleHandshakeStatus:329(20178353ns) {DropIncomingHandshake:442(343ns) DropOutgoingHandshake:457(911ns) SwitchToState:179(373ns) ProcessPendingSessionEvents:495(15569149ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14877843ns max# 2049819ns}}}
2025-11-28T14:52:49.959833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789339465722644:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.959776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.962050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789339360772619:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.963330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013631:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.951957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:49.960167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:49.963645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.959837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339852765987:2050] [node 3] ICP22 created new session: [5:7577789339852766083:2050]
2025-11-28T14:52:49.964172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013630:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.962088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789339360772692:2109] [node 6] ICP22 created new session: [2:7577789339360772725:2109]
2025-11-28T14:52:49.964217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789338924966218:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.951990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.960187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339852766083:2050] [node 3] ICS09 handshake done sender: [5:7577789339852766065:2117] self: [5:7577789339852766071:0] peer: [3:7577789339465722615:0] socket: 177 qp: -1
2025-11-28T14:52:49.961737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789339465722611:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.962148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789339360772618:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.962077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789337933416346:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.961763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:49.967159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338239013621:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:49.960217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339852766083:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:49.965485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789338924966123:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.962364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.962110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789337933416346:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.961789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789339465722611:2110] [node 6] ICP22 created new session: [3:7577789339465722647:2110]
2025-11-28T14:52:49.967538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338239013622:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:49.962288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789339465722647:2110] [node 6] ICS09 handshake done sender: [3:7577789339465722622:2116] self: [3:7577789339465722619:0] peer: [6:7577789338239013626:0] socket: 202 qp: -1
2025-11-28T14:52:49.960255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:49.967634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789338239013628:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.963086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789339465722647:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:49.962389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789339360772619:2049] [node 3] ICP22 created new session: [2:7577789339360772726:2049]
2025-11-28T14:52:49.969282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338239013623:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:49.962169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339852766067:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.963106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.969352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338239013544:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.962401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.965440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789339465722646:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11337
2025-11-28T14:52:49.962888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789339852765985:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.962812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789339360772618:2048] [node 1] ICP22 created new session: [2:7577789339360772727:2048]
2025-11-28T14:52:49.965250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789337933416346:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:49.969568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338239013543:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11959) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789338239013621:2116] held: no
2025-11-28T14:52:49.966451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789339465722638:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:49.971313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:49.963289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789339360772725:2109] [node 6] ICS09 handshake done sender: [2:7577789339360772700:2122] self: [2:7577789339360772695:0] peer: [6:7577789338239013625:0] socket: 158 qp: -1
2025-11-28T14:52:49.965872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789337933416347:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.971936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338239013607:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.963226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.967568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789339465722546:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.965893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789337933416347:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.963654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339852765985:2048] [node 1] ICP22 created new session: [5:7577789339852766085:2048]
2025-11-28T14:52:49.971989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338239013544:2050] [node 3] ICP22 created new session: [6:7577789338239013635:2050]
2025-11-28T14:52:49.967861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789339465722612:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(19545463ns) {HandleHandshakeStatus:329(19543910ns) {DropIncomingHandshake:442(22228ns) DropOutgoingHandshake:457(635ns) SwitchToState:179(233ns) ProcessPendingSessionEvents:495(9411082ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8555873ns max# 1527575ns}}}
2025-11-28T14:52:49.966406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789337933416338:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:49.964174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339852766085:2048] [node 1] ICS09 handshake done sender: [5:7577789339852766067:2119] self: [5:7577789339852766075:0] peer: [1:7577789337933416340:0] socket: 187 qp: -1
2025-11-28T14:52:49.972032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338239013635:2050] [node 3] ICS09 handshake done sender: [6:7577789338239013622:2117] self: [6:7577789338239013626:0] peer: [3:7577789339465722619:0] socket: 203 qp: -1
2025-11-28T14:52:49.963753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789339360772727:2048] [node 1] ICS09 handshake done sender: [2:7577789339360772708:2126] self: [2:7577789339360772714:0] peer: [1:7577789337933416344:0] socket: 189 qp: -1
2025-11-28T14:52:49.968046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.967527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-11-28T14:52:49.964214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339852766085:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.969192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789339465722546:2049] [node 2] ICP22 created new session: [3:7577789339465722649:2049]
2025-11-28T14:52:49.972044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:49.967817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789337933416338:2109] [node 6] ICP22 created new session: [1:7577789337933416372:2109]
2025-11-28T14:52:49.972180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789338924966223:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54568
2025-11-28T14:52:49.963789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789339360772727:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.967995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789337933416372:2109] [node 6] ICS09 handshake done sender: [1:7577789337933416346:2112] self: [1:7577789337933416341:0] peer: [6:7577789338239013632:0] socket: 212 qp: -1
2025-11-28T14:52:49.964117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789339360772725:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:49.968379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789337933416372:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:49.965161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.964202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.969482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789339465722649:2049] [node 2] ICS09 handshake done sender: [3:7577789339465722638:2120] self: [3:7577789339465722642:0] peer: [2:7577789339360772701:0] socket: 161 qp: -1
2025-11-28T14:52:49.964210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789339360772726:2049] [node 3] ICS09 handshake done sender: [2:7577789339360772703:2124] self: [2:7577789339360772701:0] peer: [3:7577789339465722642:0] socket: 206 qp: -1
2025-11-28T14:52:49.972057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338239013635:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:49.969515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789339465722649:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:49.964390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:49.969481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.964951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789339360772726:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:49.964973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:49.969515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789337933416347:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:49.972087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338239013607:2110] [node 4] ICP22 created new session: [6:7577789338239013636:2110]
2025-11-28T14:52:49.967881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789339360772690:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21961486ns) {HandleHandshakeStatus:329(21959319ns) {DropIncomingHandshake:442(12815ns) DropOutgoingHandshake:457(265ns) SwitchToState:179(638ns) ProcessPendingSessionEvents:495(12041599ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11212872ns max# 1232217ns}}}
2025-11-28T14:52:49.972140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789337933416336:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.972097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:49.969982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.968462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789339360772691:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(20803151ns) {HandleHandshakeStatus:329(20796019ns) {DropIncomingHandshake:442(910ns) DropOutgoingHandshake:457(3635ns) SwitchToState:179(367ns) ProcessPendingSessionEvents:495(12521768ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11628590ns max# 939589ns}}}
2025-11-28T14:52:49.972123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338239013636:2110] [node 4] ICS09 handshake done sender: [6:7577789338239013623:2118] self: [6:7577789338239013627:0] peer: [4:7577789338924966189:0] socket: 207 qp: -1
2025-11-28T14:52:49.970014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789339465722545:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:49.972125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789338239013605:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:49.972149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338239013636:2110] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.972825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789338924966218:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.972167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.972526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013633:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.973125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789338924966123:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.973328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338239013543:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.973085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:49.973293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.973318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789337933416375:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52782
2025-11-28T14:52:49.973547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789338924966123:2048] [node 1] ICP22 created new session: [4:7577789338924966224:2048]
2025-11-28T14:52:49.975179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789338924966224:2048] [node 1] ICS09 handshake done sender: [4:7577789338924966218:2120] self: [4:7577789338924966222:0] peer: [1:7577789337933416345:0] socket: 176 qp: -1
2025-11-28T14:52:49.973527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789337933416336:2107] [node 4] ICP22 created new session: [1:7577789337933416376:2107]
2025-11-28T14:52:49.973875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:49.975211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789338924966224:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.974010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338239013624:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:49.974045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789339852766058:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:49.973810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789337933416376:2107] [node 4] ICS09 handshake done sender: [1:7577789337933416347:2113] self: [1:7577789337933416345:0] peer: [4:7577789338924966222:0] socket: 165 qp: -1
2025-11-28T14:52:49.974101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338239013543:2049] [node 2] ICP22 created new session: [6:7577789338239013640:2049]
2025-11-28T14:52:49.974343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338239013640:2049] [node 2] ICS09 handshake done sender: [6:7577789338239013621:2116] self: [6:7577789338239013625:0] peer: [2:7577789339360772695:0] socket: 156 qp: -1
2025-11-28T14:52:49.975301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789337933416376:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:49.975250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:49.974088Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789339852766058:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:49.975345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:49.976302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338239013640:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:49.974092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789339852766048:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(18056531ns) {HandleHandshakeStatus:329(18054794ns) {DropIncomingHandshake:442(579ns) DropOutgoingHandshake:457(4995ns) SwitchToState:179(266ns) ProcessPendingSessionEvents:495(13787410ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13695549ns max# 2010783ns}}}
2025-11-28T14:52:49.976345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:49.977335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338239013542:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.977173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789339852766058:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:49.977486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789339852766049:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:49.977533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:49.977843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:49.977559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789339852766049:2109] [node 6] ICP22 created new session: [5:7577789339852766088:2109]
2025-11-28T14:52:49.977889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338239013542:2048] [node 1] ICP22 created new session: [6:7577789338239013642:2048]
2025-11-28T14:52:49.978095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789339852766088:2109] [node 6] ICS09 handshake done sender: [5:7577789339852766058:2113] self: [5:7577789339852766053:0] peer: [6:7577789338239013638:0] socket: 155 qp: -1
2025-11-28T14:52:49.978144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338239013642:2048] [node 1] ICS09 handshake done sender: [6:7577789338239013624:2119] self: [6:7577789338239013632:0] peer: [1:7577789337933416341:0] socket: 213 qp: -1
2025-11-28T14:52:49.978231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789339852766088:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:49.978267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:49.978263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338239013642:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:49.978285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:49.980012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789337933416338:2109] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:49.980260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338239013605:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62565) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789338239013628:2120] held: no
2025-11-28T14:52:49.980310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789338239013643:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41490
2025-11-28T14:52:49.981905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789338239013628:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:49.982035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789338239013605:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:49.982173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:49.982223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789338239013605:2106] [node 5] ICP22 created new session: [6:7577789338239013645:2106]
2025-11-28T14:52:49.982276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789338239013645:2106] [node 5] ICS09 handshake done sender: [6:7577789338239013628:2120] self: [6:7577789338239013638:0] peer: [5:7577789339852766053:0] socket: 162 qp: -1
2025-11-28T14:52:49.982314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789338239013645:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:49.982357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:50.027422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:50.027577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789339360772727:2048] [node 1] ICS01 socket: 189 reason# EndOfStream
2025-11-28T14:52:50.027612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789339360772618:2048] [node 1] ICP30 unregister session Session# [2:7577789339360772727:2048] VirtualId# [2:7577789339360772714:0]
2025-11-28T14:52:50.027634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789339360772727:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.027710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.027755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.028412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:50.028698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789339465722644:2048] [node 1] ICS07 socket disconnect 186 reason# EndOfStream
2025-11-28T14:52:50.028736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789339465722644:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.028841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.028892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789339465722644:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:50.029769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789339465722545:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.030118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789339465722644:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.030170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789339465722545:2048] [node 1] ICP30 unregister session Session# [3:7577789339465722644:2048] VirtualId# [3:7577789339465722633:0]
2025-11-28T14:52:50.030286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.030327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789339465722545:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.030342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.030341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:50.030490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789338924966224:2048] [node 1] ICS07 socket disconnect 176 reason# EndOfStream
2025-11-28T14:52:50.030513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789338924966224:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.030570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.030610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789338924966224:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:50.031087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:50.031241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789338924966123:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.031228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789339852766085:2048] [node 1] ICS01 socket: 187 reason# EndOfStream
2025-11-28T14:52:50.031292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789338924966224:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.031258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789339852765985:2048] [node 1] ICP30 unregister session Session# [5:7577789339852766085:2048] VirtualId# [5:7577789339852766075:0]
2025-11-28T14:52:50.031278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789339852766085:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.031311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789338924966123:2048] [node 1] ICP30 unregister session Session# [4:7577789338924966224:2048] VirtualId# [4:7577789338924966222:0]
2025-11-28T14:52:50.031461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.031334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.031621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789338924966123:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.031412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.031642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.031991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:50.032245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789338239013642:2048] [node 1] ICS07 socket disconnect 213 reason# EndOfStream
2025-11-28T14:52:50.032280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338239013642:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.032354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.032418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789338239013642:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:50.033208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338239013542:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.033287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338239013642:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.033310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338239013542:2048] [node 1] ICP30 unregister session Session# [6:7577789338239013642:2048] VirtualId# [6:7577789338239013632:0]
2025-11-28T14:52:50.033427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.033474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789338239013542:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.033502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11982) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.040506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789339465722545:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:50.041454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789338924966123:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:50.043521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789338239013542:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:50.094962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:50.095140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789339465722649:2049] [node 2] ICS07 socket disconnect 161 reason# EndOfStream
2025-11-28T14:52:50.095167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789339465722649:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.095264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:50.095310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789339465722649:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:50.096130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789339465722546:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11959) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.096190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789339465722649:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.096218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789339465722546:2049] [node 2] ICP30 unregister session Session# [3:7577789339465722649:2049] VirtualId# [3:7577789339465722642:0]
2025-11-28T14:52:50.096399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:50.096461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789339465722546:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11959) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.096472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11959) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.097489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:50.097625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789338924966213:2049] [node 2] ICS01 socket: 167 reason# EndOfStream
2025-11-28T14:52:50.097653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789338924966124:2049] [node 2] ICP30 unregister session Session# [4:7577789338924966213:2049] VirtualId# [4:7577789338924966205:0]
2025-11-28T14:52:50.097675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789338924966213:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.097748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:50.097786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:50.098367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:50.098561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789339852766077:2049] [node 2] ICS01 socket: 171 reason# EndOfStream
2025-11-28T14:52:50.098600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789339852765986:2049] [node 2] ICP30 unregister session Session# [5:7577789339852766077:2049] VirtualId# [5:7577789339852766069:0]
2025-11-28T14:52:50.098633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789339852766077:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.098708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:50.098769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:50.099118Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:50.099261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338239013640:2049] [node 2] ICS01 socket: 156 reason# EndOfStream
2025-11-28T14:52:50.099295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338239013543:2049] [node 2] ICP30 unregister session Session# [6:7577789338239013640:2049] VirtualId# [6:7577789338239013625:0]
2025-11-28T14:52:50.099316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338239013640:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.099382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:50.099428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:50.187720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:50.187949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789338924966216:2050] [node 3] ICS01 socket: 175 reason# EndOfStream
2025-11-28T14:52:50.187998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789338924966125:2050] [node 3] ICP30 unregister session Session# [4:7577789338924966216:2050] VirtualId# [4:7577789338924966206:0]
2025-11-28T14:52:50.188055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789338924966216:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.188170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:50.188222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:50.188514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:50.188668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789339852766083:2050] [node 3] ICS01 socket: 177 reason# EndOfStream
2025-11-28T14:52:50.188698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789339852765987:2050] [node 3] ICP30 unregister session Session# [5:7577789339852766083:2050] VirtualId# [5:7577789339852766071:0]
2025-11-28T14:52:50.188722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789339852766083:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.188801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:50.188839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:50.189200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:50.189336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789338239013635:2050] [node 3] ICS07 socket disconnect 203 reason# EndOfStream
2025-11-28T14:52:50.189356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338239013635:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.189423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:50.189471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789338239013635:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:50.190045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789338239013544:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.190083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338239013635:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.190140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338239013544:2050] [node 3] ICP30 unregister session Session# [6:7577789338239013635:2050] VirtualId# [6:7577789338239013626:0]
2025-11-28T14:52:50.190220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:50.190241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789338239013544:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.190251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.200263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789338239013544:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:50.249233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:50.249397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789339852766078:2108] [node 4] ICS07 socket disconnect 159 reason# EndOfStream
2025-11-28T14:52:50.249435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789339852766078:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.249516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:50.249552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789339852766078:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:50.250287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789339852766048:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10453) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.250330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789339852766078:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.250357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789339852766048:2108] [node 4] ICP30 unregister session Session# [5:7577789339852766078:2108] VirtualId# [5:7577789339852766068:0]
2025-11-28T14:52:50.250428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:50.250455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:50.250482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789339852766048:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.250493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.250578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338239013636:2110] [node 4] ICS01 socket: 207 reason# EndOfStream
2025-11-28T14:52:50.250610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338239013607:2110] [node 4] ICP30 unregister session Session# [6:7577789338239013636:2110] VirtualId# [6:7577789338239013627:0]
2025-11-28T14:52:50.250635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338239013636:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.250696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:50.250737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:50.260636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789339852766048:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:50.318848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:50.318997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789338239013645:2106] [node 5] ICS01 socket: 162 reason# EndOfStream
2025-11-28T14:52:50.319028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789338239013605:2106] [node 5] ICP30 unregister session Session# [6:7577789338239013645:2106] VirtualId# [6:7577789338239013638:0]
2025-11-28T14:52:50.319052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789338239013645:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.319114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:50.319148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:50.393803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789345400021886:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:50.393847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789345400021887:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:50.409016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789344756491515:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.409054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789344756491516:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:50.425801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789345669655166:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.425855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789345669655167:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:50.442105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789346064261320:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.442141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789346064261321:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:50.442168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789346064261322:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:50.461742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789342440582240:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.461782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789342440582241:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:50.461809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789342440582242:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:50.481482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789345611224712:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.481530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789345611224713:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:50.481564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789345611224714:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:50.543998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789345611224772:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:50.561494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789345400021949:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:50.561361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789344756491578:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:50.562268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789344756491579:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:50.562291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789345669655229:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:50.562383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789345400021950:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:50.562485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789346064261384:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:50.562529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789345611224776:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:50.562491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789342440582312:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:50.562711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789344756491580:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:50.562814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789345400021951:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:50.562916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789345669655230:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:50.563016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789346064261385:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:50.563508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789345669655231:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:50.566249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789342440582315:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.572007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789345400021965:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25616
2025-11-28T14:52:50.572012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789344756491594:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9554
2025-11-28T14:52:50.572052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789345400021966:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25616
2025-11-28T14:52:50.572061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789344756491595:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9554
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.575504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789345669655242:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16255
2025-11-28T14:52:50.575569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789345669655243:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16255
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.575710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789342440582322:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60004
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.576292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789346064261396:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60781
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.576508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789346064261397:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60781
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.576872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789346064261401:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60781
2025-11-28T14:52:50.579737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789344756491594:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.579835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021965:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.579988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021966:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.580176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789344756491595:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.580249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789344756491596:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9554
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.580229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789345400021967:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25616
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.580307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789344756491597:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9554
2025-11-28T14:52:50.580384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789344756491598:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9554
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.580402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789345400021968:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25616
2025-11-28T14:52:50.580722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789344756491515:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.580745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789345400021969:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25616
2025-11-28T14:52:50.580965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021953:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.580992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789345400021953:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.581881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789345400021953:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.582382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789345400021886:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.582393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789344756491601:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9554
2025-11-28T14:52:50.582485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:50.582473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789342440582322:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.582567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789345400021886:2048] [node 2] ICP22 created new session: [1:7577789345400021973:2048]
2025-11-28T14:52:50.582615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789345400021973:2048] [node 2] ICS09 handshake done sender: [1:7577789345400021953:2110] self: [1:7577789345400021952:0] peer: [2:7577789344756491599:0] socket: 157 qp: -1
2025-11-28T14:52:50.582648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789345400021973:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:50.582678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:50.583029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789344756491598:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.583086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789344756491594:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:50.583031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789345669655243:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.583109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789345669655242:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.583237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789346064261396:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.583307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789344756491515:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.583345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789346064261397:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.583398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:50.583441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789344756491515:2048] [node 1] ICP22 created new session: [2:7577789344756491602:2048]
No RDMA context found for GID: 2025-11-28T14:52:50.583415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789345669655247:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16255
::ffff:127.1.0.0
2025-11-28T14:52:50.583481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789344756491602:2048] [node 1] ICS09 handshake done sender: [2:7577789344756491594:2115] self: [2:7577789344756491599:0] peer: [1:7577789345400021952:0] socket: 164 qp: -1
2025-11-28T14:52:50.583506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789344756491602:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:50.583522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789345669655248:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16255
2025-11-28T14:52:50.583571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:50.583645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789345669655166:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.583686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789345669655167:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.583833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789346064261321:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.583879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789346064261320:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.583863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789345669655248:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.583944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789346064261401:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.583958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789345669655247:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.584114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789346064261322:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.584277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789345669655237:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.584301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789345669655237:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.584422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021969:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.584586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021967:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.584684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021968:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.584772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021955:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.584794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789345400021955:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.584909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789345669655237:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.585544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789346064261405:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60781
2025-11-28T14:52:50.585608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789345669655229:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.585685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021957:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.585645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:50.585712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789345400021957:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.585677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789345669655229:2107] [node 4] ICP22 created new session: [3:7577789345669655254:2107]
2025-11-28T14:52:50.585707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789345669655254:2107] [node 4] ICS09 handshake done sender: [3:7577789345669655237:2112] self: [3:7577789345669655236:0] peer: [4:7577789346064261404:0] socket: 180 qp: -1
2025-11-28T14:52:50.585746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789344756491596:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.585726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789345669655254:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:50.585762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:50.586398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789346064261401:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:50.587004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789346064261322:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.587084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789344756491597:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.587099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:50.587130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789346064261322:2050] [node 3] ICP22 created new session: [4:7577789346064261406:2050]
2025-11-28T14:52:50.587182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789346064261406:2050] [node 3] ICS09 handshake done sender: [4:7577789346064261401:2118] self: [4:7577789346064261404:0] peer: [3:7577789345669655236:0] socket: 188 qp: -1
2025-11-28T14:52:50.587190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789344756491584:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.587212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789346064261406:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:50.587233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:50.587213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789344756491584:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.587822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789344756491586:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.587845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789344756491586:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.588880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789342440582329:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60004
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.589076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789342440582330:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60004
2025-11-28T14:52:50.589243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789345400021955:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:50.589250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789344756491584:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:50.589704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789344756491586:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:50.589889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789345400021957:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:50.590527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789344756491516:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.590594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789345400021887:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.590637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:50.590646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.590686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789345400021887:2049] [node 3] ICP22 created new session: [1:7577789345400021981:2049]
2025-11-28T14:52:50.590677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789344756491516:2049] [node 3] ICP22 created new session: [2:7577789344756491610:2049]
2025-11-28T14:52:50.590727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789345400021981:2049] [node 3] ICS09 handshake done sender: [1:7577789345400021955:2111] self: [1:7577789345400021954:0] peer: [3:7577789345669655249:0] socket: 161 qp: -1
2025-11-28T14:52:50.590710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789342440582331:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60004
2025-11-28T14:52:50.590727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789344756491610:2049] [node 3] ICS09 handshake done sender: [2:7577789344756491584:2111] self: [2:7577789344756491583:0] peer: [3:7577789345669655250:0] socket: 159 qp: -1
2025-11-28T14:52:50.590755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789345400021981:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:50.590757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789344756491610:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:50.590783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:50.590782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:50.591271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789342440582329:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.591346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789342440582242:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.591356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789346064261408:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60781
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.591641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789345669655239:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.591668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789345669655239:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.591755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789346064261409:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60781
2025-11-28T14:52:50.591855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789344756491578:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.591873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789345400021949:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.591919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:50.591963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789345400021949:2107] [node 4] ICP22 created new session: [1:7577789345400021983:2107]
2025-11-28T14:52:50.591957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.591996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789344756491578:2107] [node 4] ICP22 created new session: [2:7577789344756491612:2107]
2025-11-28T14:52:50.592043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789345400021983:2107] [node 4] ICS09 handshake done sender: [1:7577789345400021957:2112] self: [1:7577789345400021956:0] peer: [4:7577789346064261403:0] socket: 168 qp: -1
2025-11-28T14:52:50.592083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789345400021983:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:50.592062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789344756491612:2107] [node 4] ICS09 handshake done sender: [2:7577789344756491586:2112] self: [2:7577789344756491585:0] peer: [4:7577789346064261402:0] socket: 160 qp: -1
2025-11-28T14:52:50.592072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789345669655257:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16255
2025-11-28T14:52:50.592111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789344756491612:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:50.592122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:50.592136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.592342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789346064261397:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:50.592428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789345669655258:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16255
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.592658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789346064261396:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:50.592622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789342440582333:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60004
2025-11-28T14:52:50.592876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789345669655243:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:50.592918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789346064261320:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.593040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:50.593032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789342440582334:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60004
2025-11-28T14:52:50.593079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789346064261320:2048] [node 1] ICP22 created new session: [4:7577789346064261411:2048]
2025-11-28T14:52:50.593086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789345669655242:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:50.593109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789346064261411:2048] [node 1] ICS09 handshake done sender: [4:7577789346064261397:2117] self: [4:7577789346064261403:0] peer: [1:7577789345400021956:0] socket: 176 qp: -1
2025-11-28T14:52:50.593130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789346064261411:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:50.593167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:50.593308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789345669655239:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:50.593555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789345669655166:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.593584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:50.593610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789345669655166:2048] [node 1] ICP22 created new session: [3:7577789345669655260:2048]
2025-11-28T14:52:50.593675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789345669655260:2048] [node 1] ICS09 handshake done sender: [3:7577789345669655243:2116] self: [3:7577789345669655249:0] peer: [1:7577789345400021954:0] socket: 175 qp: -1
2025-11-28T14:52:50.593696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789345669655260:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:50.593813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789342440582329:2125] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:50.593719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:50.594043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789342440582242:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.594121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:50.594150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789342440582242:2050] [node 3] ICP22 created new session: [5:7577789342440582335:2050]
2025-11-28T14:52:50.594177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789342440582335:2050] [node 3] ICS09 handshake done sender: [5:7577789342440582329:2125] self: [5:7577789342440582332:0] peer: [3:7577789345669655238:0] socket: 193 qp: -1
2025-11-28T14:52:50.594195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789342440582335:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:50.594219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.595222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224790:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
2025-11-28T14:52:50.595373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789345669655167:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.595430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:50.595475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789346064261321:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.595463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789345669655167:2049] [node 2] ICP22 created new session: [3:7577789345669655262:2049]
2025-11-28T14:52:50.595860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:50.595484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789345669655262:2049] [node 2] ICS09 handshake done sender: [3:7577789345669655242:2115] self: [3:7577789345669655250:0] peer: [2:7577789344756491583:0] socket: 174 qp: -1
2025-11-28T14:52:50.596129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789346064261321:2049] [node 2] ICP22 created new session: [4:7577789346064261413:2049]
2025-11-28T14:52:50.595839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789345669655262:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:50.596028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:50.596310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789346064261413:2049] [node 2] ICS09 handshake done sender: [4:7577789346064261396:2116] self: [4:7577789346064261402:0] peer: [2:7577789344756491585:0] socket: 173 qp: -1
2025-11-28T14:52:50.596345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789346064261413:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:50.596390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:50.596751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789342440582331:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.597294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789345669655230:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.597358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:50.597390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789345669655230:2108] [node 5] ICP22 created new session: [3:7577789345669655264:2108]
2025-11-28T14:52:50.597414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789345669655264:2108] [node 5] ICS09 handshake done sender: [3:7577789345669655239:2113] self: [3:7577789345669655238:0] peer: [5:7577789342440582332:0] socket: 156 qp: -1
2025-11-28T14:52:50.597441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789345669655264:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:50.597559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:50.598075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789342440582241:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.599674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224791:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
2025-11-28T14:52:50.599906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789342440582330:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.601283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789342440582240:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.601535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021959:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.601573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789345400021959:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.602416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789345611224791:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.602494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789345611224712:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.602620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789345400021959:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:50.602602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789344756491588:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.602609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789345611224790:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.602636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789344756491588:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.603077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789345611224713:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.603553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789345400021961:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.603577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789345400021961:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.603680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789342440582340:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60004
2025-11-28T14:52:50.604139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789345400021950:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.604189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:50.604231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789345400021950:2108] [node 5] ICP22 created new session: [1:7577789345400021987:2108]
2025-11-28T14:52:50.604271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789345400021987:2108] [node 5] ICS09 handshake done sender: [1:7577789345400021959:2113] self: [1:7577789345400021958:0] peer: [5:7577789342440582339:0] socket: 202 qp: -1
2025-11-28T14:52:50.604301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789345400021987:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:50.604311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789345669655167:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:50.604348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:50.604469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789344756491590:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.604518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789342440582330:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:50.604497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789344756491590:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.604496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789346064261321:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:50.604970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789342440582240:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.605025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:50.605062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789342440582240:2048] [node 1] ICP22 created new session: [5:7577789342440582341:2048]
2025-11-28T14:52:50.605097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789342440582341:2048] [node 1] ICS09 handshake done sender: [5:7577789342440582330:2126] self: [5:7577789342440582339:0] peer: [1:7577789345400021958:0] socket: 204 qp: -1
2025-11-28T14:52:50.605128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789342440582341:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:50.605156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.605577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789346064261415:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60781
2025-11-28T14:52:50.605619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789344756491588:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.605959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224794:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
2025-11-28T14:52:50.606290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789344756491590:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.606608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789344756491579:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.606655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224795:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
2025-11-28T14:52:50.606699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:50.606746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789344756491579:2108] [node 5] ICP22 created new session: [2:7577789344756491617:2108]
2025-11-28T14:52:50.606790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789344756491617:2108] [node 5] ICS09 handshake done sender: [2:7577789344756491588:2113] self: [2:7577789344756491587:0] peer: [5:7577789342440582338:0] socket: 207 qp: -1
2025-11-28T14:52:50.606965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789344756491617:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:50.607021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.607459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224796:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
2025-11-28T14:52:50.607657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789345611224790:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:50.608020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789345611224794:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.608162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789345611224796:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.608187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789344756491580:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.608236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:50.608200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789345611224713:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.608302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789344756491580:2109] [node 6] ICP22 created new session: [2:7577789344756491619:2109]
2025-11-28T14:52:50.608304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:50.608355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789344756491619:2109] [node 6] ICS09 handshake done sender: [2:7577789344756491590:2114] self: [2:7577789344756491589:0] peer: [6:7577789345611224793:0] socket: 190 qp: -1
2025-11-28T14:52:50.608395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789344756491619:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:50.608352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789345611224713:2049] [node 2] ICP22 created new session: [6:7577789345611224798:2049]
2025-11-28T14:52:50.608415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789345400021961:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:50.608446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789345611224798:2049] [node 2] ICS09 handshake done sender: [6:7577789345611224790:2115] self: [6:7577789345611224793:0] peer: [2:7577789344756491589:0] socket: 191 qp: -1
2025-11-28T14:52:50.608462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:50.608474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789345611224798:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:50.608502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:50.609281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789345400021951:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.609290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789342440582333:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.609353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:50.609397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789345400021951:2109] [node 6] ICP22 created new session: [1:7577789345400021990:2109]
2025-11-28T14:52:50.609442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789345400021990:2109] [node 6] ICS09 handshake done sender: [1:7577789345400021961:2114] self: [1:7577789345400021960:0] peer: [6:7577789345611224792:0] socket: 192 qp: -1
2025-11-28T14:52:50.609470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789345400021990:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:50.609495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:50.609568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789346064261415:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.610083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789342440582343:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60004
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.610753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224797:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
2025-11-28T14:52:50.611394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789342440582312:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.613631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789342440582241:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789342440582331:2127] held: no
2025-11-28T14:52:50.613762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789345611224712:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789345611224791:2116] held: no
2025-11-28T14:52:50.614348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789345611224714:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.614415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789342440582331:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:50.614401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789345611224776:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.614443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789345611224791:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:50.614439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789346064261393:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.614466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789346064261393:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.615327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789346064261393:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:50.615543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789346064261384:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.615611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789342440582241:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.615621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:50.615668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789346064261384:2108] [node 5] ICP22 created new session: [4:7577789346064261419:2108]
2025-11-28T14:52:50.615702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789346064261419:2108] [node 5] ICS09 handshake done sender: [4:7577789346064261393:2114] self: [4:7577789346064261392:0] peer: [5:7577789342440582345:0] socket: 212 qp: -1
2025-11-28T14:52:50.615699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:50.615726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789346064261419:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:50.615758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:50.615761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789342440582241:2049] [node 2] ICP22 created new session: [5:7577789342440582346:2049]
2025-11-28T14:52:50.615814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789342440582346:2049] [node 2] ICS09 handshake done sender: [5:7577789342440582331:2127] self: [5:7577789342440582338:0] peer: [2:7577789344756491587:0] socket: 208 qp: -1
2025-11-28T14:52:50.615847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789342440582346:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:50.615853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789345611224712:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.615929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:50.615897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:50.615923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789345611224712:2048] [node 1] ICP22 created new session: [6:7577789345611224803:2048]
2025-11-28T14:52:50.615957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789345669655241:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.615989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789345669655241:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:50.615958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789345611224803:2048] [node 1] ICS09 handshake done sender: [6:7577789345611224791:2116] self: [6:7577789345611224792:0] peer: [1:7577789345400021960:0] socket: 194 qp: -1
2025-11-28T14:52:50.615978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789345611224803:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:50.615994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:50.616926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789345669655241:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:50.616977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789346064261395:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.616996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789346064261395:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.617640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224800:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
2025-11-28T14:52:50.617822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789345669655231:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.617923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:50.617988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789345669655231:2109] [node 6] ICP22 created new session: [3:7577789345669655268:2109]
2025-11-28T14:52:50.617992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789346064261395:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:50.618050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789345669655268:2109] [node 6] ICS09 handshake done sender: [3:7577789345669655241:2114] self: [3:7577789345669655240:0] peer: [6:7577789345611224801:0] socket: 181 qp: -1
2025-11-28T14:52:50.618096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789345669655268:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:50.618137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:50.618756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789346064261385:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.618822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:50.618856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789346064261385:2109] [node 6] ICP22 created new session: [4:7577789346064261423:2109]
2025-11-28T14:52:50.618925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789346064261423:2109] [node 6] ICS09 handshake done sender: [4:7577789346064261395:2115] self: [4:7577789346064261394:0] peer: [6:7577789345611224802:0] socket: 219 qp: -1
2025-11-28T14:52:50.618953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789346064261423:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:50.618924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224807:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
2025-11-28T14:52:50.618974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.619304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224808:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.619847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789346064261422:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60781
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.620336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789345669655270:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16255
2025-11-28T14:52:50.620900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789342440582312:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28141) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789342440582333:2128] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.621978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789345611224794:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:50.621921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789342440582348:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60004
2025-11-28T14:52:50.622427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789345611224714:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.622438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789342440582333:2128] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:50.622449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789345669655270:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.622489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:50.622543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789345669655231:2109] [node 6] ICP18 handshake (actor [3:7577789345669655270:2121]) is too old
2025-11-28T14:52:50.622526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789345611224714:2050] [node 3] ICP22 created new session: [6:7577789345611224809:2050]
2025-11-28T14:52:50.622500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789346064261422:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.622609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789345611224809:2050] [node 3] ICS09 handshake done sender: [6:7577789345611224794:2117] self: [6:7577789345611224801:0] peer: [3:7577789345669655240:0] socket: 197 qp: -1
2025-11-28T14:52:50.622619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789346064261385:2109] [node 6] ICP18 handshake (actor [4:7577789346064261422:2123]) is too old
2025-11-28T14:52:50.622645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789345611224809:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:50.622685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:50.622720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789342440582312:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.622829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:50.622930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789342440582312:2108] [node 4] ICP22 created new session: [5:7577789342440582349:2108]
2025-11-28T14:52:50.622994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789342440582349:2108] [node 4] ICS09 handshake done sender: [5:7577789342440582333:2128] self: [5:7577789342440582345:0] peer: [4:7577789346064261392:0] socket: 213 qp: -1
2025-11-28T14:52:50.623037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789342440582349:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:50.623123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:50.623980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789345611224796:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:50.624204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789345611224800:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.625930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789345611224776:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.625969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:50.625996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789345611224776:2107] [node 4] ICP22 created new session: [6:7577789345611224811:2107]
2025-11-28T14:52:50.626025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789345611224811:2107] [node 4] ICS09 handshake done sender: [6:7577789345611224796:2119] self: [6:7577789345611224802:0] peer: [4:7577789346064261394:0] socket: 220 qp: -1
2025-11-28T14:52:50.626056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789345611224811:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:50.626076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:50.626527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789345611224776:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:50.627588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789345611224772:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:50.627839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789342440582324:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:50.627877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789342440582324:2124] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:50.628729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789342440582324:2124] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:50.628807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789345611224814:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64788
2025-11-28T14:52:50.628993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789342440582315:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:50.629117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:50.629155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789345611224800:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:50.629168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789342440582315:2109] [node 6] ICP22 created new session: [5:7577789342440582352:2109]
2025-11-28T14:52:50.629217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789342440582352:2109] [node 6] ICS09 handshake done sender: [5:7577789342440582324:2124] self: [5:7577789342440582323:0] peer: [6:7577789345611224813:0] socket: 165 qp: -1
2025-11-28T14:52:50.629259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789342440582352:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:50.629318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:50.629343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789345611224772:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:50.629388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:50.629412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789345611224772:2105] [node 5] ICP22 created new session: [6:7577789345611224815:2105]
2025-11-28T14:52:50.629447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789345611224815:2105] [node 5] ICS09 handshake done sender: [6:7577789345611224800:2121] self: [6:7577789345611224813:0] peer: [5:7577789342440582323:0] socket: 162 qp: -1
2025-11-28T14:52:50.629490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789345611224815:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:50.629535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:50.645328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:50.645493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789344756491602:2048] [node 1] ICS07 socket disconnect 164 reason# EndOfStream
2025-11-28T14:52:50.645531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789344756491602:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.645627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.645667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789344756491602:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:50.646807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:50.646965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789345669655260:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-28T14:52:50.646997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789345669655166:2048] [node 1] ICP30 unregister session Session# [3:7577789345669655260:2048] VirtualId# [3:7577789345669655249:0]
2025-11-28T14:52:50.647020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789345669655260:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.647144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.647187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.648811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789344756491515:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:50.648857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789344756491602:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:50.649520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:50.649703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789346064261411:2048] [node 1] ICS07 socket disconnect 176 reason# EndOfStream
2025-11-28T14:52:50.649735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789346064261411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.649792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.649763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789344756491515:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.649798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789344756491602:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.649836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789346064261411:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:50.649818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789344756491515:2048] [node 1] ICP30 unregister session Session# [2:7577789344756491602:2048] VirtualId# [2:7577789344756491599:0]
2025-11-28T14:52:50.649890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.649914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789344756491515:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.649942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.650389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789346064261320:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.650424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789346064261411:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.650440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789346064261320:2048] [node 1] ICP30 unregister session Session# [4:7577789346064261411:2048] VirtualId# [4:7577789346064261403:0]
2025-11-28T14:52:50.650502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.650523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789346064261320:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.650532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.650707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:50.650834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789342440582341:2048] [node 1] ICS07 socket disconnect 204 reason# EndOfStream
2025-11-28T14:52:50.650861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789342440582341:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.650907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.650937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789342440582341:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:50.651411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789342440582240:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.651441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789342440582341:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.651484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:50.651460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789342440582240:2048] [node 1] ICP30 unregister session Session# [5:7577789342440582341:2048] VirtualId# [5:7577789342440582339:0]
2025-11-28T14:52:50.651539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.651559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789342440582240:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.651566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.651617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789345611224803:2048] [node 1] ICS01 socket: 194 reason# EndOfStream
2025-11-28T14:52:50.651646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789345611224712:2048] [node 1] ICP30 unregister session Session# [6:7577789345611224803:2048] VirtualId# [6:7577789345611224792:0]
2025-11-28T14:52:50.651672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789345611224803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.651735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:50.651783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:50.659982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789344756491515:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:50.659984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789346064261320:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:50.661002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789342440582240:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:50.688092Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:50.688249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789345669655262:2049] [node 2] ICS07 socket disconnect 174 reason# EndOfStream
2025-11-28T14:52:50.688281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789345669655262:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.688404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:50.688454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789345669655262:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:50.690244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:50.690425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789346064261413:2049] [node 2] ICS07 socket disconnect 173 reason# EndOfStream
2025-11-28T14:52:50.690450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789346064261413:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.690517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:50.690551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789346064261413:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:50.690711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789345669655167:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:50.690751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789345669655262:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:50.691185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:50.691326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789345669655167:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.691380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789346064261321:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.691379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789342440582346:2049] [node 2] ICS01 socket: 208 reason# EndOfStream
2025-11-28T14:52:50.691367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789345669655262:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.691422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789346064261413:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.691393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789345669655167:2049] [node 2] ICP30 unregister session Session# [3:7577789345669655262:2049] VirtualId# [3:7577789345669655250:0]
2025-11-28T14:52:50.691421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789342440582241:2049] [node 2] ICP30 unregister session Session# [5:7577789342440582346:2049] VirtualId# [5:7577789342440582338:0]
2025-11-28T14:52:50.691447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789346064261321:2049] [node 2] ICP30 unregister session Session# [4:7577789346064261413:2049] VirtualId# [4:7577789346064261402:0]
2025-11-28T14:52:50.691453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789342440582346:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.691471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:50.691495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789345669655167:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.691504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.691508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:50.691522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:50.691531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789346064261321:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.691538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.691582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:50.692072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:50.692208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789345611224798:2049] [node 2] ICS07 socket disconnect 191 reason# EndOfStream
2025-11-28T14:52:50.692230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789345611224798:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.692273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:50.692314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789345611224798:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:50.692908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789345611224713:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.692941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789345611224798:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.692963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789345611224713:2049] [node 2] ICP30 unregister session Session# [6:7577789345611224798:2049] VirtualId# [6:7577789345611224793:0]
2025-11-28T14:52:50.692993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:50.693015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789345611224713:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.693024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.700936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789346064261321:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:50.702978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789345611224713:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:50.711811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:50.711965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789346064261406:2050] [node 3] ICS07 socket disconnect 188 reason# EndOfStream
2025-11-28T14:52:50.711994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789346064261406:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.712074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:50.712115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789346064261406:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:50.712615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:50.712733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789342440582335:2050] [node 3] ICS07 socket disconnect 193 reason# EndOfStream
2025-11-28T14:52:50.712753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789342440582335:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.712792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:50.712824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789342440582335:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:50.713308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789346064261322:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32575) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.713354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789346064261406:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.713375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789346064261322:2050] [node 3] ICP30 unregister session Session# [4:7577789346064261406:2050] VirtualId# [4:7577789346064261404:0]
2025-11-28T14:52:50.713464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:50.713501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789346064261322:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32575) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.713510Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789342440582242:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32575) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.713512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32575) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.713546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789342440582335:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.713505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:50.713573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789342440582242:2050] [node 3] ICP30 unregister session Session# [5:7577789342440582335:2050] VirtualId# [5:7577789342440582332:0]
2025-11-28T14:52:50.713601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:50.713626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789342440582242:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32575) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.713636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32575) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.713649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789345611224809:2050] [node 3] ICS07 socket disconnect 197 reason# EndOfStream
2025-11-28T14:52:50.713673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789345611224809:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.713739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:50.713894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789345611224809:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:50.714423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789345611224714:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32575) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.714455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789345611224809:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.714475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789345611224714:2050] [node 3] ICP30 unregister session Session# [6:7577789345611224809:2050] VirtualId# [6:7577789345611224801:0]
2025-11-28T14:52:50.714538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:50.714557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789345611224714:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32575) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.714566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32575) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.723473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789346064261322:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:50.723488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789342440582242:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:50.724466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789345611224714:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:50.744090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:50.744248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789342440582349:2108] [node 4] ICS01 socket: 213 reason# EndOfStream
2025-11-28T14:52:50.744284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789342440582312:2108] [node 4] ICP30 unregister session Session# [5:7577789342440582349:2108] VirtualId# [5:7577789342440582345:0]
2025-11-28T14:52:50.744315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789342440582349:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.744452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:50.744499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:50.744870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:50.745014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789345611224811:2107] [node 4] ICS07 socket disconnect 220 reason# EndOfStream
2025-11-28T14:52:50.745039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789345611224811:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.745100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:50.745144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789345611224811:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:50.745710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789345611224776:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28141) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:50.745745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789345611224811:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:50.745776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789345611224776:2107] [node 4] ICP30 unregister session Session# [6:7577789345611224811:2107] VirtualId# [6:7577789345611224802:0]
2025-11-28T14:52:50.745843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:50.745867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789345611224776:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28141) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.745877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28141) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:50.755233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789345611224776:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:50.766012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:50.766162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789345611224815:2105] [node 5] ICS01 socket: 162 reason# EndOfStream
2025-11-28T14:52:50.766192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789345611224772:2105] [node 5] ICP30 unregister session Session# [6:7577789345611224815:2105] VirtualId# [6:7577789345611224813:0]
2025-11-28T14:52:50.766218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789345611224815:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:50.766303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:50.766348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:50.828843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789344434511518:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:50.828862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789344434511517:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:50.846860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789343984846123:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.846965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789343984846124:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:50.864463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789345582800820:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.864609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789345582800821:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:50.881998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789343464503357:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.882058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789343464503359:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:50.882068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789343464503358:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:50.898181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789343255805743:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.898236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789343255805745:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:50.898284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789343255805744:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:50.914636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789344307370455:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:50.914706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789344307370457:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:50.914704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789344307370456:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:50.989554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789344434511580:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:51.000830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789343464503420:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:51.001769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789348729478877:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:51.001891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789348602337814:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:51.001686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789343464503421:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:51.001752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789345582800883:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:51.001845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789347550773103:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:51.001764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789348279813483:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:51.001776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789343984846186:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:51.002136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789347550773102:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:51.001775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789349877768180:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:51.002140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789348729478878:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:51.002479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789349877768181:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:51.002992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789348279813484:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:51.003411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789348602337815:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.013814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789348279813498:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.014175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789348279813499:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4621
2025-11-28T14:52:51.015072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789348279813500:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4621
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.017280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348729478888:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17448
2025-11-28T14:52:51.017259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349877768193:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22613
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:51.017824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348729478887:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17448
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.017261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349877768194:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22613
2025-11-28T14:52:51.018150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349877768195:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22613
2025-11-28T14:52:51.018188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348729478889:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17448
2025-11-28T14:52:51.019579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789347550773117:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27663
2025-11-28T14:52:51.019587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348729478890:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17448
No RDMA context found for GID: 2025-11-28T14:52:51.019994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789347550773118:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27663
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:51.020069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478888:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:51.020223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478887:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.020406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348729478896:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17448
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.020394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789347759470731:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61704
2025-11-28T14:52:51.020674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789348279813498:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.020710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349877768198:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22613
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.020865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349877768194:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.021116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789348279813501:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4621
2025-11-28T14:52:51.022779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789348279813502:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4621
2025-11-28T14:52:51.021477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349877768199:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22613
2025-11-28T14:52:51.022259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349877768193:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.022994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789348279813499:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.022592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349877768195:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.023628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789347550773117:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.024658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789343255805744:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.024671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789348279813501:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.025403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789348279813490:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.025467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478889:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.025431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789348279813490:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.026381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789348279813502:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.025953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789345582800821:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.026584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478896:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.026771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789343984846123:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.027046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789348279813490:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.027231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349877768198:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.027368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478890:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.027070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789348279813500:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.027399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789348279813486:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.027439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349877768199:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:51.027423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789348279813486:2110] [node 3] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-28T14:52:51.027473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478880:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.027566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789345582800820:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.027497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348729478880:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.027663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789347759470731:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.027813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789347550773118:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.028230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789348279813483:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.028236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789343464503357:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.028096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789347550773120:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27663
2025-11-28T14:52:51.028243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789343255805745:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.029292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.029288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478882:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.029377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789348279813483:2108] [node 5] ICP22 created new session: [2:7577789348279813508:2108]
2025-11-28T14:52:51.029315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348729478882:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.029457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478884:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.029473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349877768187:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.029509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789347550773117:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.029503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789349877768187:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.029473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789348279813508:2108] [node 5] ICS09 handshake done sender: [2:7577789348279813490:2112] self: [2:7577789348279813489:0] peer: [5:7577789347550773119:0] socket: 180 qp: -1
2025-11-28T14:52:51.029488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348729478884:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.029526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789348279813508:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:51.030026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:51.031787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789343255805744:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.031861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348729478880:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.031833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:51.031882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789343255805744:2049] [node 2] ICP22 created new session: [5:7577789347550773122:2049]
2025-11-28T14:52:51.031911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348729478882:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.031867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349877768187:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.032239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789347550773122:2049] [node 2] ICS09 handshake done sender: [5:7577789347550773117:2115] self: [5:7577789347550773119:0] peer: [2:7577789348279813489:0] socket: 184 qp: -1
2025-11-28T14:52:51.032260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789347550773122:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:51.032287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:51.033108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789349877768180:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:51.033089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789344434511518:2049] [node 3] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:51.033257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.033319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789349877768180:2108] [node 5] ICP22 created new session: [3:7577789349877768207:2108]
2025-11-28T14:52:51.033126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789344434511517:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.033393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349877768207:2108] [node 5] ICS09 handshake done sender: [3:7577789349877768187:2112] self: [3:7577789349877768186:0] peer: [5:7577789347550773121:0] socket: 185 qp: -1
2025-11-28T14:52:51.033129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.033152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789344434511518:2049] [node 3] ICP22 created new session: [1:7577789348729478905:2049]
2025-11-28T14:52:51.033408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789347759470733:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61704
2025-11-28T14:52:51.033468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349877768207:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:51.033171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:51.033205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789344434511517:2048] [node 2] ICP22 created new session: [1:7577789348729478906:2048]
2025-11-28T14:52:51.033515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:51.033332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348729478905:2049] [node 3] ICS09 handshake done sender: [1:7577789348729478882:2111] self: [1:7577789348729478881:0] peer: [3:7577789349877768203:0] socket: 194 qp: -1
2025-11-28T14:52:51.033354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348729478906:2048] [node 2] ICS09 handshake done sender: [1:7577789348729478880:2110] self: [1:7577789348729478879:0] peer: [2:7577789348279813507:0] socket: 193 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.033362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348729478905:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:51.033385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.033386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348729478906:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:51.033653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789347550773124:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27663
2025-11-28T14:52:51.033652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789348279813486:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.033441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.033749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789343255805744:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.034135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789347759470734:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61704
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.034313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349877768206:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22613
2025-11-28T14:52:51.034248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789348279813512:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.035050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789343255805745:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21848) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789347550773118:2116] held: no
2025-11-28T14:52:51.035043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789347759470735:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61704
2025-11-28T14:52:51.035231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348729478884:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.035089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789343984846124:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.035150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.035230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789343984846124:2049] [node 3] ICP22 created new session: [2:7577789348279813514:2049]
2025-11-28T14:52:51.035490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789347550773118:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.035270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789348279813514:2049] [node 3] ICS09 handshake done sender: [2:7577789348279813486:2110] self: [2:7577789348279813485:0] peer: [3:7577789349877768202:0] socket: 173 qp: -1
2025-11-28T14:52:51.035307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789348279813514:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.035777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789347759470731:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.035369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.035901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789343255805745:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.036164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789343464503357:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.035926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.036276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349877768208:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22613
2025-11-28T14:52:51.036519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:51.036316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789343255805745:2050] [node 3] ICP22 created new session: [5:7577789347550773126:2050]
2025-11-28T14:52:51.036573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789343464503357:2048] [node 1] ICP22 created new session: [4:7577789347759470736:2048]
2025-11-28T14:52:51.036678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789344434511580:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.036613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789347550773126:2050] [node 3] ICS09 handshake done sender: [5:7577789347550773118:2116] self: [5:7577789347550773121:0] peer: [3:7577789349877768186:0] socket: 186 qp: -1
2025-11-28T14:52:51.036658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789347759470736:2048] [node 1] ICS09 handshake done sender: [4:7577789347759470731:2115] self: [4:7577789347759470732:0] peer: [1:7577789348729478883:0] socket: 198 qp: -1
2025-11-28T14:52:51.036716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.036647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789347550773126:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:51.036687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789347759470736:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.036686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.036738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789344434511580:2107] [node 4] ICP22 created new session: [1:7577789348729478911:2107]
2025-11-28T14:52:51.036714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:51.036771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348729478911:2107] [node 4] ICS09 handshake done sender: [1:7577789348729478884:2112] self: [1:7577789348729478883:0] peer: [4:7577789347759470732:0] socket: 196 qp: -1
2025-11-28T14:52:51.036801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348729478911:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:51.036823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:51.037956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349877768199:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.039165Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789345582800820:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789349877768199:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.039661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349877768195:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.039625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337829:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
2025-11-28T14:52:51.039704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789347759470738:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61704
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.040252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789345582800820:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.043126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789347550773128:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27663
2025-11-28T14:52:51.041183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:51.043232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789347550773129:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27663
2025-11-28T14:52:51.042199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789345582800820:2048] [node 1] ICP22 created new session: [3:7577789349877768211:2048]
2025-11-28T14:52:51.043051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349877768211:2048] [node 1] ICS09 handshake done sender: [3:7577789349877768199:2119] self: [3:7577789349877768203:0] peer: [1:7577789348729478881:0] socket: 200 qp: -1
2025-11-28T14:52:51.043280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789348279813502:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.043186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349877768211:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.043223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.043661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789343984846123:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789348279813502:2119] held: no
2025-11-28T14:52:51.044031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337830:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
2025-11-28T14:52:51.044258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789345582800821:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.044282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789347759470735:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.044284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789347550773129:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.044321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:51.044390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789345582800821:2049] [node 2] ICP22 created new session: [3:7577789349877768213:2049]
2025-11-28T14:52:51.044346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789343984846123:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.044433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349877768213:2049] [node 2] ICS09 handshake done sender: [3:7577789349877768195:2116] self: [3:7577789349877768202:0] peer: [2:7577789348279813485:0] socket: 175 qp: -1
2025-11-28T14:52:51.044407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:51.044406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789347550773128:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.044475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349877768213:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:51.044447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789343984846123:2048] [node 1] ICP22 created new session: [2:7577789348279813517:2048]
2025-11-28T14:52:51.044485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789348279813517:2048] [node 1] ICS09 handshake done sender: [2:7577789348279813502:2119] self: [2:7577789348279813507:0] peer: [1:7577789348729478879:0] socket: 195 qp: -1
2025-11-28T14:52:51.044510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:51.044512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789348279813517:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.044537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:51.044547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789347550773103:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.044630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789343255805743:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.044630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348602337829:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.044792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789344307370455:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.046074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478894:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.046071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789347759470725:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.046111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348729478894:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.046102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789347759470725:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.046681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348729478886:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.046707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348729478886:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.046922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348602337830:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.046973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348729478894:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:51.047893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348729478886:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.047896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789348602337815:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.048105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789348729478878:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.048002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789347759470725:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.048147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:51.048229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789348729478878:2109] [node 6] ICP22 created new session: [1:7577789348729478915:2109]
No RDMA context found for GID: 2025-11-28T14:52:51.048279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789343464503420:2108] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:51.048247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337832:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
2025-11-28T14:52:51.048261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348729478915:2109] [node 6] ICS09 handshake done sender: [1:7577789348729478894:2118] self: [1:7577789348729478891:0] peer: [6:7577789348602337831:0] socket: 182 qp: -1
2025-11-28T14:52:51.048353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348729478915:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:51.048438Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789345582800821:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:51.048496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789347550773116:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.048382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337833:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
2025-11-28T14:52:51.048463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:51.048392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.048528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789347550773116:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.048621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789343464503420:2108] [node 5] ICP22 created new session: [4:7577789347759470742:2108]
2025-11-28T14:52:51.048802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789347759470742:2108] [node 5] ICS09 handshake done sender: [4:7577789347759470725:2113] self: [4:7577789347759470724:0] peer: [5:7577789347550773131:0] socket: 179 qp: -1
2025-11-28T14:52:51.048888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789347759470742:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:51.048923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:51.049915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348602337829:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.049993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789347550773134:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27663
2025-11-28T14:52:51.050285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789344307370455:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.050422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:51.050069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789347550773133:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27663
2025-11-28T14:52:51.050450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789344307370455:2048] [node 1] ICP22 created new session: [6:7577789348602337835:2048]
2025-11-28T14:52:51.051538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348602337835:2048] [node 1] ICS09 handshake done sender: [6:7577789348602337829:2115] self: [6:7577789348602337831:0] peer: [1:7577789348729478891:0] socket: 188 qp: -1
2025-11-28T14:52:51.051982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348602337835:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.052011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:51.052012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789347759470734:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.052001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789347550773103:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789347550773129:2120] held: no
2025-11-28T14:52:51.052022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789348729478877:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.052085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.052654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789348729478877:2108] [node 5] ICP22 created new session: [1:7577789348729478917:2108]
2025-11-28T14:52:51.052743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789343464503358:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.052768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348729478917:2108] [node 5] ICS09 handshake done sender: [1:7577789348729478886:2113] self: [1:7577789348729478885:0] peer: [5:7577789347550773132:0] socket: 167 qp: -1
2025-11-28T14:52:51.053424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348729478917:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:51.053456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:51.053827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789348279813492:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.054033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789347550773129:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.053862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789348279813492:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.055152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789347550773103:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.055193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.055294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789347550773103:2108] [node 4] ICP22 created new session: [5:7577789347550773136:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:51.055699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337837:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
::ffff:127.1.0.0
2025-11-28T14:52:51.055322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789347550773136:2108] [node 4] ICS09 handshake done sender: [5:7577789347550773129:2120] self: [5:7577789347550773131:0] peer: [4:7577789347759470724:0] socket: 197 qp: -1
2025-11-28T14:52:51.055703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789347550773136:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:51.055737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:51.055849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337838:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
2025-11-28T14:52:51.055983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789347550773128:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.056078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789347550773116:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:51.056234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348602337833:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.056215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789348279813492:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.056616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789347759470738:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.056986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789343255805743:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.057012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789347550773102:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.057060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:51.057103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789343984846186:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.057363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789347759470747:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61704
2025-11-28T14:52:51.057709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.057091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:51.057721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348602337838:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.057103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789347550773102:2109] [node 6] ICP22 created new session: [5:7577789347550773138:2109]
2025-11-28T14:52:51.057388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789343464503359:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.057762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789343984846186:2107] [node 4] ICP22 created new session: [2:7577789348279813520:2107]
2025-11-28T14:52:51.057369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789343255805743:2048] [node 1] ICP22 created new session: [5:7577789347550773139:2048]
2025-11-28T14:52:51.057548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789347550773138:2109] [node 6] ICS09 handshake done sender: [5:7577789347550773116:2114] self: [5:7577789347550773115:0] peer: [6:7577789348602337834:0] socket: 176 qp: -1
2025-11-28T14:52:51.058569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789348279813520:2107] [node 4] ICS09 handshake done sender: [2:7577789348279813492:2113] self: [2:7577789348279813491:0] peer: [4:7577789347759470745:0] socket: 215 qp: -1
2025-11-28T14:52:51.057748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789347759470746:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61704
2025-11-28T14:52:51.058558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349877768189:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.057714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789347550773138:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:51.058603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789348279813520:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:51.058607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789349877768189:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.057714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789347550773139:2048] [node 1] ICS09 handshake done sender: [5:7577789347550773128:2119] self: [5:7577789347550773132:0] peer: [1:7577789348729478885:0] socket: 187 qp: -1
2025-11-28T14:52:51.058967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:51.057744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789347550773139:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.057745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:51.058379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:51.058940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789343464503358:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789347759470734:2117] held: no
2025-11-28T14:52:51.059734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349877768189:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.059816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348602337830:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.059856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789344307370456:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.059930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789348602337814:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.060012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789345582800883:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.060162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789347759470734:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.060347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337840:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
2025-11-28T14:52:51.060071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.060338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789347759470749:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61704
2025-11-28T14:52:51.060295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789345582800883:2107] [node 4] ICP22 created new session: [3:7577789349877768217:2107]
2025-11-28T14:52:51.060416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349877768217:2107] [node 4] ICS09 handshake done sender: [3:7577789349877768189:2113] self: [3:7577789349877768188:0] peer: [4:7577789347759470748:0] socket: 181 qp: -1
2025-11-28T14:52:51.060448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349877768217:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:51.060837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789343464503358:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.061068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:51.060847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789348279813494:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.060837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789348602337815:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.060709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:51.060878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789348279813494:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.060951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.060999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789348602337815:2108] [node 5] ICP22 created new session: [6:7577789348602337843:2108]
2025-11-28T14:52:51.061966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789348279813494:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:51.062015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789343464503358:2049] [node 2] ICP22 created new session: [4:7577789347759470750:2049]
2025-11-28T14:52:51.062445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348602337843:2108] [node 5] ICS09 handshake done sender: [6:7577789348602337830:2116] self: [6:7577789348602337834:0] peer: [5:7577789347550773115:0] socket: 218 qp: -1
2025-11-28T14:52:51.062515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789347759470750:2049] [node 2] ICS09 handshake done sender: [4:7577789347759470734:2117] self: [4:7577789347759470745:0] peer: [2:7577789348279813491:0] socket: 214 qp: -1
2025-11-28T14:52:51.062479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348602337843:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:51.062817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789348279813484:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.062777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789347759470750:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:51.062515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:51.062858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:51.062785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789347759470738:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.062890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789348279813484:2109] [node 6] ICP22 created new session: [2:7577789348279813523:2109]
2025-11-28T14:52:51.063033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789348279813523:2109] [node 6] ICS09 handshake done sender: [2:7577789348279813494:2114] self: [2:7577789348279813493:0] peer: [6:7577789348602337841:0] socket: 219 qp: -1
2025-11-28T14:52:51.062799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:51.063060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789348279813523:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:51.063082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:51.063325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789343464503359:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21848) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789347759470738:2119] held: no
2025-11-28T14:52:51.063802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348602337840:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.064141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789343464503359:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.064475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789343255805743:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:51.064631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.065154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789347759470730:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.065172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789347759470730:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.065226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789343464503359:2050] [node 3] ICP22 created new session: [4:7577789347759470752:2050]
2025-11-28T14:52:51.065751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789347759470752:2050] [node 3] ICS09 handshake done sender: [4:7577789347759470738:2119] self: [4:7577789347759470748:0] peer: [3:7577789349877768188:0] socket: 217 qp: -1
2025-11-28T14:52:51.065783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789347759470752:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:51.065820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.066116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789344307370456:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789348602337833:2118] held: no
2025-11-28T14:52:51.066610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789347759470747:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.066780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789347759470730:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:51.066812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789344307370457:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.066816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337845:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
2025-11-28T14:52:51.066947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789343464503421:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.066987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:51.067018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789343464503421:2109] [node 6] ICP22 created new session: [4:7577789347759470756:2109]
2025-11-28T14:52:51.067053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789347759470756:2109] [node 6] ICS09 handshake done sender: [4:7577789347759470730:2114] self: [4:7577789347759470729:0] peer: [6:7577789348602337842:0] socket: 216 qp: -1
2025-11-28T14:52:51.067128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789347759470756:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.067148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:51.067252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349877768197:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.067291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789349877768197:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.067313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337847:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
2025-11-28T14:52:51.067320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348602337833:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.068100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789344307370456:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.068152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:51.068213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789344307370456:2049] [node 2] ICP22 created new session: [6:7577789348602337848:2049]
2025-11-28T14:52:51.068261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348602337848:2049] [node 2] ICS09 handshake done sender: [6:7577789348602337833:2118] self: [6:7577789348602337841:0] peer: [2:7577789348279813493:0] socket: 221 qp: -1
2025-11-28T14:52:51.068297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348602337848:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:51.068256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349877768197:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:51.068331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:51.068499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789349877768181:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.068572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:51.068868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789349877768181:2109] [node 6] ICP22 created new session: [3:7577789349877768220:2109]
2025-11-28T14:52:51.068916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349877768220:2109] [node 6] ICS09 handshake done sender: [3:7577789349877768197:2117] self: [3:7577789349877768196:0] peer: [6:7577789348602337846:0] socket: 172 qp: -1
2025-11-28T14:52:51.068952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349877768220:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:51.069125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:51.069279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789348602337814:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789348602337838:2120] held: no
2025-11-28T14:52:51.069946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348602337838:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.070917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789348602337814:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.070969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.070971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789344307370457:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21848) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789348602337840:2121] held: no
2025-11-28T14:52:51.071016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789348602337814:2107] [node 4] ICP22 created new session: [6:7577789348602337851:2107]
2025-11-28T14:52:51.071068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348602337851:2107] [node 4] ICS09 handshake done sender: [6:7577789348602337838:2120] self: [6:7577789348602337842:0] peer: [4:7577789347759470729:0] socket: 226 qp: -1
2025-11-28T14:52:51.071107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348602337851:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:51.071142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.071679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348602337850:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53055
2025-11-28T14:52:51.072026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348602337840:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.072160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789344307370457:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.072204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.072251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789344307370457:2050] [node 3] ICP22 created new session: [6:7577789348602337853:2050]
2025-11-28T14:52:51.072303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348602337853:2050] [node 3] ICS09 handshake done sender: [6:7577789348602337840:2121] self: [6:7577789348602337846:0] peer: [3:7577789349877768196:0] socket: 230 qp: -1
2025-11-28T14:52:51.072338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348602337853:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:51.072401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.089073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:51.089232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789348279813517:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-11-28T14:52:51.089270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789343984846123:2048] [node 1] ICP30 unregister session Session# [2:7577789348279813517:2048] VirtualId# [2:7577789348279813507:0]
2025-11-28T14:52:51.089301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789348279813517:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.089370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.089428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.090447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:51.090600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789349877768211:2048] [node 1] ICS01 socket: 200 reason# EndOfStream
2025-11-28T14:52:51.090627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789345582800820:2048] [node 1] ICP30 unregister session Session# [3:7577789349877768211:2048] VirtualId# [3:7577789349877768203:0]
2025-11-28T14:52:51.090647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789349877768211:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.090701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.090739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.092733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:51.092935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789347759470736:2048] [node 1] ICS07 socket disconnect 198 reason# EndOfStream
2025-11-28T14:52:51.092970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789347759470736:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.093042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.093091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789347759470736:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:51.093748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789343464503357:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.093787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789347759470736:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.093888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789343464503357:2048] [node 1] ICP30 unregister session Session# [4:7577789347759470736:2048] VirtualId# [4:7577789347759470732:0]
2025-11-28T14:52:51.093944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.093889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:51.093980Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789343464503357:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.093994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.094079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789347550773139:2048] [node 1] ICS07 socket disconnect 187 reason# EndOfStream
2025-11-28T14:52:51.094112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789347550773139:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.094167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.094225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789347550773139:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:51.094812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789343255805743:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.094852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789347550773139:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.094876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789343255805743:2048] [node 1] ICP30 unregister session Session# [5:7577789347550773139:2048] VirtualId# [5:7577789347550773132:0]
2025-11-28T14:52:51.094973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.095003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789343255805743:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.095012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.095150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:51.095402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789348602337835:2048] [node 1] ICS07 socket disconnect 188 reason# EndOfStream
2025-11-28T14:52:51.095445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348602337835:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.095550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.095603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789348602337835:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:51.096391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789344307370455:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.096442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348602337835:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.096579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789344307370455:2048] [node 1] ICP30 unregister session Session# [6:7577789348602337835:2048] VirtualId# [6:7577789348602337831:0]
2025-11-28T14:52:51.096723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.096779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789344307370455:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.096793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10308) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.103411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789343464503357:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:51.104478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789343255805743:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:51.106520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789344307370455:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:51.167855Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:51.168014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789349877768213:2049] [node 2] ICS07 socket disconnect 175 reason# EndOfStream
2025-11-28T14:52:51.168046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789349877768213:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.168123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:51.168159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789349877768213:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:51.170046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:51.170126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789345582800821:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3346) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:51.170150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789349877768213:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:51.170330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789347759470750:2049] [node 2] ICS01 socket: 214 reason# EndOfStream
2025-11-28T14:52:51.170377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789343464503358:2049] [node 2] ICP30 unregister session Session# [4:7577789347759470750:2049] VirtualId# [4:7577789347759470745:0]
2025-11-28T14:52:51.170416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789347759470750:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.170484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:51.170528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:51.170846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789345582800821:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3346) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.170894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789349877768213:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.170924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789345582800821:2049] [node 2] ICP30 unregister session Session# [3:7577789349877768213:2049] VirtualId# [3:7577789349877768202:0]
2025-11-28T14:52:51.170999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:51.171023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789345582800821:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.171032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.171079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:51.171297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789347550773122:2049] [node 2] ICS01 socket: 184 reason# EndOfStream
2025-11-28T14:52:51.171358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789343255805744:2049] [node 2] ICP30 unregister session Session# [5:7577789347550773122:2049] VirtualId# [5:7577789347550773119:0]
2025-11-28T14:52:51.171396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789347550773122:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.171465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:51.171511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:51.171860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:51.172108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348602337848:2049] [node 2] ICS01 socket: 221 reason# EndOfStream
2025-11-28T14:52:51.172161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789344307370456:2049] [node 2] ICP30 unregister session Session# [6:7577789348602337848:2049] VirtualId# [6:7577789348602337841:0]
2025-11-28T14:52:51.172190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348602337848:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.172279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:51.172336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:51.191982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:51.192213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789347759470752:2050] [node 3] ICS01 socket: 217 reason# EndOfStream
2025-11-28T14:52:51.192262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789343464503359:2050] [node 3] ICP30 unregister session Session# [4:7577789347759470752:2050] VirtualId# [4:7577789347759470748:0]
2025-11-28T14:52:51.192312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789347759470752:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.192422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:51.192470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:51.192772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:51.192981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789347550773126:2050] [node 3] ICS01 socket: 186 reason# EndOfStream
2025-11-28T14:52:51.193071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789343255805745:2050] [node 3] ICP30 unregister session Session# [5:7577789347550773126:2050] VirtualId# [5:7577789347550773121:0]
2025-11-28T14:52:51.193109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789347550773126:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.193173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:51.193219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:51.193518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:51.193752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348602337853:2050] [node 3] ICS01 socket: 230 reason# EndOfStream
2025-11-28T14:52:51.193809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789344307370457:2050] [node 3] ICP30 unregister session Session# [6:7577789348602337853:2050] VirtualId# [6:7577789348602337846:0]
2025-11-28T14:52:51.193851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348602337853:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.193953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:51.194007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:51.243919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:51.244159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789347550773136:2108] [node 4] ICS01 socket: 197 reason# EndOfStream
2025-11-28T14:52:51.244237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789347550773103:2108] [node 4] ICP30 unregister session Session# [5:7577789347550773136:2108] VirtualId# [5:7577789347550773131:0]
2025-11-28T14:52:51.244278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789347550773136:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.244391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:51.244446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:51.244843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:51.245095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348602337851:2107] [node 4] ICS01 socket: 226 reason# EndOfStream
2025-11-28T14:52:51.245171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789348602337814:2107] [node 4] ICP30 unregister session Session# [6:7577789348602337851:2107] VirtualId# [6:7577789348602337842:0]
2025-11-28T14:52:51.245210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348602337851:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.245326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:51.245386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:51.282089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:51.282320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789348602337843:2108] [node 5] ICS07 socket disconnect 218 reason# EndOfStream
2025-11-28T14:52:51.282358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348602337843:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.282488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:51.282549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789348602337843:2108] [node 5] ICS15 start handshake
2025-11-28T14:52:51.283291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789348602337815:2108] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3948) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.283338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348602337843:2108] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.283399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789348602337815:2108] [node 5] ICP30 unregister session Session# [6:7577789348602337843:2108] VirtualId# [6:7577789348602337834:0]
2025-11-28T14:52:51.283486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:51.283514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789348602337815:2108] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3948) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.283523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3948) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:51.373783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789348674784589:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:51.373884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789348674784590:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:51.391381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789347543020503:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:51.391461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789347543020504:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:51.410018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789349580120282:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:51.410174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789349580120283:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:51.427865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789348752027387:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:51.428049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789348752027388:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:51.428108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789348752027386:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:51.444115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789348924144450:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:51.444153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789348924144451:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:51.444185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789348924144452:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:51.463116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789348048356333:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:51.463151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789348048356335:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:51.463167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789348048356334:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:51.541060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789348674784652:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:51.557007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789347543020567:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:51.561878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789349580120345:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:51.562788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789347543020568:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:51.562964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789348048356396:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:51.562928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789348674784653:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:51.563211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789348924144513:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:51.562846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789349580120347:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:51.562861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789348752027449:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:51.563443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789348674784654:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:51.562799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789347543020569:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:51.563205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789348752027450:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:51.562959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789349580120346:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:51.563408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789348048356397:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:51.563517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789348924144514:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:51.569319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789348674784589:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19838059ns) {Configure:104(12136670ns) {ProcessConfigured:135(185136ns) {StartInitialHandshake:157(182145ns) {DropHandshakes:470(4349ns) {DropIncomingHandshake:442(3139ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:477(1808ns) {SwitchToState:179(426ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.574509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789347543020576:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24654
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.576748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789348752027457:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42006
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.577895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789348752027457:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.577337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348674784668:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:51.577481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789347543020576:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.577964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348674784669:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-11-28T14:52:51.578040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789348752027386:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.577920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789347543020584:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24654
2025-11-28T14:52:51.577969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348674784670:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-11-28T14:52:51.579765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349580120354:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61529
2025-11-28T14:52:51.578689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789347543020585:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24654
2025-11-28T14:52:51.579855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784660:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.578697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789347543020503:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.579882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348674784660:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.582739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789348924144524:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10855
2025-11-28T14:52:51.580029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784658:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.580054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348674784658:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.583621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349580120354:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.584140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789348924144524:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.584490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348674784658:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.585242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349580120357:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61529
2025-11-28T14:52:51.585289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789347543020584:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.584493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348674784660:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.586209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789348752027466:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42006
2025-11-28T14:52:51.586051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349580120358:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61529
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.587311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789348924144451:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.586097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784670:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.587023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789348752027467:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42006
2025-11-28T14:52:51.586083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789349580120282:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.586457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784669:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.587828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784659:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.587852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348674784659:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.586726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349580120358:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.588467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789348674784589:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.588790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789347543020587:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24654
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.587039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349580120357:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.588639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789348674784652:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.588914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789347543020575:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.589848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789348752027457:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.591234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789348752027386:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.588671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.587235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349580120359:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61529
2025-11-28T14:52:51.588932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789347543020575:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.588722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:51.587923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789349580120283:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.588774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789348674784589:2048] [node 2] ICP22 created new session: [1:7577789348674784675:2048]
2025-11-28T14:52:51.589854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789347543020574:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.588937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349580120359:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.588784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789348674784652:2107] [node 4] ICP22 created new session: [1:7577789348674784674:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.589885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789347543020574:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.588944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348674784674:2107] [node 4] ICS09 handshake done sender: [1:7577789348674784660:2112] self: [1:7577789348674784655:0] peer: [4:7577789348752027465:0] socket: 170 qp: -1
2025-11-28T14:52:51.592563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:51.588974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348674784674:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:51.590619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789347543020588:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24654
2025-11-28T14:52:51.593630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349580120368:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61529
2025-11-28T14:52:51.588997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:51.593830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349580120353:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.592620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789348752027386:2048] [node 1] ICP22 created new session: [4:7577789348752027468:2048]
2025-11-28T14:52:51.592683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789347543020585:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.589270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348674784675:2048] [node 2] ICS09 handshake done sender: [1:7577789348674784658:2110] self: [1:7577789348674784657:0] peer: [2:7577789347543020586:0] socket: 168 qp: -1
2025-11-28T14:52:51.593849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789349580120353:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.593521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789347543020576:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.592657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789348752027468:2048] [node 1] ICS09 handshake done sender: [4:7577789348752027457:2113] self: [4:7577789348752027465:0] peer: [1:7577789348674784655:0] socket: 173 qp: -1
2025-11-28T14:52:51.593615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789347543020575:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.589984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348674784675:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:51.590012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:51.592887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789348752027466:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.594426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789347543020574:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.592890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789348752027468:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.591016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348674784672:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-11-28T14:52:51.594822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789347543020567:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.593016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789348752027388:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.594855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.594465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784672:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.595033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789347543020567:2109] [node 5] ICP22 created new session: [2:7577789347543020594:2109]
2025-11-28T14:52:51.593647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:51.595045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349580120370:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61529
2025-11-28T14:52:51.595070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789347543020594:2109] [node 5] ICS09 handshake done sender: [2:7577789347543020575:2113] self: [2:7577789347543020571:0] peer: [5:7577789348924144529:0] socket: 183 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.596445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789348924144530:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10855
2025-11-28T14:52:51.597216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789349580120371:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61529
2025-11-28T14:52:51.596174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789347543020594:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:51.597210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784668:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.596454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789347543020587:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.596946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789348924144531:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10855
2025-11-28T14:52:51.596492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789347543020503:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.597712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349580120353:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.598253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348674784659:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.598446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789348924144531:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.598656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789348924144524:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.597738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349580120357:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.596524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:51.599105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789348752027472:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42006
2025-11-28T14:52:51.598402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789349580120283:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.596986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789347543020503:2048] [node 1] ICP22 created new session: [2:7577789347543020596:2048]
2025-11-28T14:52:51.599264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789348924144451:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.599300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.599325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789348924144451:2049] [node 2] ICP22 created new session: [5:7577789348924144533:2049]
2025-11-28T14:52:51.598822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.597185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:51.598943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789349580120283:2049] [node 2] ICP22 created new session: [3:7577789349580120373:2049]
2025-11-28T14:52:51.600424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356411:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
2025-11-28T14:52:51.600272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789348674784679:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-11-28T14:52:51.600396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789348752027466:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.599453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349580120373:2049] [node 2] ICS09 handshake done sender: [3:7577789349580120357:2115] self: [3:7577789349580120367:0] peer: [2:7577789347543020572:0] socket: 191 qp: -1
2025-11-28T14:52:51.599546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789348924144533:2049] [node 2] ICS09 handshake done sender: [5:7577789348924144524:2113] self: [5:7577789348924144529:0] peer: [2:7577789347543020571:0] socket: 189 qp: -1
2025-11-28T14:52:51.597208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789347543020596:2048] [node 1] ICS09 handshake done sender: [2:7577789347543020576:2114] self: [2:7577789347543020586:0] peer: [1:7577789348674784657:0] socket: 172 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.599474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349580120373:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:51.600334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789348674784652:2107] [node 4] ICP18 handshake (actor [1:7577789348674784669:2116]) is too old
2025-11-28T14:52:51.600438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789348674784590:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.597503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789347543020596:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.599495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:51.600478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.600342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789348924144533:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:51.600692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789348674784589:2048] [node 2] ICP18 handshake (actor [1:7577789348674784670:2117]) is too old
2025-11-28T14:52:51.597984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:51.601285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789348752027388:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.600222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789349580120345:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.600665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789348674784590:2049] [node 3] ICP22 created new session: [1:7577789348674784683:2049]
2025-11-28T14:52:51.599426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789347543020504:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.600446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.600709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789348924144514:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.601433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.599467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.600653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789349580120345:2107] [node 4] ICP22 created new session: [3:7577789349580120375:2107]
2025-11-28T14:52:51.601272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:51.601382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348674784683:2049] [node 3] ICS09 handshake done sender: [1:7577789348674784659:2111] self: [1:7577789348674784656:0] peer: [3:7577789349580120365:0] socket: 169 qp: -1
2025-11-28T14:52:51.601474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789348752027388:2050] [node 3] ICP22 created new session: [4:7577789348752027473:2050]
2025-11-28T14:52:51.601405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348674784683:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:51.601429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.601246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349580120368:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.601406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789348924144532:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10855
2025-11-28T14:52:51.599513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789347543020504:2049] [node 3] ICP22 created new session: [2:7577789347543020598:2049]
2025-11-28T14:52:51.601591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789348752027473:2050] [node 3] ICS09 handshake done sender: [4:7577789348752027466:2116] self: [4:7577789348752027470:0] peer: [3:7577789349580120348:0] socket: 201 qp: -1
2025-11-28T14:52:51.601372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349580120375:2107] [node 4] ICS09 handshake done sender: [3:7577789349580120353:2112] self: [3:7577789349580120348:0] peer: [4:7577789348752027470:0] socket: 199 qp: -1
2025-11-28T14:52:51.601703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789348752027473:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:51.601405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349580120375:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:51.601730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.599553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789347543020598:2049] [node 3] ICS09 handshake done sender: [2:7577789347543020574:2112] self: [2:7577789347543020572:0] peer: [3:7577789349580120367:0] socket: 181 qp: -1
2025-11-28T14:52:51.601433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:51.602981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789348752027459:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.599582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789347543020598:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:51.600063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.603007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789348752027459:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.603725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349580120354:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.605346Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789348924144451:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:51.605579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784679:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.606610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789347543020590:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24654
2025-11-28T14:52:51.606657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789349580120282:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.606713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:51.606756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789349580120282:2048] [node 1] ICP22 created new session: [3:7577789349580120379:2048]
2025-11-28T14:52:51.606804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349580120379:2048] [node 1] ICS09 handshake done sender: [3:7577789349580120354:2113] self: [3:7577789349580120365:0] peer: [1:7577789348674784656:0] socket: 174 qp: -1
2025-11-28T14:52:51.606878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349580120379:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.606943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:51.606929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789348752027475:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42006
2025-11-28T14:52:51.606894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356412:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.607060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789347543020590:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.607179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789348924144537:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10855
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.607331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789348924144538:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10855
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.607761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356413:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
2025-11-28T14:52:51.607949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789348752027459:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.608117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789348752027475:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.608431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789348924144531:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.608691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789348752027449:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.608824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356414:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
2025-11-28T14:52:51.608728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.608924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789348924144514:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.609075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789347543020504:2049] [node 3] ICP18 handshake (actor [2:7577789347543020587:2119]) is too old
2025-11-28T14:52:51.608871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789348752027449:2108] [node 5] ICP22 created new session: [4:7577789348752027479:2108]
2025-11-28T14:52:51.609083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348048356411:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.608928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789348752027479:2108] [node 5] ICS09 handshake done sender: [4:7577789348752027459:2114] self: [4:7577789348752027458:0] peer: [5:7577789348924144534:0] socket: 176 qp: -1
2025-11-28T14:52:51.609077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789348752027479:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:51.609144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.609256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789348924144514:2108] [node 4] ICP22 created new session: [5:7577789348924144539:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.609288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789348924144539:2108] [node 4] ICS09 handshake done sender: [5:7577789348924144531:2117] self: [5:7577789348924144534:0] peer: [4:7577789348752027458:0] socket: 219 qp: -1
2025-11-28T14:52:51.609179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:51.609287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789348048356333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.609315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789348924144539:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:51.609566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:51.609358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789348752027478:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42006
2025-11-28T14:52:51.611539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784665:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.611570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348674784665:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.611374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789348924144532:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.611636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789348924144450:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.612047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348048356413:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.613229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789348752027478:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.613341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789348048356396:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.613685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348048356414:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.613641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789348752027464:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.613663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789348752027464:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.613957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789348048356334:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.613900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789348752027387:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.614361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789347543020583:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.614375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789347543020582:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.614396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789347543020583:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.618347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356417:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
2025-11-28T14:52:51.614401Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789347543020582:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.619037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789348674784667:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.619074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789348674784667:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.618371Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789347543020582:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.619084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348674784665:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:51.619300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348048356412:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.619347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789349580120282:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:51.619214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789347543020583:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:51.619379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789348674784653:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.620033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789348752027464:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:51.620186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789348924144537:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.619268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789347543020568:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.619423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.620340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789348752027449:2108] [node 5] ICP18 handshake (actor [4:7577789348752027475:2119]) is too old
2025-11-28T14:52:51.620220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.620273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789347543020568:2107] [node 4] ICP22 created new session: [2:7577789347543020605:2107]
2025-11-28T14:52:51.620237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789348674784653:2109] [node 6] ICP22 created new session: [1:7577789348674784688:2109]
2025-11-28T14:52:51.620674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789348048356335:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:51.620678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789348924144452:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.620475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789347543020605:2107] [node 4] ICS09 handshake done sender: [2:7577789347543020582:2115] self: [2:7577789347543020581:0] peer: [4:7577789348752027481:0] socket: 227 qp: -1
2025-11-28T14:52:51.621696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789348752027450:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.621128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356420:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
2025-11-28T14:52:51.620297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348674784688:2109] [node 6] ICS09 handshake done sender: [1:7577789348674784665:2113] self: [1:7577789348674784664:0] peer: [6:7577789348048356415:0] socket: 223 qp: -1
2025-11-28T14:52:51.621583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789348048356417:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.620512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789347543020605:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:51.621743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349580120364:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.621743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:51.621786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789349580120364:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.620330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348674784688:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:51.622507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789348752027450:2109] [node 6] ICP22 created new session: [4:7577789348752027484:2109]
2025-11-28T14:52:51.621654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356419:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
2025-11-28T14:52:51.620668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.621063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789347543020569:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.620353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:51.622461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789349580120356:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.621958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789348674784667:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.624207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789348048356333:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789348048356411:2115] held: no
2025-11-28T14:52:51.624879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789348752027484:2109] [node 6] ICS09 handshake done sender: [4:7577789348752027464:2115] self: [4:7577789348752027463:0] peer: [6:7577789348048356416:0] socket: 200 qp: -1
2025-11-28T14:52:51.622486Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789349580120356:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.621112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:51.625062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789348674784654:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.625079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789348752027484:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:51.622786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349580120364:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.625190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356421:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
2025-11-28T14:52:51.621250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789347543020569:2110] [node 6] ICP22 created new session: [2:7577789347543020607:2110]
2025-11-28T14:52:51.625112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.624695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789349580120346:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.625139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789348674784654:2108] [node 5] ICP22 created new session: [1:7577789348674784691:2108]
2025-11-28T14:52:51.624928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.625125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:51.626430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789348924144544:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10855
2025-11-28T14:52:51.625229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789349580120346:2108] [node 5] ICP22 created new session: [3:7577789349580120384:2108]
2025-11-28T14:52:51.625179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789348674784691:2108] [node 5] ICS09 handshake done sender: [1:7577789348674784667:2114] self: [1:7577789348674784666:0] peer: [5:7577789348924144541:0] socket: 224 qp: -1
2025-11-28T14:52:51.626905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789348924144545:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10855
2025-11-28T14:52:51.625297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789349580120356:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:51.625205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789348674784691:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.629650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789348752027483:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42006
2025-11-28T14:52:51.621568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789347543020607:2110] [node 6] ICS09 handshake done sender: [2:7577789347543020583:2116] self: [2:7577789347543020580:0] peer: [6:7577789348048356418:0] socket: 229 qp: -1
2025-11-28T14:52:51.629692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348048356414:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.625492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349580120384:2108] [node 5] ICS09 handshake done sender: [3:7577789349580120364:2118] self: [3:7577789349580120363:0] peer: [5:7577789348924144543:0] socket: 188 qp: -1
2025-11-28T14:52:51.625377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:51.629555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789348924144452:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23024) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789348924144537:2119] held: no
2025-11-28T14:52:51.626369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349580120384:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:51.621689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789347543020607:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:51.622469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:51.630196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789348924144450:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789348924144532:2118] held: no
2025-11-28T14:52:51.628253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:51.631240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789348048356397:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:51.628296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789349580120347:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.628335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:51.629487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789349580120347:2109] [node 6] ICP22 created new session: [3:7577789349580120386:2109]
2025-11-28T14:52:51.631802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789348048356334:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20064) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789348048356414:2118] held: no
2025-11-28T14:52:51.631834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348048356411:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.629795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789349580120386:2109] [node 6] ICS09 handshake done sender: [3:7577789349580120356:2114] self: [3:7577789349580120355:0] peer: [6:7577789348048356422:0] socket: 182 qp: -1
2025-11-28T14:52:51.629836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789349580120386:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:51.629867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:51.632108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789348752027387:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20064) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789348752027478:2120] held: no
2025-11-28T14:52:51.632270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789348924144532:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:51.632356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348048356413:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:51.632771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789348924144450:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.632737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789348048356334:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.632886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789348924144537:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.632950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:51.633123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789348924144452:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.633184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:51.633064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789348048356334:2049] [node 2] ICP22 created new session: [6:7577789348048356425:2049]
2025-11-28T14:52:51.633592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.633672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789348924144452:2050] [node 3] ICP22 created new session: [5:7577789348924144546:2050]
2025-11-28T14:52:51.633595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348048356425:2049] [node 2] ICS09 handshake done sender: [6:7577789348048356414:2118] self: [6:7577789348048356418:0] peer: [2:7577789347543020580:0] socket: 230 qp: -1
2025-11-28T14:52:51.634102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789348924144450:2048] [node 1] ICP22 created new session: [5:7577789348924144547:2048]
2025-11-28T14:52:51.633617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348048356425:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:51.634118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789348924144546:2050] [node 3] ICS09 handshake done sender: [5:7577789348924144537:2119] self: [5:7577789348924144543:0] peer: [3:7577789349580120363:0] socket: 192 qp: -1
2025-11-28T14:52:51.634179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789348924144547:2048] [node 1] ICS09 handshake done sender: [5:7577789348924144532:2118] self: [5:7577789348924144541:0] peer: [1:7577789348674784666:0] socket: 226 qp: -1
2025-11-28T14:52:51.634199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789348924144547:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.634115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:51.634204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789348924144546:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:51.634330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.634533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789348048356333:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.634519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:51.634568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:51.634619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789348048356333:2048] [node 1] ICP22 created new session: [6:7577789348048356427:2048]
2025-11-28T14:52:51.634673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348048356427:2048] [node 1] ICS09 handshake done sender: [6:7577789348048356411:2115] self: [6:7577789348048356415:0] peer: [1:7577789348674784664:0] socket: 225 qp: -1
2025-11-28T14:52:51.634740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789348752027478:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:51.634706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348048356427:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:51.634745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:51.634848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789348924144528:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:51.634895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789348752027387:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.635070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:51.634870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789348924144528:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:51.635105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789348048356396:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.635740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789348752027387:2049] [node 2] ICP22 created new session: [4:7577789348752027486:2049]
2025-11-28T14:52:51.635783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:51.635801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789348752027486:2049] [node 2] ICS09 handshake done sender: [4:7577789348752027478:2120] self: [4:7577789348752027481:0] peer: [2:7577789347543020581:0] socket: 231 qp: -1
2025-11-28T14:52:51.635840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789348048356396:2107] [node 4] ICP22 created new session: [6:7577789348048356431:2107]
2025-11-28T14:52:51.635841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789348752027486:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:51.635907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348048356431:2107] [node 4] ICS09 handshake done sender: [6:7577789348048356413:2117] self: [6:7577789348048356416:0] peer: [4:7577789348752027463:0] socket: 228 qp: -1
2025-11-28T14:52:51.635935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:51.635946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348048356431:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:51.635981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:51.636196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789348048356335:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23024) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789348048356412:2116] held: no
2025-11-28T14:52:51.636437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789348924144528:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.636590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789348924144513:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:51.636741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:51.636746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356423:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
2025-11-28T14:52:51.636772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789348924144513:2109] [node 6] ICP22 created new session: [5:7577789348924144551:2109]
2025-11-28T14:52:51.636808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789348924144551:2109] [node 6] ICS09 handshake done sender: [5:7577789348924144528:2115] self: [5:7577789348924144525:0] peer: [6:7577789348048356424:0] socket: 180 qp: -1
2025-11-28T14:52:51.636832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789348924144551:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:51.636866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.637607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789348752027488:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42006
2025-11-28T14:52:51.637658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789348924144553:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10855
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:51.637935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348048356412:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:51.638021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789348048356433:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36868
2025-11-28T14:52:51.638263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789348048356335:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.638294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:51.638325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789348048356335:2050] [node 3] ICP22 created new session: [6:7577789348048356434:2050]
2025-11-28T14:52:51.638352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348048356434:2050] [node 3] ICS09 handshake done sender: [6:7577789348048356412:2116] self: [6:7577789348048356422:0] peer: [3:7577789349580120355:0] socket: 187 qp: -1
2025-11-28T14:52:51.638371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348048356434:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:51.638388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:51.638590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789348048356417:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:51.638925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789348048356397:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:51.638991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:51.639034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789348048356397:2108] [node 5] ICP22 created new session: [6:7577789348048356436:2108]
2025-11-28T14:52:51.639077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789348048356436:2108] [node 5] ICS09 handshake done sender: [6:7577789348048356417:2119] self: [6:7577789348048356424:0] peer: [5:7577789348924144525:0] socket: 197 qp: -1
2025-11-28T14:52:51.639110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789348048356436:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:51.639149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:51.640590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789348924144513:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:51.677920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:51.678167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789347543020596:2048] [node 1] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-28T14:52:51.678205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789347543020596:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.678348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.678412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789347543020596:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:51.678991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:51.679176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789349580120379:2048] [node 1] ICS07 socket disconnect 174 reason# EndOfStream
2025-11-28T14:52:51.679198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789349580120379:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.679305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.679385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789349580120379:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:51.680103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789349580120282:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.680213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789349580120379:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.680278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789349580120282:2048] [node 1] ICP30 unregister session Session# [3:7577789349580120379:2048] VirtualId# [3:7577789349580120365:0]
2025-11-28T14:52:51.680785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.680832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789349580120282:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.680852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.680979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789347543020503:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:51.681024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789347543020596:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:51.682434Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:51.682706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789348752027468:2048] [node 1] ICS07 socket disconnect 173 reason# EndOfStream
2025-11-28T14:52:51.682741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789348752027468:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.682827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.682895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789348752027468:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:51.683036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789347543020503:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.683140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789347543020596:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.683178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789347543020503:2048] [node 1] ICP30 unregister session Session# [2:7577789347543020596:2048] VirtualId# [2:7577789347543020586:0]
2025-11-28T14:52:51.683281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.683323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789347543020503:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.683336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.683336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:51.683467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789348924144547:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-11-28T14:52:51.683504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789348924144450:2048] [node 1] ICP30 unregister session Session# [5:7577789348924144547:2048] VirtualId# [5:7577789348924144541:0]
2025-11-28T14:52:51.683524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789348924144547:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.683586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.683632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.683696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789348752027386:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.683784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789348752027468:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.683877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789348752027386:2048] [node 1] ICP30 unregister session Session# [4:7577789348752027468:2048] VirtualId# [4:7577789348752027465:0]
2025-11-28T14:52:51.683976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.684018Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789348752027386:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.684037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.684223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:51.684415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348048356427:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-11-28T14:52:51.684461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789348048356333:2048] [node 1] ICP30 unregister session Session# [6:7577789348048356427:2048] VirtualId# [6:7577789348048356415:0]
2025-11-28T14:52:51.684481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348048356427:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.684532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:51.684574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:51.691162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789349580120282:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:51.694221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789348752027386:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:51.756476Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:51.756741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789349580120373:2049] [node 2] ICS07 socket disconnect 191 reason# EndOfStream
2025-11-28T14:52:51.756782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789349580120373:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.756907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:51.756965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789349580120373:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:51.759442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:51.759645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789348752027486:2049] [node 2] ICS01 socket: 231 reason# EndOfStream
2025-11-28T14:52:51.759682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789348752027387:2049] [node 2] ICP30 unregister session Session# [4:7577789348752027486:2049] VirtualId# [4:7577789348752027481:0]
2025-11-28T14:52:51.759705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789348752027486:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.759710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789349580120283:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20064) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:51.759771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:51.759749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789349580120373:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:51.759812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:51.760842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:51.760858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789349580120283:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.760924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789349580120373:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.760961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789349580120283:2049] [node 2] ICP30 unregister session Session# [3:7577789349580120373:2049] VirtualId# [3:7577789349580120367:0]
2025-11-28T14:52:51.760972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789348924144533:2049] [node 2] ICS07 socket disconnect 189 reason# EndOfStream
2025-11-28T14:52:51.760996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789348924144533:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.761048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:51.761060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:51.761082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789348924144533:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:51.761098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789349580120283:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.761119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.761751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789348924144451:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.761824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789348924144533:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.761743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:51.761860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789348924144451:2049] [node 2] ICP30 unregister session Session# [5:7577789348924144533:2049] VirtualId# [5:7577789348924144529:0]
2025-11-28T14:52:51.761918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348048356425:2049] [node 2] ICS01 socket: 230 reason# EndOfStream
2025-11-28T14:52:51.761950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:51.761952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789348048356334:2049] [node 2] ICP30 unregister session Session# [6:7577789348048356425:2049] VirtualId# [6:7577789348048356418:0]
2025-11-28T14:52:51.761974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789348924144451:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.761971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348048356425:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.761984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.762026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:51.762054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:51.772075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789348924144451:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:51.812132Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:51.812291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789348752027473:2050] [node 3] ICS07 socket disconnect 201 reason# EndOfStream
2025-11-28T14:52:51.812322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789348752027473:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.812453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:51.812518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789348752027473:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:51.813056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:51.813213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789348752027388:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23024) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.813238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789348924144546:2050] [node 3] ICS01 socket: 192 reason# EndOfStream
2025-11-28T14:52:51.813260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789348752027473:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.813296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789348752027388:2050] [node 3] ICP30 unregister session Session# [4:7577789348752027473:2050] VirtualId# [4:7577789348752027470:0]
2025-11-28T14:52:51.813306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789348924144452:2050] [node 3] ICP30 unregister session Session# [5:7577789348924144546:2050] VirtualId# [5:7577789348924144543:0]
2025-11-28T14:52:51.813348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:51.813361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789348924144546:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.813375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789348752027388:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23024) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.813458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:51.813387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23024) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.813513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:51.813860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:51.813994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348048356434:2050] [node 3] ICS01 socket: 187 reason# EndOfStream
2025-11-28T14:52:51.814024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789348048356335:2050] [node 3] ICP30 unregister session Session# [6:7577789348048356434:2050] VirtualId# [6:7577789348048356422:0]
2025-11-28T14:52:51.814048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348048356434:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.814115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:51.814164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:51.823277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789348752027388:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:51.875713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:51.875975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789348924144539:2108] [node 4] ICS07 socket disconnect 219 reason# EndOfStream
2025-11-28T14:52:51.876024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789348924144539:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.876149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:51.876200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789348924144539:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:51.876527Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:51.876671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789348048356431:2107] [node 4] ICS07 socket disconnect 228 reason# EndOfStream
2025-11-28T14:52:51.876694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348048356431:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.876746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:51.876780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789348048356431:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:51.877092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789348924144514:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5796) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.877254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789348924144539:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.877970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789348924144514:2108] [node 4] ICP30 unregister session Session# [5:7577789348924144539:2108] VirtualId# [5:7577789348924144534:0]
2025-11-28T14:52:51.878043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:51.878080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789348924144514:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5796) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.878101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5796) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.878269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789348048356396:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5796) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.878319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348048356431:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.878477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789348048356396:2107] [node 4] ICP30 unregister session Session# [6:7577789348048356431:2107] VirtualId# [6:7577789348048356416:0]
2025-11-28T14:52:51.878589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:51.878999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789348048356396:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5796) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.879016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5796) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.888792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789348048356396:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:51.919175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:51.919446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789348048356436:2108] [node 5] ICS07 socket disconnect 197 reason# EndOfStream
2025-11-28T14:52:51.919487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789348048356436:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:51.919617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:51.919673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789348048356436:2108] [node 5] ICS15 start handshake
2025-11-28T14:52:51.920439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789348048356397:2108] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26410) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:51.920578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789348048356436:2108] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:51.920625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789348048356397:2108] [node 5] ICP30 unregister session Session# [6:7577789348048356436:2108] VirtualId# [6:7577789348048356424:0]
2025-11-28T14:52:51.920731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:51.920781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789348048356397:2108] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26410) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.920800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26410) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:51.930765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789348048356397:2108] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:51.989618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789350092853459:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:51.989869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789350092853460:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:52.007988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789352423429473:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.008021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789352423429474:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:52.026527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354596801959:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.026557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354596801960:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:52.044478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789350892452972:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.044606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789350892452973:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:52.044709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789350892452974:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:52.062508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789354600441932:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.062509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789354600441933:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:52.062660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789354600441934:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:52.082354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789353246086138:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.082516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789353246086139:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:52.082572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789353246086140:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:52.151154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789353246086199:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:52.159089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789350892453036:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:52.161833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789352423429536:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:52.161793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789350892453035:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:52.163300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789354387820818:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:52.164058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354596802022:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:52.164263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789354600441995:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:52.164325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789354387820819:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:52.164442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789352423429537:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:52.164201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789353246086202:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:52.164186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354596802023:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:52.164743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789354387820820:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:52.164846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789352423429538:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:52.164956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789354600441996:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:52.164324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354596802024:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:52.173284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789353246086138:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38427501ns) {Configure:104(38418462ns) {ProcessConfigured:135(50407ns) {StartInitialHandshake:157(47716ns) {DropHandshakes:470(1550ns) {DropIncomingHandshake:442(656ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:477(559ns) {SwitchToState:179(195ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.179798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789354600442008:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2848
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.180166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789350892453048:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24141
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.182134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789354600442009:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2848
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.184419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086214:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.185189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789352423429550:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34586
2025-11-28T14:52:52.185469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354596802036:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42488
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.185757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086215:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
2025-11-28T14:52:52.185946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789354600442009:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.186005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789352423429551:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34586
2025-11-28T14:52:52.186199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789354387820832:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26404
2025-11-28T14:52:52.186467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789354600442008:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.186099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354596802037:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42488
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.186418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789352423429552:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34586
2025-11-28T14:52:52.186121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354596802038:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42488
2025-11-28T14:52:52.186722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789354387820834:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26404
2025-11-28T14:52:52.186699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789354600441934:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.186594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789352423429553:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34586
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.186641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354596802036:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.186734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789354387820833:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26404
2025-11-28T14:52:52.186846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789352423429551:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.186705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354596802038:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789352423429554:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34586
2025-11-28T14:52:52.187143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789354387820835:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26404
2025-11-28T14:52:52.187033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789352423429552:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789353246086214:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354596802029:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789354387820836:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26404
2025-11-28T14:52:52.187218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789350892453048:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820832:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789352423429553:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789353246086138:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.187164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789354596802029:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.187538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789350892452972:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.187161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789352423429550:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820833:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.188657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354596802029:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:52.187246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789352423429473:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.187446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820835:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789352423429554:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.190657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789353246086215:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.187909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820827:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.191021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789354596802023:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.191424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:52.191274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789353246086139:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.187930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789354387820827:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.191532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354596802023:2108] [node 5] ICP22 created new session: [3:7577789354596802044:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.188138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820825:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.191592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354596802044:2108] [node 5] ICS09 handshake done sender: [3:7577789354596802029:2110] self: [3:7577789354596802028:0] peer: [5:7577789354600442013:0] socket: 184 qp: -1
2025-11-28T14:52:52.192135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789352423429542:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:52.191677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354596802044:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:52.188151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789354387820825:2111] [node 4] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0::ffff:127.1.0.02025-11-28T14:52:52.192188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789354600441934:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63653) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789354600442008:2114] held: no
2025-11-28T14:52:52.191715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.192156Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789352423429542:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.193513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789354600442014:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2848
2025-11-28T14:52:52.188585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820824:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.191812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354596802037:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.193620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789352423429545:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.193674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789350892453052:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24141
2025-11-28T14:52:52.188602Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789354387820824:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.193640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789352423429545:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.194402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086219:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
2025-11-28T14:52:52.193435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789354596801960:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.188610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820836:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.194109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789352423429542:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.194925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789352423429562:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34586
2025-11-28T14:52:52.190895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820834:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.195544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789350892453053:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24141
2025-11-28T14:52:52.190897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789354387820824:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.196481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789353246086138:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789353246086214:2114] held: no
2025-11-28T14:52:52.196136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789352423429545:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.196106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789350892452972:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789350892453048:2114] held: no
2025-11-28T14:52:52.197184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789354600442008:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:52.197132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789352423429550:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:52.197798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789350892453053:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.190943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789354387820825:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:52.198703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789354600441934:2050] [node 3] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:52.198009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789353246086214:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:52.199527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354596802048:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42488
2025-11-28T14:52:52.191283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789354387820827:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.197236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789352423429536:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.200581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086221:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
2025-11-28T14:52:52.198514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789350892452974:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.192226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789354387820818:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.199090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.198518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:52.199559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354596802032:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.199554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789354600441934:2050] [node 3] ICP22 created new session: [5:7577789354600442016:2050]
2025-11-28T14:52:52.193083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789354387820819:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.199591Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789354596802032:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.198688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789352423429536:2109] [node 6] ICP22 created new session: [2:7577789352423429565:2109]
2025-11-28T14:52:52.200504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789354600442016:2050] [node 3] ICS09 handshake done sender: [5:7577789354600442008:2114] self: [5:7577789354600442013:0] peer: [3:7577789354596802028:0] socket: 206 qp: -1
2025-11-28T14:52:52.193275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:52.200530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789354600442016:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.199628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789352423429474:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.193291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789350092853459:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.203736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789350892453048:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:52.202147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.201970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354596802037:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.203053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789353246086139:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6791) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789353246086215:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.193326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:52.202274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354596802032:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:52.199946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789352423429473:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.203248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789354596801960:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.200471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789352423429565:2109] [node 6] ICS09 handshake done sender: [2:7577789352423429542:2111] self: [2:7577789352423429540:0] peer: [6:7577789353246086220:0] socket: 193 qp: -1
2025-11-28T14:52:52.193427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: 2025-11-28T14:52:52.203701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-28T14:52:52.200507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.203831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789353246086138:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.204722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789350892452972:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.193509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789354387820819:2109] [node 6] ICP22 created new session: [1:7577789354387820847:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.204782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.203861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.203751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354596801960:2049] [node 2] ICP22 created new session: [3:7577789354596802051:2049]
2025-11-28T14:52:52.200586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789352423429565:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:52.204835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789350892452972:2048] [node 1] ICP22 created new session: [4:7577789350892453057:2048]
2025-11-28T14:52:52.193518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789354387820818:2107] [node 4] ICP22 created new session: [1:7577789354387820848:2107]
2025-11-28T14:52:52.205322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789354600442019:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2848
2025-11-28T14:52:52.204008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354596802051:2049] [node 2] ICS09 handshake done sender: [3:7577789354596802037:2115] self: [3:7577789354596802047:0] peer: [2:7577789352423429543:0] socket: 199 qp: -1
2025-11-28T14:52:52.203969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789353246086138:2048] [node 1] ICP22 created new session: [6:7577789353246086223:2048]
2025-11-28T14:52:52.200509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.193562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789350092853459:2048] [node 2] ICP22 created new session: [1:7577789354387820849:2048]
2025-11-28T14:52:52.204054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789354596802022:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.204001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789353246086223:2048] [node 1] ICS09 handshake done sender: [6:7577789353246086214:2114] self: [6:7577789353246086218:0] peer: [1:7577789354387820822:0] socket: 194 qp: -1
2025-11-28T14:52:52.200917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789352423429474:2049] [node 3] ICP22 created new session: [2:7577789352423429566:2049]
2025-11-28T14:52:52.204133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354596802051:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:52.204879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789350892453057:2048] [node 1] ICS09 handshake done sender: [4:7577789350892453048:2114] self: [4:7577789350892453051:0] peer: [1:7577789354387820823:0] socket: 204 qp: -1
2025-11-28T14:52:52.193568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789354387820847:2109] [node 6] ICS09 handshake done sender: [1:7577789354387820824:2110] self: [1:7577789354387820822:0] peer: [6:7577789353246086218:0] socket: 177 qp: -1
2025-11-28T14:52:52.204024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789353246086223:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:52.200963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.205667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789354600442017:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2848
2025-11-28T14:52:52.204154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.200964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789352423429473:2048] [node 1] ICP22 created new session: [2:7577789352423429567:2048]
2025-11-28T14:52:52.193655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789354387820848:2107] [node 4] ICS09 handshake done sender: [1:7577789354387820825:2111] self: [1:7577789354387820823:0] peer: [4:7577789350892453051:0] socket: 180 qp: -1
2025-11-28T14:52:52.204179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.204906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789350892453056:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24141
2025-11-28T14:52:52.193689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789354387820847:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:52.202153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789352423429567:2048] [node 1] ICS09 handshake done sender: [2:7577789352423429550:2114] self: [2:7577789352423429559:0] peer: [1:7577789354387820826:0] socket: 188 qp: -1
2025-11-28T14:52:52.204697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086222:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
2025-11-28T14:52:52.204194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:52.194050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789354387820849:2048] [node 2] ICS09 handshake done sender: [1:7577789354387820827:2112] self: [1:7577789354387820826:0] peer: [2:7577789352423429559:0] socket: 174 qp: -1
2025-11-28T14:52:52.204919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789350892453057:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:52.202174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789352423429567:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:52.205584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789353246086222:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.204901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354596802022:2107] [node 4] ICP22 created new session: [3:7577789354596802053:2107]
2025-11-28T14:52:52.194083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.207201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789354600442002:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.205018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.205072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354596802053:2107] [node 4] ICS09 handshake done sender: [3:7577789354596802032:2113] self: [3:7577789354596802025:0] peer: [4:7577789350892453054:0] socket: 224 qp: -1
2025-11-28T14:52:52.202172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789352423429566:2049] [node 3] ICS09 handshake done sender: [2:7577789352423429545:2112] self: [2:7577789352423429543:0] peer: [3:7577789354596802047:0] socket: 186 qp: -1
2025-11-28T14:52:52.194418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789354387820849:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:52.206702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789353246086199:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.202443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.205331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789350892453053:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:52.207246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789354600442002:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.205216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354596802053:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:52.194070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789354387820848:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:52.207152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789353246086215:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.205347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:52.194949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.206656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789350892452974:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.207463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789353246086139:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.202463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789352423429566:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: 2025-11-28T14:52:52.207371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789354600442019:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.195145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-28T14:52:52.208166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789354600441932:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.206695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.207808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:52.209431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789354600442017:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.202502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.208130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789353246086139:2049] [node 2] ICP22 created new session: [6:7577789353246086228:2049]
2025-11-28T14:52:52.206818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789350892452974:2050] [node 3] ICP22 created new session: [4:7577789350892453061:2050]
2025-11-28T14:52:52.209779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354596802050:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42488
2025-11-28T14:52:52.207188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789350892453061:2050] [node 3] ICS09 handshake done sender: [4:7577789350892453053:2117] self: [4:7577789350892453054:0] peer: [3:7577789354596802025:0] socket: 205 qp: -1
2025-11-28T14:52:52.209420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789353246086228:2049] [node 2] ICS09 handshake done sender: [6:7577789353246086215:2115] self: [6:7577789353246086220:0] peer: [2:7577789352423429540:0] socket: 201 qp: -1
2025-11-28T14:52:52.209795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789354600442020:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2848
2025-11-28T14:52:52.207214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789350892453061:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:52.209455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789353246086228:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:52.207784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.209866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789354600441996:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.210508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820838:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.209623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.210374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789354600442002:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.210532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789354387820838:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.209792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789350892453060:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24141
2025-11-28T14:52:52.210941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789354600441995:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.209795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789350892453059:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24141
2025-11-28T14:52:52.210952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789350892453059:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.210938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789352423429473:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:52.211533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789350892453044:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.211629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.211551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789350892453044:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.211666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789354600441995:2109] [node 6] ICP22 created new session: [5:7577789354600442025:2109]
2025-11-28T14:52:52.212028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086230:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
2025-11-28T14:52:52.212763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354596802055:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42488
2025-11-28T14:52:52.212063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789354600442025:2109] [node 6] ICS09 handshake done sender: [5:7577789354600442002:2111] self: [5:7577789354600442000:0] peer: [6:7577789353246086227:0] socket: 196 qp: -1
2025-11-28T14:52:52.212106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789350892453060:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.212600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789354387820838:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:52.212593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789354600442025:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:52.213868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.214496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789353246086199:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8203) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789353246086222:2119] held: no
2025-11-28T14:52:52.214527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354596802050:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.214778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789350892452973:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.214902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789354387820819:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16145878ns) {HandleHandshakeStatus:329(15923987ns) {DropIncomingHandshake:442(9023ns) DropOutgoingHandshake:457(196ns) SwitchToState:179(114ns) ProcessPendingSessionEvents:495(12139214ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12056297ns max# 1009097ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.215219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789350892453044:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:52.215120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789354387820820:2108] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:52.215160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:52.215425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789354600442020:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.215944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789354387820820:2108] [node 5] ICP22 created new session: [1:7577789354387820856:2108]
2025-11-28T14:52:52.216001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789354600442026:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2848
2025-11-28T14:52:52.216012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789350892453065:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24141
2025-11-28T14:52:52.216046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789352423429546:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.216004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789354387820856:2108] [node 5] ICS09 handshake done sender: [1:7577789354387820838:2119] self: [1:7577789354387820837:0] peer: [5:7577789354600442021:0] socket: 209 qp: -1
2025-11-28T14:52:52.216072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789350892453035:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.216018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789354596801959:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.216043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789354387820856:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:52.216087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789352423429546:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.216672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086231:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
2025-11-28T14:52:52.216571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354596802055:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.216825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:52.216531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.216890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789350892453035:2108] [node 5] ICP22 created new session: [4:7577789350892453068:2108]
2025-11-28T14:52:52.216691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789354387820818:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17159667ns) {HandleHandshakeStatus:329(17158005ns) {DropIncomingHandshake:442(9857ns) DropOutgoingHandshake:457(233ns) SwitchToState:179(275ns) ProcessPendingSessionEvents:495(13212976ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13131852ns max# 1011921ns}}}
2025-11-28T14:52:52.217666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789350892453068:2108] [node 5] ICS09 handshake done sender: [4:7577789350892453044:2113] self: [4:7577789350892453041:0] peer: [5:7577789354600442023:0] socket: 183 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.217694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789350892453068:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:52.217728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.217738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789353246086222:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:52.217766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086232:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
2025-11-28T14:52:52.218044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789352423429546:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:52.218064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789353246086231:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.218227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789350892453065:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.218290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789353246086199:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.218290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789353246086232:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.218077Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789354600441932:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789354600442019:2119] held: no
2025-11-28T14:52:52.218837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:52.218879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789353246086199:2105] [node 5] ICP22 created new session: [6:7577789353246086233:2105]
2025-11-28T14:52:52.218931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789353246086233:2105] [node 5] ICS09 handshake done sender: [6:7577789353246086222:2119] self: [6:7577789353246086227:0] peer: [5:7577789354600442000:0] socket: 227 qp: -1
2025-11-28T14:52:52.219064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789353246086233:2105] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.219102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.219254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789354387820828:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.219279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789354387820828:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.219312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789350892453070:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24141
2025-11-28T14:52:52.219328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789352423429537:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.219351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789354600441933:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.219387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.219557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789352423429537:2107] [node 4] ICP22 created new session: [2:7577789352423429574:2107]
2025-11-28T14:52:52.219499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789354600442019:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:52.219597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789352423429574:2107] [node 4] ICS09 handshake done sender: [2:7577789352423429546:2113] self: [2:7577789352423429544:0] peer: [4:7577789350892453066:0] socket: 231 qp: -1
2025-11-28T14:52:52.219633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789353246086202:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.219633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789352423429574:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:52.219661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:52.220667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789354600442028:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2848
2025-11-28T14:52:52.220691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789354600441932:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.220719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.220887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789354600441932:2048] [node 1] ICP22 created new session: [5:7577789354600442030:2048]
2025-11-28T14:52:52.220905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789354387820828:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:52.220919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789352423429556:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.220924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789354600442030:2048] [node 1] ICS09 handshake done sender: [5:7577789354600442019:2119] self: [5:7577789354600442021:0] peer: [1:7577789354387820837:0] socket: 230 qp: -1
2025-11-28T14:52:52.221105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789353246086140:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.220944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789352423429556:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.221303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789350092853460:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.220944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789354600442030:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.221381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.221062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789354600442017:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:52.223961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354596802040:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.223990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789354596802040:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.223834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789352423429556:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:52.224356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354596802059:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42488
2025-11-28T14:52:52.223138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789350092853460:2049] [node 3] ICP22 created new session: [1:7577789354387820859:2049]
2025-11-28T14:52:52.221085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.224020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789354387820859:2049] [node 3] ICS09 handshake done sender: [1:7577789354387820828:2113] self: [1:7577789354387820821:0] peer: [3:7577789354596802057:0] socket: 179 qp: -1
2025-11-28T14:52:52.224331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789350892453060:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.225081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789352423429538:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.224057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789354387820859:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:52.221449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789354600441996:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.225120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:52.225292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789352423429538:2108] [node 5] ICP22 created new session: [2:7577789352423429577:2108]
2025-11-28T14:52:52.225051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.225110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789350892453050:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.224321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:52.225131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789350892453050:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.225377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789352423429577:2108] [node 5] ICS09 handshake done sender: [2:7577789352423429556:2119] self: [2:7577789352423429555:0] peer: [5:7577789354600442029:0] socket: 234 qp: -1
2025-11-28T14:52:52.226833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789350892452973:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6791) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789350892453060:2120] held: no
2025-11-28T14:52:52.226204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789352423429577:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:52.224850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789354600441996:2108] [node 4] ICP22 created new session: [5:7577789354600442032:2108]
2025-11-28T14:52:52.226886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789350892453035:2108] [node 5] ICP18 handshake (actor [4:7577789350892453065:2121]) is too old
2025-11-28T14:52:52.225281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789354600442032:2108] [node 4] ICS09 handshake done sender: [5:7577789354600442017:2118] self: [5:7577789354600442023:0] peer: [4:7577789350892453041:0] socket: 182 qp: -1
2025-11-28T14:52:52.227229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789354596801959:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789354596802050:2119] held: no
2025-11-28T14:52:52.226245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.227574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354596802050:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:52.225311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789354600442032:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:52.227985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354596802040:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.228168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789350892453050:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.226006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:52.227167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789354600442033:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2848
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.228203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789354600441933:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6791) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789354600442020:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.228829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086238:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
2025-11-28T14:52:52.229100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789354596801959:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.229080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789350892453036:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.229131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789350892452973:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.229266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:52.229198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.229379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789353246086237:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36346
2025-11-28T14:52:52.229235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354596801959:2048] [node 1] ICP22 created new session: [3:7577789354596802061:2048]
2025-11-28T14:52:52.229295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789350892452973:2049] [node 2] ICP22 created new session: [4:7577789350892453072:2049]
2025-11-28T14:52:52.229277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354596802061:2048] [node 1] ICS09 handshake done sender: [3:7577789354596802050:2119] self: [3:7577789354596802057:0] peer: [1:7577789354387820821:0] socket: 207 qp: -1
2025-11-28T14:52:52.229438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789353246086232:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:52.229318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:52.229306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354596802061:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:52.229327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789350892453072:2049] [node 2] ICS09 handshake done sender: [4:7577789350892453060:2120] self: [4:7577789350892453066:0] peer: [2:7577789352423429544:0] socket: 233 qp: -1
2025-11-28T14:52:52.229823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789353246086231:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:52.229329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.229388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789350892453072:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:52.229461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789354596802024:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.229442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.229826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:52.230151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789353246086140:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.230193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.229451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789350892453036:2109] [node 6] ICP22 created new session: [4:7577789350892453073:2109]
2025-11-28T14:52:52.230456Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789354600442020:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.230054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354596802024:2109] [node 6] ICP22 created new session: [3:7577789354596802063:2109]
2025-11-28T14:52:52.230220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789353246086140:2050] [node 3] ICP22 created new session: [6:7577789353246086239:2050]
2025-11-28T14:52:52.229658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789350892453073:2109] [node 6] ICS09 handshake done sender: [4:7577789350892453050:2115] self: [4:7577789350892453049:0] peer: [6:7577789353246086235:0] socket: 237 qp: -1
2025-11-28T14:52:52.230161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354596802063:2109] [node 6] ICS09 handshake done sender: [3:7577789354596802040:2117] self: [3:7577789354596802039:0] peer: [6:7577789353246086236:0] socket: 211 qp: -1
2025-11-28T14:52:52.229835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789350892453073:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:52.231635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789354600441933:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.230234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354596802063:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:52.230406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789353246086202:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.230264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.230140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.231757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:52.230433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:52.232196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789354600441933:2049] [node 2] ICP22 created new session: [5:7577789354600442035:2049]
2025-11-28T14:52:52.231052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789353246086239:2050] [node 3] ICS09 handshake done sender: [6:7577789353246086232:2122] self: [6:7577789353246086236:0] peer: [3:7577789354596802039:0] socket: 242 qp: -1
2025-11-28T14:52:52.232714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789354600442035:2049] [node 2] ICS09 handshake done sender: [5:7577789354600442020:2120] self: [5:7577789354600442029:0] peer: [2:7577789352423429555:0] socket: 235 qp: -1
2025-11-28T14:52:52.232750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789354600442035:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:52.231121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789353246086202:2107] [node 4] ICP22 created new session: [6:7577789353246086240:2107]
2025-11-28T14:52:52.232829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.231129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789353246086239:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:52.231735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.232187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789353246086240:2107] [node 4] ICS09 handshake done sender: [6:7577789353246086231:2121] self: [6:7577789353246086235:0] peer: [4:7577789350892453049:0] socket: 238 qp: -1
2025-11-28T14:52:52.232212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789353246086240:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:52.232490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:52.233497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789353246086202:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:52.280633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:52.280801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789352423429567:2048] [node 1] ICS07 socket disconnect 188 reason# EndOfStream
2025-11-28T14:52:52.280821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789352423429567:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.280900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.280938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789352423429567:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:52.289762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:52.289932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789354596802061:2048] [node 1] ICS01 socket: 207 reason# EndOfStream
2025-11-28T14:52:52.289966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789354596801959:2048] [node 1] ICP30 unregister session Session# [3:7577789354596802061:2048] VirtualId# [3:7577789354596802057:0]
2025-11-28T14:52:52.289996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789354596802061:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.290096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.290151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:52.290567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789352423429473:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:52.290601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789352423429567:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:52.291945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789352423429473:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:52.292019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789352423429567:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:52.292044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789352423429473:2048] [node 1] ICP30 unregister session Session# [2:7577789352423429567:2048] VirtualId# [2:7577789352423429559:0]
2025-11-28T14:52:52.292120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:52.292072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:52.292148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789352423429473:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.292158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.292274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789350892453057:2048] [node 1] ICS01 socket: 204 reason# EndOfStream
2025-11-28T14:52:52.292333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789350892452972:2048] [node 1] ICP30 unregister session Session# [4:7577789350892453057:2048] VirtualId# [4:7577789350892453051:0]
2025-11-28T14:52:52.292389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789350892453057:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.292476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.292535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:52.293000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:52.293218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789354600442030:2048] [node 1] ICS01 socket: 230 reason# EndOfStream
2025-11-28T14:52:52.293261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789354600441932:2048] [node 1] ICP30 unregister session Session# [5:7577789354600442030:2048] VirtualId# [5:7577789354600442021:0]
2025-11-28T14:52:52.293293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789354600442030:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.293373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.293465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:52.293966Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:52.294161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789353246086223:2048] [node 1] ICS01 socket: 194 reason# EndOfStream
2025-11-28T14:52:52.294206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789353246086138:2048] [node 1] ICP30 unregister session Session# [6:7577789353246086223:2048] VirtualId# [6:7577789353246086218:0]
2025-11-28T14:52:52.294245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789353246086223:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.294310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.294348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:52.353886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:52.354048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789354596802051:2049] [node 2] ICS01 socket: 199 reason# EndOfStream
2025-11-28T14:52:52.354082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789354596801960:2049] [node 2] ICP30 unregister session Session# [3:7577789354596802051:2049] VirtualId# [3:7577789354596802047:0]
2025-11-28T14:52:52.354111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789354596802051:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.354204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:52.354260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:52.355802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:52.356025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789350892453072:2049] [node 2] ICS01 socket: 233 reason# EndOfStream
2025-11-28T14:52:52.356084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789350892452973:2049] [node 2] ICP30 unregister session Session# [4:7577789350892453072:2049] VirtualId# [4:7577789350892453066:0]
2025-11-28T14:52:52.356141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789350892453072:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.356227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:52.356275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:52.356492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:52.356771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789354600442035:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-11-28T14:52:52.356836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789354600441933:2049] [node 2] ICP30 unregister session Session# [5:7577789354600442035:2049] VirtualId# [5:7577789354600442029:0]
2025-11-28T14:52:52.356879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789354600442035:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.356972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:52.357025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:52.357117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:52.357394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789353246086228:2049] [node 2] ICS01 socket: 201 reason# EndOfStream
2025-11-28T14:52:52.357446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789353246086139:2049] [node 2] ICP30 unregister session Session# [6:7577789353246086228:2049] VirtualId# [6:7577789353246086220:0]
2025-11-28T14:52:52.357479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789353246086228:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.357546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:52.357596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:52.410595Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:52.410844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789350892453061:2050] [node 3] ICS07 socket disconnect 205 reason# EndOfStream
2025-11-28T14:52:52.410884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789350892453061:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.410978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:52.411028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789350892453061:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:52.411427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:52.411568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789354600442016:2050] [node 3] ICS01 socket: 206 reason# EndOfStream
2025-11-28T14:52:52.411599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789354600441934:2050] [node 3] ICP30 unregister session Session# [5:7577789354600442016:2050] VirtualId# [5:7577789354600442013:0]
2025-11-28T14:52:52.411625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789354600442016:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.411700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:52.411759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789350892452974:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63653) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:52.411743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:52.411871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789350892453061:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:52.411908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789350892452974:2050] [node 3] ICP30 unregister session Session# [4:7577789350892453061:2050] VirtualId# [4:7577789350892453054:0]
2025-11-28T14:52:52.412011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:52.412064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789350892452974:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63653) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.412083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63653) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.412180Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:52.412319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789353246086239:2050] [node 3] ICS07 socket disconnect 242 reason# EndOfStream
2025-11-28T14:52:52.412349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789353246086239:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.412429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:52.412471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789353246086239:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:52.413341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789353246086140:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63653) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:52.413385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789353246086239:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:52.413466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789353246086140:2050] [node 3] ICP30 unregister session Session# [6:7577789353246086239:2050] VirtualId# [6:7577789353246086236:0]
2025-11-28T14:52:52.413545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:52.413582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789353246086140:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63653) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.413595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63653) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.423324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789353246086140:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:52.472304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:52.472494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789354600442032:2108] [node 4] ICS07 socket disconnect 182 reason# EndOfStream
2025-11-28T14:52:52.472526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789354600442032:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.472640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:52.472679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789354600442032:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:52.473032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:52.473166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789353246086240:2107] [node 4] ICS07 socket disconnect 238 reason# EndOfStream
2025-11-28T14:52:52.473191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789353246086240:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.473251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:52.473262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789354600441996:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19806) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:52.473299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789353246086240:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:52.473312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789354600442032:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:52.473459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789354600441996:2108] [node 4] ICP30 unregister session Session# [5:7577789354600442032:2108] VirtualId# [5:7577789354600442023:0]
2025-11-28T14:52:52.473586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:52.473612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789354600441996:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.473622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.474133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789353246086202:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19806) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:52.474169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789353246086240:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:52.474200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789353246086202:2107] [node 4] ICP30 unregister session Session# [6:7577789353246086240:2107] VirtualId# [6:7577789353246086235:0]
2025-11-28T14:52:52.474305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:52.474326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789353246086202:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.474339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:52.483930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789353246086202:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:52.536843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:52.537130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789353246086233:2105] [node 5] ICS01 socket: 227 reason# EndOfStream
2025-11-28T14:52:52.537204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789353246086199:2105] [node 5] ICP30 unregister session Session# [6:7577789353246086233:2105] VirtualId# [6:7577789353246086227:0]
2025-11-28T14:52:52.537248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789353246086233:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.537368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:52.537433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:52.630515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789351696378944:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:52.630732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789351696378945:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:52.648321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789351448371025:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.648530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789351448371026:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:52.667379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354520518246:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.667423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354520518247:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:52.686030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789351153714773:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.686231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789351153714775:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:52.686324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789351153714774:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:52.704644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789351421516911:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.704720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789351421516912:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:52.704723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789351421516913:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:52.723291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789354770791765:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:52.723318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789354770791766:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:52.723433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789354770791767:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:52.804704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789351448371088:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:52.811436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789354770791826:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:52.811580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354520518309:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:52.812059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789351153714838:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:52.811987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789351448371089:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:52.811994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789351421516975:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:52.812308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789351696379008:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:52.811642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354520518310:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:52.812707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789354770791829:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:52.812007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789351696379007:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:52.811894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789354520518311:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:52.812741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789351696379009:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:52.812831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789351153714839:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:52.812841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789351421516976:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:52.812861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789351448371090:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:52.825502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789351153714850:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3148
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.825556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789351421516990:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6972
2025-11-28T14:52:52.825848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791837:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
2025-11-28T14:52:52.825711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789351448371101:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42331
2025-11-28T14:52:52.826968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789351696379019:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61459
2025-11-28T14:52:52.826956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789351448371102:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42331
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.829863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789351448371102:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.829842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789351421516990:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.829876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379019:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.830002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791841:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
2025-11-28T14:52:52.829978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789351153714854:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3148
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.831807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354520518319:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.831612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789351421516913:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.830020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789351448371101:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.831803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789351696379021:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61459
2025-11-28T14:52:52.831837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789354770791837:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.831836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789351153714850:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.833211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789351448371104:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42331
2025-11-28T14:52:52.831845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789354520518319:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.833229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791845:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
2025-11-28T14:52:52.833961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379021:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.833214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789351153714855:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3148
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.833920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354520518326:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54646
2025-11-28T14:52:52.834130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789351696379025:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61459
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.834129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354520518319:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:52.833858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789354770791767:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.834463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789354770791841:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.834574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789351421516992:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6972
2025-11-28T14:52:52.834584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354520518325:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54646
2025-11-28T14:52:52.834738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789351153714855:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.834650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789354770791845:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.834586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789351421516993:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6972
2025-11-28T14:52:52.836388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789351153714774:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.834584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354520518320:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.836507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789351448371104:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.834726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789354770791829:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.834615Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789354520518320:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.836493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789351153714845:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.837415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789351448371099:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.836530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789354770791766:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.834711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789351421516991:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6972
2025-11-28T14:52:52.838490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379025:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.836389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789354520518311:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.836351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789351421516991:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.836518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789351153714845:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.837085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:52.840248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379016:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.837171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789351421516912:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.841967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791849:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.837442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789351448371099:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.837821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789351153714854:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.838567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789351421516993:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.837163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354520518320:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.837960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789351448371097:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.840265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789351696379016:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.838655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789351153714845:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.837526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789354520518309:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.838790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789351421516992:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.837985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789351448371097:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.842018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379013:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.839516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789351153714773:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.841902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789351421516911:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.837811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354520518311:2108] [node 5] ICP22 created new session: [3:7577789354520518329:2108]
2025-11-28T14:52:52.838514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789351448371096:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.842040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789351696379013:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.845966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789354770791837:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:52.842075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789351153714839:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.837880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354520518326:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.842653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789351421516995:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6972
2025-11-28T14:52:52.838536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789351448371096:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.842483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379018:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.847428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789354770791767:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.838353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:52.842520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:52.847035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789351421516990:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:52.842516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789351696379018:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.839632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789351448371099:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:52.847481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.838762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354520518329:2108] [node 5] ICS09 handshake done sender: [3:7577789354520518319:2111] self: [3:7577789354520518315:0] peer: [5:7577789351421516994:0] socket: 184 qp: -1
2025-11-28T14:52:52.842666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789351153714839:2110] [node 6] ICP22 created new session: [4:7577789351153714860:2110]
2025-11-28T14:52:52.848478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789351421516999:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6972
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.842669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789351448371096:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:52.843825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789351153714860:2110] [node 6] ICS09 handshake done sender: [4:7577789351153714845:2112] self: [4:7577789351153714843:0] peer: [6:7577789354770791847:0] socket: 194 qp: -1
No RDMA context found for GID: No RDMA context found for GID: 2025-11-28T14:52:52.843858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789351696379016:2112] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:52.847679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791850:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.848843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789351421516913:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.843902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789351448371097:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.839548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354520518329:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.847754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789354770791767:2050] [node 3] ICP22 created new session: [6:7577789354770791852:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.845692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789351696379009:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.844514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789351153714860:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:52.839567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354520518309:2109] [node 6] ICP22 created new session: [3:7577789354520518331:2109]
2025-11-28T14:52:52.845300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789351448371089:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.845930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:52.850372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789351421516912:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23461) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789351421516991:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.844540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.850340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789354770791852:2050] [node 3] ICS09 handshake done sender: [6:7577789354770791837:2113] self: [6:7577789354770791846:0] peer: [3:7577789354520518313:0] socket: 188 qp: -1
2025-11-28T14:52:52.840289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354520518325:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.845595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789351448371088:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.852643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.850961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789354770791852:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:52.846546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789351696379013:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:52.851333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789351153714773:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5104) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789351153714854:2118] held: no
2025-11-28T14:52:52.840354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789354520518246:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.845658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:52.846632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789351696379018:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:52.853055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789351421516913:2050] [node 3] ICP22 created new session: [5:7577789351421517001:2050]
2025-11-28T14:52:52.852635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789354770791841:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:52.854707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789351153714774:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23461) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789351153714855:2119] held: no
2025-11-28T14:52:52.840414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.845818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789351448371090:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.853336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789351421517001:2050] [node 3] ICS09 handshake done sender: [5:7577789351421516990:2116] self: [5:7577789351421516994:0] peer: [3:7577789354520518315:0] socket: 187 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.852878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.846782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789351696379009:2107] [node 4] ICP22 created new session: [1:7577789351696379033:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.856313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789351153714864:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3148
2025-11-28T14:52:52.841939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354520518331:2109] [node 6] ICS09 handshake done sender: [3:7577789354520518320:2113] self: [3:7577789354520518313:0] peer: [6:7577789354770791846:0] socket: 185 qp: -1
2025-11-28T14:52:52.846686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.854756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789351421517001:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:52.847541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789351696379007:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.853870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789354770791829:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.856413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789351153714862:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3148
2025-11-28T14:52:52.841971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354520518331:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:52.846723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789351448371088:2107] [node 4] ICP22 created new session: [2:7577789351448371113:2107]
2025-11-28T14:52:52.854792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.847593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.854738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791851:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
2025-11-28T14:52:52.842078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.859358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789351153714865:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3148
2025-11-28T14:52:52.847029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.854754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:52.855306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789351421516911:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5104) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789351421516993:2119] held: no
2025-11-28T14:52:52.847736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789351696379033:2107] [node 4] ICS09 handshake done sender: [1:7577789351696379016:2112] self: [1:7577789351696379012:0] peer: [4:7577789351153714859:0] socket: 203 qp: -1
2025-11-28T14:52:52.859930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789351153714864:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.854728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354520518337:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54646
2025-11-28T14:52:52.847335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789351448371090:2109] [node 6] ICP22 created new session: [2:7577789351448371114:2109]
2025-11-28T14:52:52.855202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354520518336:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54646
2025-11-28T14:52:52.847774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789351696379033:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:52.855248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789354770791829:2107] [node 4] ICP22 created new session: [6:7577789354770791855:2107]
2025-11-28T14:52:52.857114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789351421517000:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6972
2025-11-28T14:52:52.847780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789351696379007:2108] [node 5] ICP22 created new session: [1:7577789351696379034:2108]
2025-11-28T14:52:52.856504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354520518326:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:52.856284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789354770791855:2107] [node 4] ICS09 handshake done sender: [6:7577789354770791841:2116] self: [6:7577789354770791847:0] peer: [4:7577789351153714843:0] socket: 199 qp: -1
2025-11-28T14:52:52.847386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789351448371089:2108] [node 5] ICP22 created new session: [2:7577789351448371117:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.858520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789351421516991:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.861774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789351153714855:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.856448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789354770791855:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:52.848232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:52.848147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789351448371113:2107] [node 4] ICS09 handshake done sender: [2:7577789351448371099:2113] self: [2:7577789351448371091:0] peer: [4:7577789351153714857:0] socket: 205 qp: -1
2025-11-28T14:52:52.861354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789351421516912:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.856861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354520518338:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54646
2025-11-28T14:52:52.848468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789351696378945:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.856648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789354770791845:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.847781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789351448371117:2108] [node 5] ICS09 handshake done sender: [2:7577789351448371096:2110] self: [2:7577789351448371094:0] peer: [5:7577789351421516996:0] socket: 202 qp: -1
2025-11-28T14:52:52.861955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789351153714854:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:52.858488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789354520518246:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.861391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:52.856928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:52.848817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.848476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789351448371114:2109] [node 6] ICS09 handshake done sender: [2:7577789351448371097:2111] self: [2:7577789351448371095:0] peer: [6:7577789354770791848:0] socket: 201 qp: -1
2025-11-28T14:52:52.861969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789351153714775:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.858527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.857685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789354770791766:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.849476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789351696379034:2108] [node 5] ICS09 handshake done sender: [1:7577789351696379013:2110] self: [1:7577789351696379011:0] peer: [5:7577789351421516997:0] socket: 215 qp: -1
2025-11-28T14:52:52.861785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789351421516912:2049] [node 2] ICP22 created new session: [5:7577789351421517004:2049]
2025-11-28T14:52:52.848788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789351448371113:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:52.862855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789351153714774:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.858564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354520518246:2048] [node 1] ICP22 created new session: [3:7577789354520518339:2048]
2025-11-28T14:52:52.858503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:52.849482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789351696378945:2049] [node 3] ICP22 created new session: [1:7577789351696379036:2049]
2025-11-28T14:52:52.862167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789351421517004:2049] [node 2] ICS09 handshake done sender: [5:7577789351421516991:2117] self: [5:7577789351421516996:0] peer: [2:7577789351448371094:0] socket: 206 qp: -1
2025-11-28T14:52:52.848840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789351448371114:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:52.858745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789354770791766:2049] [node 2] ICP22 created new session: [6:7577789354770791858:2049]
2025-11-28T14:52:52.859255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354520518338:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.863160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789351153714773:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.859222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789354770791858:2049] [node 2] ICS09 handshake done sender: [6:7577789354770791845:2117] self: [6:7577789354770791848:0] peer: [2:7577789351448371095:0] socket: 204 qp: -1
2025-11-28T14:52:52.849513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789351696379034:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:52.862883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789351421517004:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:52.848854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789351448371117:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:52.859361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354520518339:2048] [node 1] ICS09 handshake done sender: [3:7577789354520518326:2116] self: [3:7577789354520518333:0] peer: [1:7577789351696379014:0] socket: 217 qp: -1
2025-11-28T14:52:52.863645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.849545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.848879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.859597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354520518339:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:52.859376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789354770791858:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:52.863176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.863903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:52.850413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789351696379036:2049] [node 3] ICS09 handshake done sender: [1:7577789351696379018:2113] self: [1:7577789351696379014:0] peer: [3:7577789354520518333:0] socket: 216 qp: -1
2025-11-28T14:52:52.859403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.859959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354520518336:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.863635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789351421516993:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:52.860018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.864618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789351421516911:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.848902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.861962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789354520518247:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.852655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789351696379036:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:52.864068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789351153714773:2048] [node 1] ICP22 created new session: [4:7577789351153714868:2048]
2025-11-28T14:52:52.866272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789351421517006:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6972
2025-11-28T14:52:52.864487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791857:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
2025-11-28T14:52:52.863883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789354520518321:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.852689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.849437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:52.866279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.863913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789354520518321:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.864087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789351153714774:2049] [node 2] ICP22 created new session: [4:7577789351153714869:2049]
2025-11-28T14:52:52.853004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789351696379029:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61459
2025-11-28T14:52:52.866331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789351421516911:2048] [node 1] ICP22 created new session: [5:7577789351421517007:2048]
2025-11-28T14:52:52.864459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789351153714869:2049] [node 2] ICS09 handshake done sender: [4:7577789351153714855:2119] self: [4:7577789351153714857:0] peer: [2:7577789351448371091:0] socket: 213 qp: -1
2025-11-28T14:52:52.866210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791860:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
2025-11-28T14:52:52.859611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379029:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.866370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789351421517007:2048] [node 1] ICS09 handshake done sender: [5:7577789351421516993:2119] self: [5:7577789351421516997:0] peer: [1:7577789351696379011:0] socket: 219 qp: -1
2025-11-28T14:52:52.866258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789354770791857:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.864597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789351153714868:2048] [node 1] ICS09 handshake done sender: [4:7577789351153714854:2118] self: [4:7577789351153714859:0] peer: [1:7577789351696379012:0] socket: 207 qp: -1
2025-11-28T14:52:52.862145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789351696379041:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61459
2025-11-28T14:52:52.866407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789351421517007:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:52.862146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789351448371121:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42331
2025-11-28T14:52:52.866404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789354770791765:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.866446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.866372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379041:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.865080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789351153714868:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:52.867113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789351421517006:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.867107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789354770791860:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.867124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379020:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.863155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789351448371122:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42331
2025-11-28T14:52:52.867141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789351696379020:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.868115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354520518321:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:52.865117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.868124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789354770791826:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.867234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789351421516975:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.868169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789351696379015:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.864282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789351448371098:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.868204Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789351696379015:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.864307Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789351448371098:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.865470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789351153714869:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:52.868756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789351421516986:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.865513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.866372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789351448371122:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.868884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789351696379020:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.870399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789354520518310:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.868715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789351153714848:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:52.868795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789351421516986:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:52.866479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789351448371025:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:52.870964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789351696379015:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.870640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:52.868743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789351153714848:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.867084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789351448371098:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:52.870825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354520518310:2107] [node 4] ICP22 created new session: [3:7577789354520518345:2107]
2025-11-28T14:52:52.868169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789351448371026:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.870896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789354520518309:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(28653619ns) {HandleHandshakeStatus:329(28650585ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(1964ns) SwitchToState:179(422ns) ProcessPendingSessionEvents:495(21274290ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20458731ns max# 1758979ns}}}
2025-11-28T14:52:52.868776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.870853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789351153714848:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:52.871638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354520518345:2107] [node 4] ICS09 handshake done sender: [3:7577789354520518321:2114] self: [3:7577789354520518318:0] peer: [4:7577789351153714867:0] socket: 229 qp: -1
2025-11-28T14:52:52.868927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789351448371026:2049] [node 3] ICP22 created new session: [2:7577789351448371125:2049]
2025-11-28T14:52:52.873414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789351696379008:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.872192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354520518345:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:52.870634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789351448371125:2049] [node 3] ICS09 handshake done sender: [2:7577789351448371098:2112] self: [2:7577789351448371092:0] peer: [3:7577789354520518341:0] socket: 186 qp: -1
2025-11-28T14:52:52.872637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789351153714838:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.874747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789351421516975:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19468) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789351421517006:2123] held: no
2025-11-28T14:52:52.873899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789354770791767:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(19272205ns) {HandleHandshakeStatus:329(19270422ns) {DropIncomingHandshake:442(1107ns) DropOutgoingHandshake:457(12212ns) SwitchToState:179(320ns) ProcessPendingSessionEvents:495(11545692ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10141928ns max# 1221419ns}}}
2025-11-28T14:52:52.872600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:52.873434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789351696378944:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.870751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789351448371125:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:52.874753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791863:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
2025-11-28T14:52:52.873615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789351153714775:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30421) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789351153714864:2121] held: no
2025-11-28T14:52:52.873730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789354520518311:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(29854235ns) {HandleHandshakeStatus:329(29851839ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(742ns) SwitchToState:179(305ns) ProcessPendingSessionEvents:495(24132935ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23994827ns max# 1755410ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.875119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789351421516986:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:52.870801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.873453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789351696378945:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17958313ns) {HandleHandshakeStatus:329(17956327ns) {DropIncomingHandshake:442(179ns) DropOutgoingHandshake:457(359ns) SwitchToState:179(290ns) ProcessPendingSessionEvents:495(11649075ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10899220ns max# 1335596ns}}}
2025-11-28T14:52:52.875457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789351421517010:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6972
2025-11-28T14:52:52.874013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789354520518309:2109] [node 6] ICP18 handshake (actor [3:7577789354520518325:2115]) is too old
2025-11-28T14:52:52.873679Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789351448371088:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21835058ns) {HandleHandshakeStatus:329(21832955ns) {DropIncomingHandshake:442(18623ns) DropOutgoingHandshake:457(641ns) SwitchToState:179(321ns) ProcessPendingSessionEvents:495(13348881ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13271141ns max# 1737849ns}}}
2025-11-28T14:52:52.873879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:52.874731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789354520518247:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23461) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789354520518338:2119] held: no
2025-11-28T14:52:52.873893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:52.873795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789351448371089:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21305366ns) {HandleHandshakeStatus:329(21303090ns) {DropIncomingHandshake:442(12919ns) DropOutgoingHandshake:457(491ns) SwitchToState:179(359ns) ProcessPendingSessionEvents:495(13934144ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13101865ns max# 1328614ns}}}
2025-11-28T14:52:52.875517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789354520518344:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54646
2025-11-28T14:52:52.874094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789351696378944:2048] [node 2] ICP22 created new session: [1:7577789351696379046:2048]
2025-11-28T14:52:52.875753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789351421516913:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17942453ns) {HandleHandshakeStatus:329(17940365ns) {DropIncomingHandshake:442(1569ns) DropOutgoingHandshake:457(5868ns) SwitchToState:179(263ns) ProcessPendingSessionEvents:495(9341251ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8545680ns max# 1833002ns}}}
2025-11-28T14:52:52.877757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789354770791765:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5104) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789354770791857:2121] held: no
2025-11-28T14:52:52.876595Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789351448371090:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(21208192ns) {HandleHandshakeStatus:329(21205913ns) {DropIncomingHandshake:442(1409ns) DropOutgoingHandshake:457(279ns) SwitchToState:179(382ns) ProcessPendingSessionEvents:495(13661818ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13577790ns max# 1357041ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:52.876498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789351421516976:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:52.877812Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789354770791857:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:52.877789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789351448371127:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42331
2025-11-28T14:52:52.874807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789351696379046:2048] [node 2] ICS09 handshake done sender: [1:7577789351696379015:2111] self: [1:7577789351696379010:0] peer: [2:7577789351448371124:0] socket: 195 qp: -1
2025-11-28T14:52:52.876748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:52.878622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789354770791864:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20559
2025-11-28T14:52:52.874839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789351696379008:2109] [node 6] ICP22 created new session: [1:7577789351696379045:2109]
2025-11-28T14:52:52.874069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789351153714839:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(24245306ns) {HandleHandshakeStatus:329(24243101ns) {DropIncomingHandshake:442(519ns) DropOutgoingHandshake:457(659ns) SwitchToState:179(142ns) ProcessPendingSessionEvents:495(18987840ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18892338ns max# 1355386ns}}}
2025-11-28T14:52:52.875126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789351696379046:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:52.877760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789351421516976:2109] [node 6] ICP22 created new session: [5:7577789351421517012:2109]
2025-11-28T14:52:52.874090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:52.875468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789351696379045:2109] [node 6] ICS09 handshake done sender: [1:7577789351696379020:2115] self: [1:7577789351696379017:0] peer: [6:7577789354770791861:0] socket: 196 qp: -1
2025-11-28T14:52:52.877798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789351421517012:2109] [node 6] ICS09 handshake done sender: [5:7577789351421516986:2115] self: [5:7577789351421516981:0] peer: [6:7577789354770791862:0] socket: 247 qp: -1
2025-11-28T14:52:52.874791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789351153714838:2108] [node 5] ICP22 created new session: [4:7577789351153714874:2108]
2025-11-28T14:52:52.875510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789351696379045:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:52.877824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789351421517012:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:52.875530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789351153714874:2108] [node 5] ICS09 handshake done sender: [4:7577789351153714848:2115] self: [4:7577789351153714842:0] peer: [5:7577789351421517009:0] socket: 230 qp: -1
2025-11-28T14:52:52.880922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789354770791826:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15387) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789354770791860:2122] held: no
2025-11-28T14:52:52.875838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789351696379009:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(20776621ns) {HandleHandshakeStatus:329(20774642ns) {DropIncomingHandshake:442(11106ns) DropOutgoingHandshake:457(3325ns) SwitchToState:179(484ns) ProcessPendingSessionEvents:495(15439093ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14233321ns max# 1548821ns}}}
2025-11-28T14:52:52.882419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789354770791860:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:52.881031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789354520518338:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:52.877846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.882533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789354770791765:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.879354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789351421517006:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:52.875928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789351696379007:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18843832ns) {HandleHandshakeStatus:329(18842452ns) {DropIncomingHandshake:442(12626ns) DropOutgoingHandshake:457(391ns) SwitchToState:179(151ns) ProcessPendingSessionEvents:495(13023908ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12167575ns max# 1746833ns}}}
2025-11-28T14:52:52.875553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789351153714872:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3148
2025-11-28T14:52:52.880208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789351421516975:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.876156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:52.880988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:52.883443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789351448371122:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:52.875754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789351153714874:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:52.876753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.882460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789354520518247:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.875958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.882561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.883167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789354770791765:2048] [node 1] ICP22 created new session: [6:7577789354770791865:2048]
2025-11-28T14:52:52.883564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789351448371025:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5104) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789351448371122:2119] held: no
2025-11-28T14:52:52.883601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789354770791865:2048] [node 1] ICS09 handshake done sender: [6:7577789354770791857:2121] self: [6:7577789354770791861:0] peer: [1:7577789351696379017:0] socket: 246 qp: -1
2025-11-28T14:52:52.882093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789351421516975:2108] [node 4] ICP22 created new session: [5:7577789351421517014:2108]
2025-11-28T14:52:52.883846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789354770791865:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:52.879593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789351153714864:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:52.883887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.882948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789351421517014:2108] [node 4] ICS09 handshake done sender: [5:7577789351421517006:2123] self: [5:7577789351421517009:0] peer: [4:7577789351153714842:0] socket: 182 qp: -1
2025-11-28T14:52:52.882490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:52.883284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789351421517014:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:52.883873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789351153714775:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.883553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:52.883904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:52.883142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789354520518247:2049] [node 2] ICP22 created new session: [3:7577789354520518347:2049]
2025-11-28T14:52:52.884046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789351153714775:2050] [node 3] ICP22 created new session: [4:7577789351153714876:2050]
2025-11-28T14:52:52.884088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789351153714876:2050] [node 3] ICS09 handshake done sender: [4:7577789351153714864:2121] self: [4:7577789351153714867:0] peer: [3:7577789354520518318:0] socket: 239 qp: -1
2025-11-28T14:52:52.883942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789354520518347:2049] [node 2] ICS09 handshake done sender: [3:7577789354520518338:2119] self: [3:7577789354520518341:0] peer: [2:7577789351448371092:0] socket: 197 qp: -1
2025-11-28T14:52:52.884260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789351153714876:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:52.884296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:52.884093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789354520518347:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:52.884279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:52.884341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789354770791826:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.884447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:52.884490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789354770791826:2105] [node 5] ICP22 created new session: [6:7577789354770791867:2105]
2025-11-28T14:52:52.884543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789354770791867:2105] [node 5] ICS09 handshake done sender: [6:7577789354770791860:2122] self: [6:7577789354770791862:0] peer: [5:7577789351421516981:0] socket: 248 qp: -1
2025-11-28T14:52:52.884544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789351448371025:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:52.884592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789354770791867:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:52.884584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:52.884609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789351448371025:2048] [node 1] ICP22 created new session: [2:7577789351448371128:2048]
2025-11-28T14:52:52.884641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789351448371128:2048] [node 1] ICS09 handshake done sender: [2:7577789351448371122:2119] self: [2:7577789351448371124:0] peer: [1:7577789351696379010:0] socket: 180 qp: -1
2025-11-28T14:52:52.884704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:52.884679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789351448371128:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:52.884712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:52.884873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789351448371090:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:52.884897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789351696378945:2049] [node 3] ICP18 handshake (actor [1:7577789351696379029:2118]) is too old
2025-11-28T14:52:52.953522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:52.953701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789351448371128:2048] [node 1] ICS01 socket: 180 reason# EndOfStream
2025-11-28T14:52:52.953757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789351448371025:2048] [node 1] ICP30 unregister session Session# [2:7577789351448371128:2048] VirtualId# [2:7577789351448371124:0]
2025-11-28T14:52:52.953782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789351448371128:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.953864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.953917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:52.954655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:52.954869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789354520518339:2048] [node 1] ICS01 socket: 217 reason# EndOfStream
2025-11-28T14:52:52.954966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789354520518246:2048] [node 1] ICP30 unregister session Session# [3:7577789354520518339:2048] VirtualId# [3:7577789354520518333:0]
2025-11-28T14:52:52.955000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789354520518339:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.955090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.955144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:52.956724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:52.956870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789351153714868:2048] [node 1] ICS01 socket: 207 reason# EndOfStream
2025-11-28T14:52:52.956915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789351153714773:2048] [node 1] ICP30 unregister session Session# [4:7577789351153714868:2048] VirtualId# [4:7577789351153714859:0]
2025-11-28T14:52:52.956941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789351153714868:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.957000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.957046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:52.957512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:52.957657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789351421517007:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-28T14:52:52.957690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789351421516911:2048] [node 1] ICP30 unregister session Session# [5:7577789351421517007:2048] VirtualId# [5:7577789351421516997:0]
2025-11-28T14:52:52.957710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789351421517007:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.957771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.957812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:52.958226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:52.958438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789354770791865:2048] [node 1] ICS01 socket: 246 reason# EndOfStream
2025-11-28T14:52:52.958486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789354770791765:2048] [node 1] ICP30 unregister session Session# [6:7577789354770791865:2048] VirtualId# [6:7577789354770791861:0]
2025-11-28T14:52:52.958526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789354770791865:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:52.958605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:52.958654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:53.007617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:53.007839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789354520518347:2049] [node 2] ICS01 socket: 197 reason# EndOfStream
2025-11-28T14:52:53.007886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789354520518247:2049] [node 2] ICP30 unregister session Session# [3:7577789354520518347:2049] VirtualId# [3:7577789354520518341:0]
2025-11-28T14:52:53.007917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789354520518347:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.008011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:53.008059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:53.010389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:53.010550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789351153714869:2049] [node 2] ICS01 socket: 213 reason# EndOfStream
2025-11-28T14:52:53.010585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789351153714774:2049] [node 2] ICP30 unregister session Session# [4:7577789351153714869:2049] VirtualId# [4:7577789351153714857:0]
2025-11-28T14:52:53.010609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789351153714869:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.010711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:53.010751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:53.011143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:53.011279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789351421517004:2049] [node 2] ICS01 socket: 206 reason# EndOfStream
2025-11-28T14:52:53.011310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789351421516912:2049] [node 2] ICP30 unregister session Session# [5:7577789351421517004:2049] VirtualId# [5:7577789351421516996:0]
2025-11-28T14:52:53.011330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789351421517004:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.011397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:53.011438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:53.011823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:53.012038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789354770791858:2049] [node 2] ICS01 socket: 204 reason# EndOfStream
2025-11-28T14:52:53.012079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789354770791766:2049] [node 2] ICP30 unregister session Session# [6:7577789354770791858:2049] VirtualId# [6:7577789354770791848:0]
2025-11-28T14:52:53.012108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789354770791858:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.012195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:53.012260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:53.077712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:53.077877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789351153714876:2050] [node 3] ICS01 socket: 239 reason# EndOfStream
2025-11-28T14:52:53.077914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789351153714775:2050] [node 3] ICP30 unregister session Session# [4:7577789351153714876:2050] VirtualId# [4:7577789351153714867:0]
2025-11-28T14:52:53.077940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789351153714876:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.078042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:53.078106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:53.078662Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:53.078915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789351421517001:2050] [node 3] ICS07 socket disconnect 187 reason# EndOfStream
2025-11-28T14:52:53.078950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789351421517001:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.079039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:53.079083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789351421517001:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:53.079332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:53.079544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789354770791852:2050] [node 3] ICS07 socket disconnect 188 reason# EndOfStream
2025-11-28T14:52:53.079607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789354770791852:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.079676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:53.079726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789354770791852:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:53.079857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789351421516913:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30421) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.079993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789351421517001:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.080054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789351421516913:2050] [node 3] ICP30 unregister session Session# [5:7577789351421517001:2050] VirtualId# [5:7577789351421516994:0]
2025-11-28T14:52:53.080104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:53.080144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789351421516913:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.080161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.080485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789354770791767:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30421) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.080566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789354770791852:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.080614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789354770791767:2050] [node 3] ICP30 unregister session Session# [6:7577789354770791852:2050] VirtualId# [6:7577789354770791846:0]
2025-11-28T14:52:53.080739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:53.080776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789354770791767:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.080789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.089948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789351421516913:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:53.090959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789354770791767:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:53.128769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:53.128953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789351421517014:2108] [node 4] ICS01 socket: 182 reason# EndOfStream
2025-11-28T14:52:53.128984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789351421516975:2108] [node 4] ICP30 unregister session Session# [5:7577789351421517014:2108] VirtualId# [5:7577789351421517009:0]
2025-11-28T14:52:53.129005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789351421517014:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.129064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:53.129100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:53.129834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:53.130070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789354770791855:2107] [node 4] ICS01 socket: 199 reason# EndOfStream
2025-11-28T14:52:53.130129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789354770791829:2107] [node 4] ICP30 unregister session Session# [6:7577789354770791855:2107] VirtualId# [6:7577789354770791847:0]
2025-11-28T14:52:53.130172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789354770791855:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.130276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:53.130328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:53.183409Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:53.183671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789354770791867:2105] [node 5] ICS01 socket: 248 reason# EndOfStream
2025-11-28T14:52:53.183739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789354770791826:2105] [node 5] ICP30 unregister session Session# [6:7577789354770791867:2105] VirtualId# [6:7577789354770791862:0]
2025-11-28T14:52:53.183783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789354770791867:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.183906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:53.183974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:53.271953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789358467749235:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:53.271992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789358467749236:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:53.288685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789357514982261:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.288736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789357514982262:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:53.305095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789357472735966:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.305133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789357472735967:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:53.322078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355903637429:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.322129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355903637430:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:53.322158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355903637431:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:53.339231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789356432534307:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.339273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789356432534308:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:53.339307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789356432534309:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:53.359541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789358293144000:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.359579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789358293144001:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:53.359602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789358293144002:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:53.415555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789358293144059:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:53.445768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789358467749298:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:53.445946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789357514982325:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:53.446067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789357472736029:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:53.446711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789357472736030:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:53.446829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789357514982326:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:53.446811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355903637492:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:53.446835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789358467749299:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:53.447085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789358293144064:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:53.447052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789356432534370:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:53.447141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789357472736031:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:53.447210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355903637493:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:53.447314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789358467749300:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:53.447404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789357514982327:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:53.447682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789356432534371:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.456001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789358467749314:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1586
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.456234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355903637507:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24362
2025-11-28T14:52:53.456297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355903637508:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24362
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.457912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789357472736045:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41595
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.461841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789356432534382:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4137
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.462104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789357514982338:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7762
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.462576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789357514982339:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7762
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.462636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789358467749315:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1586
2025-11-28T14:52:53.462697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789357514982340:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7762
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.463211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355903637507:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.463190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789357472736046:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41595
2025-11-28T14:52:53.463273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355903637508:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.463421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789357472736045:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.463471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789357514982339:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.463480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789357472736047:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41595
2025-11-28T14:52:53.463494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789355903637431:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.463718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789355903637429:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.463700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789357472736046:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.463816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749315:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:53.464074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789357514982341:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7762
::ffff:127.1.0.0
2025-11-28T14:52:53.464075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749314:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.464100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789357472736048:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41595
2025-11-28T14:52:53.464269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789357472735966:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.464270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789357514982340:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.464299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355903637509:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24362
2025-11-28T14:52:53.464394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789357514982338:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.464410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789357472736037:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.464430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789357472736037:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.464507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749306:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.464534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789358467749306:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.464836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789357514982345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7762
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.465252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789357472736037:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:53.465529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789358467749316:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1586
2025-11-28T14:52:53.465552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789357514982261:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.465663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355903637512:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24362
2025-11-28T14:52:53.465695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789358467749317:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1586
2025-11-28T14:52:53.465745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789357472736029:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.465813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:53.465873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789357472736029:2107] [node 4] ICP22 created new session: [3:7577789357472736052:2107]
2025-11-28T14:52:53.465926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789357472736052:2107] [node 4] ICS09 handshake done sender: [3:7577789357472736037:2112] self: [3:7577789357472736036:0] peer: [4:7577789355903637510:0] socket: 190 qp: -1
2025-11-28T14:52:53.465962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789357472736052:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:53.466001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749304:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.466037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789358467749304:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.466011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:53.466091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355903637507:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:53.466657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789355903637431:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.466686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:53.466758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749316:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.467029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355903637431:2050] [node 3] ICP22 created new session: [4:7577789355903637513:2050]
2025-11-28T14:52:53.467087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749302:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.467098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355903637513:2050] [node 3] ICS09 handshake done sender: [4:7577789355903637507:2115] self: [4:7577789355903637510:0] peer: [3:7577789357472736036:0] socket: 196 qp: -1
2025-11-28T14:52:53.467098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789358467749302:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.467119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355903637513:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:53.467211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: 2025-11-28T14:52:53.467661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789357514982341:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:53.467775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789358467749318:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1586
2025-11-28T14:52:53.468130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789358467749306:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:53.468605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789358467749304:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:53.468536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789356432534382:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.469159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789358467749298:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.469195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:53.469219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789358467749298:2107] [node 4] ICP22 created new session: [1:7577789358467749325:2107]
2025-11-28T14:52:53.469251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789358467749325:2107] [node 4] ICS09 handshake done sender: [1:7577789358467749306:2112] self: [1:7577789358467749305:0] peer: [4:7577789355903637511:0] socket: 191 qp: -1
2025-11-28T14:52:53.469276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789358467749325:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:53.469298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:53.470079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789358467749236:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.470264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:53.470392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789358467749236:2049] [node 3] ICP22 created new session: [1:7577789358467749327:2049]
2025-11-28T14:52:53.470416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789358467749327:2049] [node 3] ICS09 handshake done sender: [1:7577789358467749304:2111] self: [1:7577789358467749303:0] peer: [3:7577789357472736049:0] socket: 189 qp: -1
2025-11-28T14:52:53.470436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789358467749327:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:53.470451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:53.470495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789357472736047:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.470724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789357472736054:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41595
2025-11-28T14:52:53.470776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355903637515:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24362
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.471045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789356432534387:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4137
2025-11-28T14:52:53.471131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789357472736045:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:53.471201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355903637508:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:53.471333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789357472735966:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.471512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789355903637429:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.471410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:53.471540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:53.471431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789357472735966:2048] [node 1] ICP22 created new session: [3:7577789357472736057:2048]
2025-11-28T14:52:53.471643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355903637429:2048] [node 1] ICP22 created new session: [4:7577789355903637517:2048]
2025-11-28T14:52:53.471765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789356432534388:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4137
2025-11-28T14:52:53.471591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789357472736057:2048] [node 1] ICS09 handshake done sender: [3:7577789357472736045:2115] self: [3:7577789357472736049:0] peer: [1:7577789358467749303:0] socket: 193 qp: -1
2025-11-28T14:52:53.471795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355903637517:2048] [node 1] ICS09 handshake done sender: [4:7577789355903637508:2116] self: [4:7577789355903637511:0] peer: [1:7577789358467749305:0] socket: 197 qp: -1
2025-11-28T14:52:53.471785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749317:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.471610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789357472736057:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.471809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355903637517:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.471825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.471629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.471986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789358467749302:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.472054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789357514982345:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.472865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789357514982349:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7762
2025-11-28T14:52:53.473012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789358467749235:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.473068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.473098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789358467749235:2048] [node 2] ICP22 created new session: [1:7577789358467749331:2048]
2025-11-28T14:52:53.473126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789358467749331:2048] [node 2] ICS09 handshake done sender: [1:7577789358467749302:2110] self: [1:7577789358467749301:0] peer: [2:7577789357514982348:0] socket: 186 qp: -1
2025-11-28T14:52:53.473156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789358467749331:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:53.473176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:53.473419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789357514982338:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:53.473685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789357514982261:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.473736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:53.473775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789357514982261:2048] [node 1] ICP22 created new session: [2:7577789357514982352:2048]
2025-11-28T14:52:53.473828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789357514982352:2048] [node 1] ICS09 handshake done sender: [2:7577789357514982338:2116] self: [2:7577789357514982348:0] peer: [1:7577789358467749301:0] socket: 205 qp: -1
2025-11-28T14:52:53.473865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789357514982352:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.473894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.475432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749318:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.476165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789357472736048:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.476598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789356432534388:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.476721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789356432534387:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.477164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789357472735967:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.477806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355903637509:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.477869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789356432534370:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.477893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789355903637430:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.477948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789356432534309:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.477944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144078:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
2025-11-28T14:52:53.478019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789356432534389:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4137
2025-11-28T14:52:53.478025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144079:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.478165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144080:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
2025-11-28T14:52:53.478167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789357514982331:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.478200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789357514982331:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.478256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355903637501:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.478276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789358293144079:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.478275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789355903637501:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.479123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789358293144064:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.479294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789357472736039:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.479372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789357514982333:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.479313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789357472736039:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.479397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789357514982333:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.479404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355903637503:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.479422Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789355903637503:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.480619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789358467749235:2048] [node 2] ICP18 handshake (actor [1:7577789358467749318:2119]) is too old
2025-11-28T14:52:53.480773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355903637501:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:53.480859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789358293144080:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.481237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355903637503:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:53.481258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144082:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
2025-11-28T14:52:53.481238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789357472736039:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:53.481278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789357514982331:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.481702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789357472736030:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.481857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.481887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789357472736030:2108] [node 5] ICP22 created new session: [3:7577789357472736063:2108]
2025-11-28T14:52:53.481848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789356432534392:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4137
2025-11-28T14:52:53.481913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789355903637492:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.481919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789357514982333:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:53.481945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.481913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789357472736063:2108] [node 5] ICS09 handshake done sender: [3:7577789357472736039:2113] self: [3:7577789357472736038:0] peer: [5:7577789356432534391:0] socket: 233 qp: -1
2025-11-28T14:52:53.481984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355903637492:2108] [node 5] ICP22 created new session: [4:7577789355903637524:2108]
2025-11-28T14:52:53.482063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355903637524:2108] [node 5] ICS09 handshake done sender: [4:7577789355903637501:2113] self: [4:7577789355903637500:0] peer: [5:7577789356432534390:0] socket: 235 qp: -1
2025-11-28T14:52:53.481933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789357472736063:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:53.482109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355903637524:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:53.481963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:53.482174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:53.482353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789358293144002:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.482412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789357514982262:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.482474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:53.482505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789358293144079:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:53.482522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789357514982262:2049] [node 3] ICP22 created new session: [2:7577789357514982357:2049]
2025-11-28T14:52:53.482563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789357514982357:2049] [node 3] ICS09 handshake done sender: [2:7577789357514982331:2112] self: [2:7577789357514982330:0] peer: [3:7577789357472736060:0] socket: 222 qp: -1
2025-11-28T14:52:53.482642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789357514982357:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.482675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:53.482824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789358293144064:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.482833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789356432534393:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4137
2025-11-28T14:52:53.483043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:53.483085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789358293144064:2107] [node 4] ICP22 created new session: [6:7577789358293144084:2107]
2025-11-28T14:52:53.483130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789358293144084:2107] [node 4] ICS09 handshake done sender: [6:7577789358293144079:2116] self: [6:7577789358293144081:0] peer: [4:7577789355903637502:0] socket: 238 qp: -1
2025-11-28T14:52:53.483160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789358293144084:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:53.483288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:53.483492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789355903637493:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.483474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789356432534394:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4137
2025-11-28T14:52:53.483452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789357472736062:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41595
2025-11-28T14:52:53.483515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.483531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789356432534388:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:53.483542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355903637493:2109] [node 6] ICP22 created new session: [4:7577789355903637526:2109]
2025-11-28T14:52:53.483572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355903637526:2109] [node 6] ICS09 handshake done sender: [4:7577789355903637503:2114] self: [4:7577789355903637502:0] peer: [6:7577789358293144081:0] socket: 192 qp: -1
2025-11-28T14:52:53.483586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355903637526:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:53.483600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:53.483884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789356432534389:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.483927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789356432534370:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.483988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:53.484051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789356432534370:2108] [node 4] ICP22 created new session: [5:7577789356432534395:2108]
2025-11-28T14:52:53.484113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789356432534395:2108] [node 4] ICS09 handshake done sender: [5:7577789356432534388:2117] self: [5:7577789356432534390:0] peer: [4:7577789355903637500:0] socket: 236 qp: -1
2025-11-28T14:52:53.484150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789356432534395:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:53.484165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789357472736041:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.484179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:53.484181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789357472736041:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.484198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789357514982325:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.484243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:53.484276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789357514982325:2107] [node 4] ICP22 created new session: [2:7577789357514982359:2107]
2025-11-28T14:52:53.484327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789357514982359:2107] [node 4] ICS09 handshake done sender: [2:7577789357514982333:2113] self: [2:7577789357514982332:0] peer: [4:7577789355903637520:0] socket: 223 qp: -1
2025-11-28T14:52:53.484665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789357514982359:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:53.484697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:53.484793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789357472736048:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.485105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789357472735967:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.485140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.485163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789357472735967:2049] [node 2] ICP22 created new session: [3:7577789357472736066:2049]
2025-11-28T14:52:53.485193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789357472736066:2049] [node 2] ICS09 handshake done sender: [3:7577789357472736048:2118] self: [3:7577789357472736060:0] peer: [2:7577789357514982330:0] socket: 224 qp: -1
2025-11-28T14:52:53.485213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789357472736066:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:53.485229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.485517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355903637523:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24362
2025-11-28T14:52:53.486315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789358293144078:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.486575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789356432534387:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.487361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
2025-11-28T14:52:53.487435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789358293144000:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.487595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789357472736041:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:53.487658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749310:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.487689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789358467749310:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.489089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789357472736031:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.489122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.489149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789357472736031:2109] [node 6] ICP22 created new session: [3:7577789357472736069:2109]
2025-11-28T14:52:53.489186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789357472736069:2109] [node 6] ICS09 handshake done sender: [3:7577789357472736041:2114] self: [3:7577789357472736040:0] peer: [6:7577789358293144083:0] socket: 240 qp: -1
2025-11-28T14:52:53.489206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789357472736069:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:53.489273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:53.489361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789358467749310:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:53.489990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355903637509:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.490013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789358293144080:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:53.489982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789356432534309:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.490124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:53.490096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789358467749300:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.490175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789356432534309:2050] [node 3] ICP22 created new session: [5:7577789356432534397:2050]
2025-11-28T14:52:53.490252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789355903637430:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21022) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789355903637509:2117] held: no
2025-11-28T14:52:53.490258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.490278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789356432534397:2050] [node 3] ICS09 handshake done sender: [5:7577789356432534387:2116] self: [5:7577789356432534391:0] peer: [3:7577789357472736038:0] socket: 234 qp: -1
2025-11-28T14:52:53.490304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789358467749300:2109] [node 6] ICP22 created new session: [1:7577789358467749335:2109]
2025-11-28T14:52:53.490325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789356432534397:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:53.490348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789358467749335:2109] [node 6] ICS09 handshake done sender: [1:7577789358467749310:2114] self: [1:7577789358467749309:0] peer: [6:7577789358293144088:0] socket: 194 qp: -1
2025-11-28T14:52:53.490369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:53.490376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789358467749335:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:53.490409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:53.490392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789358293144002:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.490469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:53.490536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789358293144002:2050] [node 3] ICP22 created new session: [6:7577789358293144090:2050]
2025-11-28T14:52:53.490577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789358293144090:2050] [node 3] ICS09 handshake done sender: [6:7577789358293144080:2117] self: [6:7577789358293144083:0] peer: [3:7577789357472736040:0] socket: 221 qp: -1
2025-11-28T14:52:53.490603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789358293144090:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:53.490612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789355903637430:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.490643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.490666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355903637430:2049] [node 2] ICP22 created new session: [4:7577789355903637528:2049]
2025-11-28T14:52:53.490666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:53.490692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355903637528:2049] [node 2] ICS09 handshake done sender: [4:7577789355903637509:2117] self: [4:7577789355903637520:0] peer: [2:7577789357514982332:0] socket: 225 qp: -1
2025-11-28T14:52:53.490714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355903637528:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:53.490736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.490883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789357472736068:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41595
2025-11-28T14:52:53.491052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789356432534309:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:53.492924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789356432534307:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.493250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789357472736031:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:53.493275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789358467749308:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.493304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789358467749308:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.493324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789356432534394:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.494142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789356432534308:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.494265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789358467749308:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:53.494721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789357514982335:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.494755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789357514982335:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.495011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789358467749299:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.495085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.495136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789358467749299:2108] [node 5] ICP22 created new session: [1:7577789358467749338:2108]
2025-11-28T14:52:53.495185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789358467749338:2108] [node 5] ICS09 handshake done sender: [1:7577789358467749308:2113] self: [1:7577789358467749307:0] peer: [5:7577789356432534399:0] socket: 187 qp: -1
2025-11-28T14:52:53.495209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789358467749338:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.495281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:53.495346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144089:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
2025-11-28T14:52:53.495763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789357514982335:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:53.496237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789358293144000:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789358293144078:2115] held: no
2025-11-28T14:52:53.496278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789357514982326:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.496327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.496380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789357514982326:2109] [node 5] ICP22 created new session: [2:7577789357514982363:2109]
2025-11-28T14:52:53.496429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789357514982363:2109] [node 5] ICS09 handshake done sender: [2:7577789357514982335:2114] self: [2:7577789357514982334:0] peer: [5:7577789356432534401:0] socket: 248 qp: -1
2025-11-28T14:52:53.496462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789357514982363:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.496518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:53.496563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789356432534402:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4137
2025-11-28T14:52:53.497297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789358293144078:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.497889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144092:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
2025-11-28T14:52:53.497969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789358293144000:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.497970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789356432534403:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4137
2025-11-28T14:52:53.498015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:53.498045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789358293144000:2048] [node 1] ICP22 created new session: [6:7577789358293144094:2048]
2025-11-28T14:52:53.498076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789358293144094:2048] [node 1] ICS09 handshake done sender: [6:7577789358293144078:2115] self: [6:7577789358293144088:0] peer: [1:7577789358467749309:0] socket: 199 qp: -1
2025-11-28T14:52:53.498098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789358293144094:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.498134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.498452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789356432534389:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:53.498652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789356432534307:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789356432534389:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.499159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789358293144092:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.499222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144093:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
2025-11-28T14:52:53.499177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789356432534307:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.499247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:53.499289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789356432534307:2048] [node 1] ICP22 created new session: [5:7577789356432534404:2048]
2025-11-28T14:52:53.499330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789356432534404:2048] [node 1] ICS09 handshake done sender: [5:7577789356432534389:2118] self: [5:7577789356432534399:0] peer: [1:7577789358467749307:0] socket: 239 qp: -1
2025-11-28T14:52:53.499363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789358293144001:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.499365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789356432534404:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.499391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.499567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789357514982337:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.499595Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789357514982337:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.500562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789357514982337:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.501174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144097:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
2025-11-28T14:52:53.501297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789357514982327:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.501385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.501436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789357514982327:2110] [node 6] ICP22 created new session: [2:7577789357514982366:2110]
2025-11-28T14:52:53.501479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789357514982366:2110] [node 6] ICS09 handshake done sender: [2:7577789357514982337:2115] self: [2:7577789357514982336:0] peer: [6:7577789358293144096:0] socket: 226 qp: -1
2025-11-28T14:52:53.501408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789356432534308:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21022) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789356432534394:2121] held: no
2025-11-28T14:52:53.501504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789357514982366:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:53.501558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:53.501587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789358293144092:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.502022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789358293144001:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.502073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.502106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789358293144001:2049] [node 2] ICP22 created new session: [6:7577789358293144098:2049]
2025-11-28T14:52:53.502131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789358293144098:2049] [node 2] ICS09 handshake done sender: [6:7577789358293144092:2121] self: [6:7577789358293144096:0] peer: [2:7577789357514982336:0] socket: 207 qp: -1
2025-11-28T14:52:53.502156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789358293144098:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:53.502176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:53.502898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789356432534394:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.503465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789356432534308:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.503556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.503613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789356432534308:2049] [node 2] ICP22 created new session: [5:7577789356432534406:2049]
2025-11-28T14:52:53.503664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789356432534406:2049] [node 2] ICS09 handshake done sender: [5:7577789356432534394:2121] self: [5:7577789356432534401:0] peer: [2:7577789357514982334:0] socket: 251 qp: -1
2025-11-28T14:52:53.503697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789356432534406:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:53.503728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:53.504204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789358293144093:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.504545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789358293144059:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.530913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789356432534381:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.530954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789356432534381:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.531966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789356432534381:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.532242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789358293144101:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38684
2025-11-28T14:52:53.532312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789356432534371:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.532398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.532446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789356432534371:2109] [node 6] ICP22 created new session: [5:7577789356432534409:2109]
2025-11-28T14:52:53.532493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789356432534409:2109] [node 6] ICS09 handshake done sender: [5:7577789356432534381:2114] self: [5:7577789356432534380:0] peer: [6:7577789358293144100:0] socket: 188 qp: -1
2025-11-28T14:52:53.532526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789356432534409:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:53.532592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:53.532681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789358293144093:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:53.532905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789358293144059:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.532975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.533020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789358293144059:2105] [node 5] ICP22 created new session: [6:7577789358293144102:2105]
2025-11-28T14:52:53.533064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789358293144102:2105] [node 5] ICS09 handshake done sender: [6:7577789358293144093:2122] self: [6:7577789358293144100:0] peer: [5:7577789356432534380:0] socket: 255 qp: -1
2025-11-28T14:52:53.533101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789358293144102:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:53.533132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:53.550049Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:53.550275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789357514982352:2048] [node 1] ICS07 socket disconnect 205 reason# EndOfStream
2025-11-28T14:52:53.550312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789357514982352:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.550449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:53.550516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789357514982352:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:53.550981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:53.551124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789357472736057:2048] [node 1] ICS01 socket: 193 reason# EndOfStream
2025-11-28T14:52:53.551155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789357472735966:2048] [node 1] ICP30 unregister session Session# [3:7577789357472736057:2048] VirtualId# [3:7577789357472736049:0]
2025-11-28T14:52:53.551180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789357472736057:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.551241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:53.551292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:53.553495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:53.553655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789355903637517:2048] [node 1] ICS01 socket: 197 reason# EndOfStream
2025-11-28T14:52:53.553690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789355903637429:2048] [node 1] ICP30 unregister session Session# [4:7577789355903637517:2048] VirtualId# [4:7577789355903637511:0]
2025-11-28T14:52:53.553723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789355903637517:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.553791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:53.553829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:53.553878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789357514982261:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12806) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:53.553923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789357514982352:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:53.554824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789357514982261:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12806) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.554881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789357514982352:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.554913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789357514982261:2048] [node 1] ICP30 unregister session Session# [2:7577789357514982352:2048] VirtualId# [2:7577789357514982348:0]
2025-11-28T14:52:53.554869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:53.555016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:53.555051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789357514982261:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.555063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.555062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789356432534404:2048] [node 1] ICS01 socket: 239 reason# EndOfStream
2025-11-28T14:52:53.555106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789356432534307:2048] [node 1] ICP30 unregister session Session# [5:7577789356432534404:2048] VirtualId# [5:7577789356432534399:0]
2025-11-28T14:52:53.555146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789356432534404:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.555213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:53.555264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:53.555992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:53.556132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789358293144094:2048] [node 1] ICS01 socket: 199 reason# EndOfStream
2025-11-28T14:52:53.556161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789358293144000:2048] [node 1] ICP30 unregister session Session# [6:7577789358293144094:2048] VirtualId# [6:7577789358293144088:0]
2025-11-28T14:52:53.556181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789358293144094:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.556231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:53.556262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:53.578744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:53.578949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789357472736066:2049] [node 2] ICS01 socket: 224 reason# EndOfStream
2025-11-28T14:52:53.578985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789357472735967:2049] [node 2] ICP30 unregister session Session# [3:7577789357472736066:2049] VirtualId# [3:7577789357472736060:0]
2025-11-28T14:52:53.579005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789357472736066:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.579077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:53.579119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:53.581673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:53.581882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789355903637528:2049] [node 2] ICS01 socket: 225 reason# EndOfStream
2025-11-28T14:52:53.581915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789355903637430:2049] [node 2] ICP30 unregister session Session# [4:7577789355903637528:2049] VirtualId# [4:7577789355903637520:0]
2025-11-28T14:52:53.581934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789355903637528:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.581985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:53.582011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:53.582848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:53.583044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789356432534406:2049] [node 2] ICS01 socket: 251 reason# EndOfStream
2025-11-28T14:52:53.583086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789356432534308:2049] [node 2] ICP30 unregister session Session# [5:7577789356432534406:2049] VirtualId# [5:7577789356432534401:0]
2025-11-28T14:52:53.583117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789356432534406:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.583178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:53.583219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:53.583831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:53.583973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789358293144098:2049] [node 2] ICS07 socket disconnect 207 reason# EndOfStream
2025-11-28T14:52:53.583998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789358293144098:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.584048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:53.584076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789358293144098:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:53.584711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789358293144001:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21022) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.584747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789358293144098:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.584769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789358293144001:2049] [node 2] ICP30 unregister session Session# [6:7577789358293144098:2049] VirtualId# [6:7577789358293144096:0]
2025-11-28T14:52:53.584840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:53.584864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789358293144001:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21022) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.584873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21022) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.594760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789358293144001:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:53.621407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:53.621573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789355903637513:2050] [node 3] ICS07 socket disconnect 196 reason# EndOfStream
2025-11-28T14:52:53.621597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789355903637513:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.621667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:53.621707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789355903637513:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:53.622184Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:53.622247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789355903637431:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31650) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.622281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789355903637513:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.622302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789355903637431:2050] [node 3] ICP30 unregister session Session# [4:7577789355903637513:2050] VirtualId# [4:7577789355903637510:0]
2025-11-28T14:52:53.622388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:53.622414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789355903637431:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.622406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789356432534397:2050] [node 3] ICS07 socket disconnect 234 reason# EndOfStream
2025-11-28T14:52:53.622422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.622445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789356432534397:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.622509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:53.622559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789356432534397:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:53.623010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:53.623135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789358293144090:2050] [node 3] ICS07 socket disconnect 221 reason# EndOfStream
2025-11-28T14:52:53.623156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789358293144090:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.623209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:53.623238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789358293144090:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:53.623302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789356432534309:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31650) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.623354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789356432534397:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.623390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789356432534309:2050] [node 3] ICP30 unregister session Session# [5:7577789356432534397:2050] VirtualId# [5:7577789356432534391:0]
2025-11-28T14:52:53.623482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:53.623510Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789356432534309:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.623525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.623814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789358293144002:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31650) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.623846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789358293144090:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.623870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789358293144002:2050] [node 3] ICP30 unregister session Session# [6:7577789358293144090:2050] VirtualId# [6:7577789358293144083:0]
2025-11-28T14:52:53.623926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:53.623942Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789358293144002:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.623950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.633685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789358293144002:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:53.633685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789356432534309:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:53.645508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:53.645689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789356432534395:2108] [node 4] ICS07 socket disconnect 236 reason# EndOfStream
2025-11-28T14:52:53.645726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789356432534395:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.645790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:53.645821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789356432534395:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:53.646157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:53.646288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789358293144084:2107] [node 4] ICS07 socket disconnect 238 reason# EndOfStream
2025-11-28T14:52:53.646312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789358293144084:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.646385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:53.646421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789358293144084:2107] [node 4] ICS15 start handshake
2025-11-28T14:52:53.646474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789356432534370:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10847) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.646567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789356432534395:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.646599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789356432534370:2108] [node 4] ICP30 unregister session Session# [5:7577789356432534395:2108] VirtualId# [5:7577789356432534390:0]
2025-11-28T14:52:53.646671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:53.646691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789356432534370:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.646701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.647006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789358293144064:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10847) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.647039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789358293144084:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.647064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789358293144064:2107] [node 4] ICP30 unregister session Session# [6:7577789358293144084:2107] VirtualId# [6:7577789358293144081:0]
2025-11-28T14:52:53.647137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:53.647158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789358293144064:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.647171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.656220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789356432534370:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:53.657206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789358293144064:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:52:53.676588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:53.676753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789358293144102:2105] [node 5] ICS07 socket disconnect 255 reason# EndOfStream
2025-11-28T14:52:53.676784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789358293144102:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:53.676847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:53.676881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789358293144102:2105] [node 5] ICS15 start handshake
2025-11-28T14:52:53.677598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789358293144059:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10512) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:53.677669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789358293144102:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:53.677722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789358293144059:2105] [node 5] ICP30 unregister session Session# [6:7577789358293144102:2105] VirtualId# [6:7577789358293144100:0]
2025-11-28T14:52:53.677848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:53.677894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789358293144059:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10512) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:53.677922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10512) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:53.746874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789359066469724:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:53.746891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789359066469725:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:53.763917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789356745238257:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.763960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789356745238258:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:53.781548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789355861853722:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.781624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789355861853723:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:53.799706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355442248343:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.799746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355442248345:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:53.799846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355442248344:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:53.815741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789357900402211:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.815792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789357900402213:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:53.815834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789357900402212:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:53.831612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789356994662278:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:53.831643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789356994662280:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:53.831674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789356994662279:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:53.914391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789356745238320:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:53.913822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789357900402274:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:53.914406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789359066469787:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:53.914808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789356994662339:2104] [node 5] ICP01 ready to work
2025-11-28T14:52:53.914413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789359066469788:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:53.915082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789356745238321:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:53.915046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355442248406:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:53.915227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789356745238322:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:53.915168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789357900402275:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:53.915141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789355861853785:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:53.914866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789359066469789:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:53.915410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789356994662341:2105] [node 4] ICP01 ready to work
2025-11-28T14:52:53.915151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789355442248407:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:53.915305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789355861853786:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:53.915662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789355861853787:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.924082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789356745238336:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.924256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789356745238337:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39262
2025-11-28T14:52:53.924329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789357900402289:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2629
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.926130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789355861853797:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43593
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.927551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789355861853799:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43593
2025-11-28T14:52:53.928148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355442248416:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.930308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789356745238336:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.930272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789357900402290:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2629
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.930759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789355861853799:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.930725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355442248416:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.930837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789356745238338:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39262
2025-11-28T14:52:53.931428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789355442248344:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.931454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789356745238337:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.931925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789356745238339:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.932435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789357900402289:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.932026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789356745238329:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.932047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789356745238329:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.933666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789359066469798:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15404
2025-11-28T14:52:53.933682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789355861853803:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43593
2025-11-28T14:52:53.934088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789356745238329:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:53.933684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789359066469799:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15404
2025-11-28T14:52:53.934163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789355861853797:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.934214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789359066469800:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15404
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.934905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469799:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.935360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789355861853723:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.935029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469798:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.935074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789359066469803:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15404
2025-11-28T14:52:53.935788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789359066469804:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15404
2025-11-28T14:52:53.935940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789356745238338:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.935983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789356745238320:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.935946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469800:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.936026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469803:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.936049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.936097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789356745238320:2107] [node 4] ICP22 created new session: [2:7577789356745238344:2107]
2025-11-28T14:52:53.936160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789356745238344:2107] [node 4] ICS09 handshake done sender: [2:7577789356745238329:2112] self: [2:7577789356745238327:0] peer: [4:7577789355442248422:0] socket: 197 qp: -1
2025-11-28T14:52:53.936189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789356745238344:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:53.936112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355442248423:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24139
2025-11-28T14:52:53.936150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789355861853804:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43593
2025-11-28T14:52:53.936136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469804:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.936178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355442248424:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24139
2025-11-28T14:52:53.936176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789357900402290:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.936301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789356745238326:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.936313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:53.936338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789357900402213:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.936337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789356745238326:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.936706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355442248416:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.936702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789355861853803:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.937027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789355861853793:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.937632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789355442248344:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.937044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789355861853793:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.937927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789356745238343:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39262
2025-11-28T14:52:53.937750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.937906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355442248344:2049] [node 2] ICP22 created new session: [4:7577789355442248425:2049]
2025-11-28T14:52:53.938062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789356745238339:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.937973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355442248425:2049] [node 2] ICS09 handshake done sender: [4:7577789355442248416:2114] self: [4:7577789355442248422:0] peer: [2:7577789356745238327:0] socket: 203 qp: -1
2025-11-28T14:52:53.938000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355442248425:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:53.939337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789355861853793:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:53.938027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:53.939728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789355861853785:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.939770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.939895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789355861853785:2108] [node 5] ICP22 created new session: [3:7577789355861853809:2108]
2025-11-28T14:52:53.940625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789356745238326:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:53.940795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789355861853809:2108] [node 5] ICS09 handshake done sender: [3:7577789355861853793:2112] self: [3:7577789355861853792:0] peer: [5:7577789357900402292:0] socket: 215 qp: -1
2025-11-28T14:52:53.940838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789355861853809:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:53.940980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.941285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789357900402293:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2629
2025-11-28T14:52:53.941363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789357900402294:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2629
2025-11-28T14:52:53.941877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789356745238258:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.942152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:53.942242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789356745238320:2107] [node 4] ICP18 handshake (actor [2:7577789356745238339:2118]) is too old
2025-11-28T14:52:53.942252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789357900402290:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:53.942255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789356745238258:2049] [node 3] ICP22 created new session: [2:7577789356745238349:2049]
2025-11-28T14:52:53.942394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789356745238349:2049] [node 3] ICS09 handshake done sender: [2:7577789356745238326:2111] self: [2:7577789356745238325:0] peer: [3:7577789355861853805:0] socket: 196 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.942422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789356745238349:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:53.942423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789355861853804:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.942557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:53.943117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789355861853811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43593
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.943396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789357900402213:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18858) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789357900402290:2116] held: no
2025-11-28T14:52:53.943416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355442248424:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.943440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789357900402295:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2629
2025-11-28T14:52:53.943726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789355442248343:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.944083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789357900402213:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.944122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789356745238343:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.944119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:53.944107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469795:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.944161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789357900402213:2050] [node 3] ICP22 created new session: [5:7577789357900402296:2050]
2025-11-28T14:52:53.944201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789357900402296:2050] [node 3] ICS09 handshake done sender: [5:7577789357900402290:2116] self: [5:7577789357900402292:0] peer: [3:7577789355861853792:0] socket: 216 qp: -1
2025-11-28T14:52:53.944139Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789359066469795:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.944228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789357900402296:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:53.944255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:53.944696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789357900402295:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.945637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789359066469795:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:53.945806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789357900402274:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.945790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789355861853723:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789355861853799:2116] held: no
2025-11-28T14:52:53.946064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789355861853722:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.946161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789355861853799:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.946193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789359066469787:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.946245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:53.946389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355442248429:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.946275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789359066469787:2107] [node 4] ICP22 created new session: [1:7577789359066469814:2107]
2025-11-28T14:52:53.946374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789359066469814:2107] [node 4] ICS09 handshake done sender: [1:7577789359066469795:2112] self: [1:7577789359066469794:0] peer: [4:7577789355442248428:0] socket: 231 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.946577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355442248430:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24139
2025-11-28T14:52:53.946397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789359066469814:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:53.946610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789355861853723:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.946567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:53.946674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355442248431:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24139
2025-11-28T14:52:53.946658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.946695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789355861853723:2049] [node 2] ICP22 created new session: [3:7577789355861853814:2049]
2025-11-28T14:52:53.946763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789355861853814:2049] [node 2] ICS09 handshake done sender: [3:7577789355861853799:2116] self: [3:7577789355861853805:0] peer: [2:7577789356745238325:0] socket: 207 qp: -1
2025-11-28T14:52:53.946911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789355861853814:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:53.946989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:53.947452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789355442248343:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789355442248424:2117] held: no
2025-11-28T14:52:53.947579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469793:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.947573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789356745238257:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.947610Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789359066469793:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.947861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789357900402294:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.947998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789357900402212:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.948690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789356745238330:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.948712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789356745238330:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.949008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355442248415:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.949018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469792:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.949030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789355442248415:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.949049Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789359066469792:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.949347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355442248424:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:53.950505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662356:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.950882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789356745238330:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:53.951219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789355442248343:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:53.951235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355442248430:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.951265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.951298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789356745238321:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.951291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355442248343:2048] [node 1] ICP22 created new session: [4:7577789355442248433:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.951349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355442248433:2048] [node 1] ICS09 handshake done sender: [4:7577789355442248424:2117] self: [4:7577789355442248428:0] peer: [1:7577789359066469794:0] socket: 234 qp: -1
2025-11-28T14:52:53.951358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.951311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662357:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.951331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789356994662356:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.951384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355442248433:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.951354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789359066469793:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:53.951413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.951379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789357900402301:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2629
2025-11-28T14:52:53.951448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789359066469792:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.951442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789356745238321:2108] [node 5] ICP22 created new session: [2:7577789356745238354:2108]
2025-11-28T14:52:53.951456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789356994662279:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.951424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789357900402302:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2629
2025-11-28T14:52:53.951494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789356745238354:2108] [node 5] ICS09 handshake done sender: [2:7577789356745238330:2113] self: [2:7577789356745238328:0] peer: [5:7577789357900402300:0] socket: 238 qp: -1
2025-11-28T14:52:53.951591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789356745238354:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:53.951619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:53.951988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789357900402294:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.952139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355442248415:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:53.952008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789359066469725:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.952075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.952114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789359066469725:2049] [node 3] ICP22 created new session: [1:7577789359066469819:2049]
2025-11-28T14:52:53.952293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789359066469819:2049] [node 3] ICS09 handshake done sender: [1:7577789359066469793:2111] self: [1:7577789359066469791:0] peer: [3:7577789355861853813:0] socket: 230 qp: -1
2025-11-28T14:52:53.952448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789355861853817:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43593
2025-11-28T14:52:53.952477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789356745238335:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.952492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662359:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.952457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789359066469819:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:53.952514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789356745238335:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.952492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:53.952651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789357900402212:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.955249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789355861853804:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:53.952956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.953002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789357900402212:2049] [node 2] ICP22 created new session: [5:7577789357900402304:2049]
2025-11-28T14:52:53.955588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789355861853722:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.954717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789357900402304:2049] [node 2] ICS09 handshake done sender: [5:7577789357900402294:2118] self: [5:7577789357900402300:0] peer: [2:7577789356745238328:0] socket: 239 qp: -1
2025-11-28T14:52:53.955647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789355442248407:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.955637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.955736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789355861853722:2048] [node 1] ICP22 created new session: [3:7577789355861853818:2048]
2025-11-28T14:52:53.954758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789357900402304:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:53.955786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789356745238356:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39262
2025-11-28T14:52:53.955740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789356994662357:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.955804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.955789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789355861853818:2048] [node 1] ICS09 handshake done sender: [3:7577789355861853804:2118] self: [3:7577789355861853813:0] peer: [1:7577789359066469791:0] socket: 233 qp: -1
2025-11-28T14:52:53.955863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355442248407:2108] [node 5] ICP22 created new session: [4:7577789355442248437:2108]
2025-11-28T14:52:53.955033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:53.955826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789355861853818:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.955855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789359066469724:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.955563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789357900402303:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2629
2025-11-28T14:52:53.955910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.955860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.956025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355442248437:2108] [node 5] ICS09 handshake done sender: [4:7577789355442248415:2113] self: [4:7577789355442248413:0] peer: [5:7577789357900402299:0] socket: 241 qp: -1
2025-11-28T14:52:53.956077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355442248437:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:53.956046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789356994662280:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.955968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789359066469724:2048] [node 2] ICP22 created new session: [1:7577789359066469821:2048]
2025-11-28T14:52:53.956569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:53.956008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789359066469821:2048] [node 2] ICS09 handshake done sender: [1:7577789359066469792:2110] self: [1:7577789359066469790:0] peer: [2:7577789356745238352:0] socket: 228 qp: -1
2025-11-28T14:52:53.956744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355442248429:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.956051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789359066469821:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.956504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:53.957083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789356745238335:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:53.957075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662360:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.957083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789357900402295:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:53.957564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789356745238343:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:53.957601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789356994662356:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:53.957605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789355861853798:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.957641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789355861853798:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.957943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789355442248345:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.958050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789357900402212:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:53.958415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789356994662279:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.958444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789356745238322:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.958429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789357900402274:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.958521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:53.958636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789356994662279:2049] [node 2] ICP22 created new session: [6:7577789356994662362:2049]
2025-11-28T14:52:53.958582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.958574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:53.958672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789356994662362:2049] [node 2] ICS09 handshake done sender: [6:7577789356994662356:2115] self: [6:7577789356994662358:0] peer: [2:7577789356745238334:0] socket: 244 qp: -1
2025-11-28T14:52:53.958830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789355861853798:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:53.958584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789356745238257:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.958618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789357900402274:2108] [node 4] ICP22 created new session: [5:7577789357900402307:2108]
2025-11-28T14:52:53.958818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789356994662362:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:53.958829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789357900402307:2108] [node 4] ICS09 handshake done sender: [5:7577789357900402295:2119] self: [5:7577789357900402299:0] peer: [4:7577789355442248413:0] socket: 242 qp: -1
2025-11-28T14:52:53.958862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:53.958634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:53.958865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789357900402307:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:53.958685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789356745238322:2109] [node 6] ICP22 created new session: [2:7577789356745238358:2109]
2025-11-28T14:52:53.958892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:53.958828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789356745238257:2048] [node 1] ICP22 created new session: [2:7577789356745238359:2048]
2025-11-28T14:52:53.958860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789356745238358:2109] [node 6] ICS09 handshake done sender: [2:7577789356745238335:2114] self: [2:7577789356745238334:0] peer: [6:7577789356994662358:0] socket: 226 qp: -1
2025-11-28T14:52:53.959955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789355861853787:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.960309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.958867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789356745238359:2048] [node 1] ICS09 handshake done sender: [2:7577789356745238343:2119] self: [2:7577789356745238352:0] peer: [1:7577789359066469790:0] socket: 229 qp: -1
2025-11-28T14:52:53.958887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789356745238358:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:53.958898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789356745238359:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.960482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789355861853787:2109] [node 6] ICP22 created new session: [3:7577789355861853821:2109]
2025-11-28T14:52:53.958937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:53.958965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.960579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789355861853821:2109] [node 6] ICS09 handshake done sender: [3:7577789355861853798:2115] self: [3:7577789355861853796:0] peer: [6:7577789356994662361:0] socket: 218 qp: -1
2025-11-28T14:52:53.960597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789355861853821:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:53.960622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.961057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662364:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.961496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789356994662359:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.961755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789356994662357:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:53.961797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789355861853795:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.961826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789355861853795:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.963184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789355861853795:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.964036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789356994662280:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.964019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789355861853823:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43593
2025-11-28T14:52:53.964064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789357900402274:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:53.964117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355442248440:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24139
2025-11-28T14:52:53.964133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789355861853786:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.964403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:53.964573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:53.964593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789355861853786:2107] [node 4] ICP22 created new session: [3:7577789355861853825:2107]
2025-11-28T14:52:53.964637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789355861853825:2107] [node 4] ICS09 handshake done sender: [3:7577789355861853795:2113] self: [3:7577789355861853794:0] peer: [4:7577789355442248439:0] socket: 195 qp: -1
2025-11-28T14:52:53.964635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789356994662280:2050] [node 3] ICP22 created new session: [6:7577789356994662367:2050]
2025-11-28T14:52:53.964680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789355861853825:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:53.964696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789356994662341:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.964727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:53.964698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789356994662367:2050] [node 3] ICS09 handshake done sender: [6:7577789356994662357:2116] self: [6:7577789356994662361:0] peer: [3:7577789355861853796:0] socket: 245 qp: -1
2025-11-28T14:52:53.964945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355442248429:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:53.964828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789357900402302:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.965484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789355442248345:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.964733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789356994662367:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:53.965537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:53.964908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:53.965814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789357900402211:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.965746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355442248345:2050] [node 3] ICP22 created new session: [4:7577789355442248441:2050]
2025-11-28T14:52:53.966572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469797:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.966329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355442248441:2050] [node 3] ICS09 handshake done sender: [4:7577789355442248429:2118] self: [4:7577789355442248439:0] peer: [3:7577789355861853794:0] socket: 217 qp: -1
2025-11-28T14:52:53.966606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789359066469797:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.966518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355442248441:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:53.966561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:53.968864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789359066469797:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:53.969159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789359066469788:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.969269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.969428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789359066469788:2108] [node 5] ICP22 created new session: [1:7577789359066469825:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.969500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789359066469825:2108] [node 5] ICS09 handshake done sender: [1:7577789359066469797:2113] self: [1:7577789359066469796:0] peer: [5:7577789357900402309:0] socket: 247 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.969554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789359066469825:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:53.969592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:53.969563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.969597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789357900402310:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2629
2025-11-28T14:52:53.970319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789355442248418:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.970439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789357900402211:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789357900402302:2121] held: no
2025-11-28T14:52:53.970346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789355442248418:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.971354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789355442248345:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:53.971472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789356994662366:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.971550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789355442248418:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.971984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662370:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.971994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789355442248406:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.972057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.972099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789355442248406:2109] [node 6] ICP22 created new session: [4:7577789355442248444:2109]
2025-11-28T14:52:53.972132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789355442248444:2109] [node 6] ICS09 handshake done sender: [4:7577789355442248418:2115] self: [4:7577789355442248417:0] peer: [6:7577789356994662368:0] socket: 255 qp: -1
2025-11-28T14:52:53.972095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789357900402302:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:53.972154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789355442248444:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:53.972173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:53.972291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789357900402211:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.972334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:53.972438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789357900402211:2048] [node 1] ICP22 created new session: [5:7577789357900402311:2048]
2025-11-28T14:52:53.972487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789357900402311:2048] [node 1] ICS09 handshake done sender: [5:7577789357900402302:2121] self: [5:7577789357900402309:0] peer: [1:7577789359066469796:0] socket: 250 qp: -1
2025-11-28T14:52:53.972525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789357900402311:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.972583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.972695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789356994662278:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.972808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789356994662370:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.972890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662372:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.972925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789359066469802:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.972953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789359066469802:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.973625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789356994662339:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:53.973523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789355442248446:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24139
2025-11-28T14:52:53.973770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789356994662359:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:53.973989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789359066469802:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:53.973979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789357900402288:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:53.974013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789357900402288:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:53.975035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789356994662341:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.975071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789359066469789:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.975181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:53.975166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.975215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789359066469789:2109] [node 6] ICP22 created new session: [1:7577789359066469828:2109]
2025-11-28T14:52:53.975291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789356994662341:2105] [node 4] ICP22 created new session: [6:7577789356994662376:2105]
2025-11-28T14:52:53.975301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789359066469828:2109] [node 6] ICS09 handshake done sender: [1:7577789359066469802:2117] self: [1:7577789359066469801:0] peer: [6:7577789356994662373:0] socket: 209 qp: -1
2025-11-28T14:52:53.975332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789356994662376:2105] [node 4] ICS09 handshake done sender: [6:7577789356994662359:2117] self: [6:7577789356994662368:0] peer: [4:7577789355442248417:0] socket: 257 qp: -1
2025-11-28T14:52:53.975334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789359066469828:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:53.975367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:53.975362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789356994662376:2105] [node 4] ICS10 traffic start
2025-11-28T14:52:53.975392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:53.975334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789357900402288:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:53.975501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789357900402275:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:53.975550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:53.975852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789357900402275:2109] [node 6] ICP22 created new session: [5:7577789357900402314:2109]
2025-11-28T14:52:53.976147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789357900402314:2109] [node 6] ICS09 handshake done sender: [5:7577789357900402288:2114] self: [5:7577789357900402287:0] peer: [6:7577789356994662374:0] socket: 202 qp: -1
2025-11-28T14:52:53.976183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789357900402314:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:53.976237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:53.976507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789355442248406:2109] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:53.978721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789356994662278:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789356994662366:2120] held: no
2025-11-28T14:52:53.978734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662375:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.979180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789356994662339:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17674) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789356994662370:2121] held: no
No RDMA context found for GID: 2025-11-28T14:52:53.979611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789356994662366:2120] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:53.979685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789356994662378:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22879
2025-11-28T14:52:53.979904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789356994662278:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.979954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:53.980013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789356994662278:2048] [node 1] ICP22 created new session: [6:7577789356994662379:2048]
2025-11-28T14:52:53.980047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789356994662379:2048] [node 1] ICS09 handshake done sender: [6:7577789356994662366:2120] self: [6:7577789356994662373:0] peer: [1:7577789359066469801:0] socket: 225 qp: -1
2025-11-28T14:52:53.980073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789356994662379:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:53.980097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:53.980107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789356994662370:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:53.980316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789356994662339:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:53.980394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:53.980437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789356994662339:2104] [node 5] ICP22 created new session: [6:7577789356994662381:2104]
2025-11-28T14:52:53.980487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789356994662381:2104] [node 5] ICS09 handshake done sender: [6:7577789356994662370:2121] self: [6:7577789356994662374:0] peer: [5:7577789357900402287:0] socket: 208 qp: -1
2025-11-28T14:52:53.980520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789356994662381:2104] [node 5] ICS10 traffic start
2025-11-28T14:52:53.980561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:54.012921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:54.013103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789356745238359:2048] [node 1] ICS07 socket disconnect 229 reason# EndOfStream
2025-11-28T14:52:54.013129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789356745238359:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.013225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.013277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789356745238359:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:54.014017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:54.014243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789355861853818:2048] [node 1] ICS07 socket disconnect 233 reason# EndOfStream
2025-11-28T14:52:54.014363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789355861853818:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.014468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.014522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789355861853818:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:54.015118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789356745238257:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Socket error# Connection reset by peer state# SendExRequestData processed# 0 remain# 34 incoming: [0:0:0] held: no
2025-11-28T14:52:54.015161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789356745238359:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:54.015419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789355861853722:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.015478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789355861853818:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.015587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789355861853722:2048] [node 1] ICP30 unregister session Session# [3:7577789355861853818:2048] VirtualId# [3:7577789355861853813:0]
2025-11-28T14:52:54.015909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.015944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789355861853722:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.015956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.016067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789356745238257:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.016121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789356745238359:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.017173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789356745238257:2048] [node 1] ICP30 unregister session Session# [2:7577789356745238359:2048] VirtualId# [2:7577789356745238352:0]
2025-11-28T14:52:54.017305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.017351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789356745238257:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.017370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.017330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:54.017517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789355442248433:2048] [node 1] ICS01 socket: 234 reason# EndOfStream
2025-11-28T14:52:54.017560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789355442248343:2048] [node 1] ICP30 unregister session Session# [4:7577789355442248433:2048] VirtualId# [4:7577789355442248428:0]
2025-11-28T14:52:54.017602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789355442248433:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.017686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.017746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.018362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:54.018511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789357900402311:2048] [node 1] ICS01 socket: 250 reason# EndOfStream
2025-11-28T14:52:54.018544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789357900402211:2048] [node 1] ICP30 unregister session Session# [5:7577789357900402311:2048] VirtualId# [5:7577789357900402309:0]
2025-11-28T14:52:54.018563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789357900402311:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.018612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.018646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.019093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:54.019317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789356994662379:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-11-28T14:52:54.019359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789356994662278:2048] [node 1] ICP30 unregister session Session# [6:7577789356994662379:2048] VirtualId# [6:7577789356994662373:0]
2025-11-28T14:52:54.019384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789356994662379:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.019452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.019496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.025868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789355861853722:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:54.044576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:54.044811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789355861853814:2049] [node 2] ICS01 socket: 207 reason# EndOfStream
2025-11-28T14:52:54.044862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789355861853723:2049] [node 2] ICP30 unregister session Session# [3:7577789355861853814:2049] VirtualId# [3:7577789355861853805:0]
2025-11-28T14:52:54.044898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789355861853814:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.044998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:54.045056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:54.047494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:54.047702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789355442248425:2049] [node 2] ICS07 socket disconnect 203 reason# EndOfStream
2025-11-28T14:52:54.047738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789355442248425:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.047821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:54.047891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789355442248425:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:54.048438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:54.048692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789355442248344:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.048671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789357900402304:2049] [node 2] ICS07 socket disconnect 239 reason# EndOfStream
2025-11-28T14:52:54.048795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789355442248425:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.048708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789357900402304:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.048837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789355442248344:2049] [node 2] ICP30 unregister session Session# [4:7577789355442248425:2049] VirtualId# [4:7577789355442248422:0]
2025-11-28T14:52:54.048798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:54.048840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789357900402304:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:54.048965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:54.049014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789355442248344:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.049035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.049152Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:54.049410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789356994662362:2049] [node 2] ICS07 socket disconnect 244 reason# EndOfStream
2025-11-28T14:52:54.049497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789356994662362:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.049576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:54.049620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789356994662362:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:54.049615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789357900402212:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.049654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789357900402304:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.049739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789357900402212:2049] [node 2] ICP30 unregister session Session# [5:7577789357900402304:2049] VirtualId# [5:7577789357900402300:0]
2025-11-28T14:52:54.050430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:54.050456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789357900402212:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.050465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.050827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789356994662279:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.050871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789356994662362:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.050896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789356994662279:2049] [node 2] ICP30 unregister session Session# [6:7577789356994662362:2049] VirtualId# [6:7577789356994662358:0]
2025-11-28T14:52:54.050935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:54.050966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789356994662279:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.050982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24217) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.058638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789355442248344:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:54.060710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789356994662279:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:54.060726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789357900402212:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:54.108990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:54.109144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789355442248441:2050] [node 3] ICS07 socket disconnect 217 reason# EndOfStream
2025-11-28T14:52:54.109170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789355442248441:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.109241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:54.109274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789355442248441:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:54.109800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789355442248345:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18858) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.109892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789355442248441:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.109954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789355442248345:2050] [node 3] ICP30 unregister session Session# [4:7577789355442248441:2050] VirtualId# [4:7577789355442248439:0]
2025-11-28T14:52:54.109869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:54.110046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:54.110072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789355442248345:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18858) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.110086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18858) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.110181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789357900402296:2050] [node 3] ICS01 socket: 216 reason# EndOfStream
2025-11-28T14:52:54.110243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789357900402213:2050] [node 3] ICP30 unregister session Session# [5:7577789357900402296:2050] VirtualId# [5:7577789357900402292:0]
2025-11-28T14:52:54.110287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789357900402296:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.110423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:54.110486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:54.110568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:54.110775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789356994662367:2050] [node 3] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-28T14:52:54.110813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789356994662367:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.110886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:54.110931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789356994662367:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:54.111600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789356994662280:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18858) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.111698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789356994662367:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.111743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789356994662280:2050] [node 3] ICP30 unregister session Session# [6:7577789356994662367:2050] VirtualId# [6:7577789356994662361:0]
2025-11-28T14:52:54.111850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:54.111886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789356994662280:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18858) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.111904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18858) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.122179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789356994662280:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:54.153330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:54.153563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789357900402307:2108] [node 4] ICS07 socket disconnect 242 reason# EndOfStream
2025-11-28T14:52:54.153605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789357900402307:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.153710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:54.153757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789357900402307:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:54.154447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:54.154642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789356994662376:2105] [node 4] ICS07 socket disconnect 257 reason# EndOfStream
2025-11-28T14:52:54.154732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789356994662376:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.154616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789357900402274:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19294) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.154807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:54.154662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789357900402307:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.154856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789356994662376:2105] [node 4] ICS15 start handshake
2025-11-28T14:52:54.154743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789357900402274:2108] [node 4] ICP30 unregister session Session# [5:7577789357900402307:2108] VirtualId# [5:7577789357900402299:0]
2025-11-28T14:52:54.154855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:54.154916Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789357900402274:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19294) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.154933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19294) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.155560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789356994662341:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19294) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.155604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789356994662376:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.155667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789356994662341:2105] [node 4] ICP30 unregister session Session# [6:7577789356994662376:2105] VirtualId# [6:7577789356994662368:0]
2025-11-28T14:52:54.155753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:54.155795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789356994662341:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19294) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.155811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19294) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.165154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789356994662341:2105] [node 4] ICP33 wake up from error state
2025-11-28T14:52:54.165155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789357900402274:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:54.188183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:54.188380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789356994662381:2104] [node 5] ICS01 socket: 208 reason# EndOfStream
2025-11-28T14:52:54.188429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789356994662339:2104] [node 5] ICP30 unregister session Session# [6:7577789356994662381:2104] VirtualId# [6:7577789356994662374:0]
2025-11-28T14:52:54.188462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789356994662381:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.188535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:54.188593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:54.270396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789360257912631:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:54.270398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789360257912630:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:54.287965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361105951216:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.287979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361105951217:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:54.304820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789362679196092:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:54.304931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789362679196091:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.322125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789360708238400:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.322169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789360708238401:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:54.322187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789360708238402:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:54.338730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362170311058:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.338831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362170311060:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:54.338833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362170311059:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:54.357533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363246719064:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.357539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363246719065:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:54.357648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363246719066:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:54.432931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361105951279:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:54.442753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789362679196154:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:54.444036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363246719125:2105] [node 4] ICP01 ready to work
2025-11-28T14:52:54.443748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789360257912693:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:54.443817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789360708238464:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:54.444401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361105951280:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:54.444585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362170311121:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:54.444383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789362679196156:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:54.444355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789360708238465:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:54.443945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789360257912694:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:54.444416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363246719124:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:54.444637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789362679196155:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:54.444761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789360257912695:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:54.444535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361105951281:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:54.445360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362170311122:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.454724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789362679196163:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13634
2025-11-28T14:52:54.455069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789360708238472:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24356
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.456123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789360708238472:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.456411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361105951295:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57646
2025-11-28T14:52:54.456580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789360708238401:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.456561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361105951296:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57646
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.456979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361105951287:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.458866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789362679196163:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.459637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789362679196172:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13634
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.456998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789361105951287:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.460213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789362679196169:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13634
2025-11-28T14:52:54.459276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361105951287:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.460687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789362679196092:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.459815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361105951297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57646
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.461793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789362679196173:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13634
2025-11-28T14:52:54.462254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362170311136:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48680
2025-11-28T14:52:54.460619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361105951298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57646
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.464094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789360708238481:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24356
2025-11-28T14:52:54.463995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789362679196169:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.461701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789361105951279:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.464582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789360257912702:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17704
2025-11-28T14:52:54.463708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361105951285:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.464001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789362679196172:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.467295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789360708238472:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:54.463724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789361105951285:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.467654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789360708238401:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.468157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912702:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.464090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:54.467708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.469304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789362679196173:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.469301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362170311136:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.464124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361105951279:2107] [node 4] ICP22 created new session: [2:7577789361105951301:2107]
2025-11-28T14:52:54.469695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789362170311060:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.469657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789360257912703:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17704
2025-11-28T14:52:54.468163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789360708238401:2049] [node 2] ICP22 created new session: [4:7577789360708238482:2049]
No RDMA context found for GID: 2025-11-28T14:52:54.469662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789360708238482:2049] [node 2] ICS09 handshake done sender: [4:7577789360708238472:2114] self: [4:7577789360708238479:0] peer: [2:7577789361105951286:0] socket: 203 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:54.464514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789361105951301:2107] [node 4] ICS09 handshake done sender: [2:7577789361105951287:2112] self: [2:7577789361105951286:0] peer: [4:7577789360708238479:0] socket: 200 qp: -1
2025-11-28T14:52:54.470136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789360257912704:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17704
2025-11-28T14:52:54.470199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789362679196162:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.464544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789361105951301:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:54.470211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719142:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.470010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789360708238482:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:54.470232Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789362679196162:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.470220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789360257912705:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17704
2025-11-28T14:52:54.465679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:54.470231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:54.465814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361105951295:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.470391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789360257912708:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17704
2025-11-28T14:52:54.468006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361105951297:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.470612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912703:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.469288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361105951296:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:54.470851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912708:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:54.469449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361105951298:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.471348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789360708238483:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24356
No RDMA context found for GID: 2025-11-28T14:52:54.471045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912704:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:54.469868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361105951285:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:54.471347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912705:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.471287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789361105951217:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.471450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789362679196178:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13634
2025-11-28T14:52:54.471322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:54.471456Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789362679196162:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:54.471366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361105951217:2049] [node 3] ICP22 created new session: [2:7577789361105951308:2049]
2025-11-28T14:52:54.471463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789362679196179:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13634
2025-11-28T14:52:54.471401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789361105951308:2049] [node 3] ICS09 handshake done sender: [2:7577789361105951285:2111] self: [2:7577789361105951284:0] peer: [3:7577789362679196174:0] socket: 199 qp: -1
2025-11-28T14:52:54.471464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789361105951308:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.471662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362170311138:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48680
2025-11-28T14:52:54.471650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363246719142:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.471486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:54.472438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789362679196154:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.472502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363246719065:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.472679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361105951305:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57646
2025-11-28T14:52:54.472655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789362679196163:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:54.473620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362170311139:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48680
2025-11-28T14:52:54.473087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:54.473444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789362679196154:2108] [node 5] ICP22 created new session: [3:7577789362679196181:2108]
2025-11-28T14:52:54.474154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361105951292:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.473573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789362679196092:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12001) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789362679196163:2113] held: no
2025-11-28T14:52:54.474195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789361105951292:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.473708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789362679196181:2108] [node 5] ICS09 handshake done sender: [3:7577789362679196162:2112] self: [3:7577789362679196159:0] peer: [5:7577789362170311137:0] socket: 218 qp: -1
2025-11-28T14:52:54.473759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789362679196181:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:54.473872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:54.475551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362170311138:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.475581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789362170311136:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.476787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789360708238483:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.476822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789362679196092:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.476217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789362170311060:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16949) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789362170311136:2115] held: no
2025-11-28T14:52:54.476956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:54.477036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789362679196092:2049] [node 2] ICP22 created new session: [3:7577789362679196183:2049]
2025-11-28T14:52:54.476943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362170311140:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48680
2025-11-28T14:52:54.477056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789361105951279:2107] [node 4] ICP18 handshake (actor [2:7577789361105951296:2116]) is too old
2025-11-28T14:52:54.477094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789362679196183:2049] [node 2] ICS09 handshake done sender: [3:7577789362679196163:2113] self: [3:7577789362679196174:0] peer: [2:7577789361105951284:0] socket: 202 qp: -1
2025-11-28T14:52:54.477264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789362679196183:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:54.477141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789360708238400:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.477376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789362170311121:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.477303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:54.477419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912700:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.477444Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789360257912700:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.477501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361105951292:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:54.477473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719144:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.477608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789360708238474:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.477609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361105951305:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.477627Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789360708238474:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.478497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789362170311060:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.478642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363246719142:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:54.478620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789362679196179:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.478568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:54.478839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789360257912700:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:54.478886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789362679196091:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.479351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789361105951280:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.478819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362170311060:2050] [node 3] ICP22 created new session: [5:7577789362170311143:2050]
2025-11-28T14:52:54.479413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:54.478932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789362170311143:2050] [node 3] ICS09 handshake done sender: [5:7577789362170311136:2115] self: [5:7577789362170311137:0] peer: [3:7577789362679196159:0] socket: 227 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.479063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789362170311143:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:54.479967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361105951280:2109] [node 6] ICP22 created new session: [2:7577789361105951312:2109]
2025-11-28T14:52:54.479971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363246719065:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.479102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:54.480081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789361105951312:2109] [node 6] ICS09 handshake done sender: [2:7577789361105951292:2113] self: [2:7577789361105951291:0] peer: [6:7577789363246719143:0] socket: 235 qp: -1
2025-11-28T14:52:54.480081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362170311144:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48680
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.480045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:54.480125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789361105951312:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:54.480120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912699:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.480157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:54.480103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363246719065:2049] [node 2] ICP22 created new session: [6:7577789363246719146:2049]
2025-11-28T14:52:54.480143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789360257912699:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.480342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789360708238474:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:54.480151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363246719146:2049] [node 2] ICS09 handshake done sender: [6:7577789363246719142:2115] self: [6:7577789363246719143:0] peer: [2:7577789361105951291:0] socket: 236 qp: -1
2025-11-28T14:52:54.480290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363246719146:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:54.480142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789360257912693:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.481335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.480301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:54.481933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719145:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.481319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789360257912693:2107] [node 4] ICP22 created new session: [1:7577789360257912720:2107]
2025-11-28T14:52:54.482187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789362170311138:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:54.482057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789360257912720:2107] [node 4] ICS09 handshake done sender: [1:7577789360257912700:2111] self: [1:7577789360257912697:0] peer: [4:7577789360708238486:0] socket: 238 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.482205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362170311140:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.483493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789360708238464:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.484856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789361105951216:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.483446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789362170311121:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.482478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789360257912720:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:54.483515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789360708238487:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24356
2025-11-28T14:52:54.485724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361105951294:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.483489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:54.485744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789361105951294:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.482513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:54.483758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:54.483535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789362170311059:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.482813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789360257912699:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:54.484276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789360708238464:2108] [node 5] ICP22 created new session: [4:7577789360708238491:2108]
2025-11-28T14:52:54.485728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789360257912631:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.484615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362170311121:2108] [node 4] ICP22 created new session: [5:7577789362170311146:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.485801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:54.484787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789362170311146:2108] [node 4] ICS09 handshake done sender: [5:7577789362170311138:2116] self: [5:7577789362170311141:0] peer: [4:7577789360708238473:0] socket: 245 qp: -1
2025-11-28T14:52:54.484610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789360708238491:2108] [node 5] ICS09 handshake done sender: [4:7577789360708238474:2115] self: [4:7577789360708238473:0] peer: [5:7577789362170311141:0] socket: 244 qp: -1
2025-11-28T14:52:54.485721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789362170311146:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:54.486024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789360257912631:2049] [node 3] ICP22 created new session: [1:7577789360257912724:2049]
2025-11-28T14:52:54.484779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789360708238491:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:54.485758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:54.487241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361105951294:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:54.486796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789360257912724:2049] [node 3] ICS09 handshake done sender: [1:7577789360257912699:2110] self: [1:7577789360257912696:0] peer: [3:7577789362679196187:0] socket: 241 qp: -1
2025-11-28T14:52:54.484814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:54.487185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719148:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.484859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789360708238489:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24356
2025-11-28T14:52:54.486804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912701:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.487336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363246719145:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.485727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789360708238490:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24356
2025-11-28T14:52:54.487444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363246719066:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.485732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789360708238400:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789360708238483:2118] held: no
2025-11-28T14:52:54.486820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789360257912724:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:54.487656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789361105951281:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.486821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789360257912701:2112] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:54.487709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:54.487162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:54.487789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789362679196168:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.487746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361105951281:2108] [node 5] ICP22 created new session: [2:7577789361105951316:2108]
2025-11-28T14:52:54.487781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789361105951316:2108] [node 5] ICS09 handshake done sender: [2:7577789361105951294:2114] self: [2:7577789361105951293:0] peer: [5:7577789362170311147:0] socket: 217 qp: -1
2025-11-28T14:52:54.487828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789362679196168:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.487812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789361105951316:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.488837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789362679196188:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13634
2025-11-28T14:52:54.487841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:54.489396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789362679196168:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.489541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789362679196179:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:54.489522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362170311149:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48680
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.489706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719151:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.490361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719152:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.490430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789362679196091:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789362679196179:2120] held: no
2025-11-28T14:52:54.490933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363246719148:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.490963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789360708238483:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:54.491128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363246719064:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.491159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789362679196155:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.492025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363246719145:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:54.492002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:54.492243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789360257912701:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:54.492109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789360708238489:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.492140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789362679196155:2109] [node 6] ICP22 created new session: [3:7577789362679196190:2109]
2025-11-28T14:52:54.492237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363246719066:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.492124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789360708238464:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:54.492615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789362170311059:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12001) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789362170311140:2118] held: no
2025-11-28T14:52:54.492129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789360708238400:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.492380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789362679196190:2109] [node 6] ICS09 handshake done sender: [3:7577789362679196168:2114] self: [3:7577789362679196167:0] peer: [6:7577789363246719150:0] socket: 208 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:54.492218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:54.492492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789362679196190:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:54.492351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789360708238400:2048] [node 1] ICP22 created new session: [4:7577789360708238495:2048]
2025-11-28T14:52:54.492682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:54.492635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912707:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.492519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:54.492704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789360708238495:2048] [node 1] ICS09 handshake done sender: [4:7577789360708238483:2118] self: [4:7577789360708238486:0] peer: [1:7577789360257912697:0] socket: 239 qp: -1
2025-11-28T14:52:54.492738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789360708238495:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:54.493093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363246719066:2050] [node 3] ICP22 created new session: [6:7577789363246719154:2050]
2025-11-28T14:52:54.493084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:54.493197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363246719154:2050] [node 3] ICS09 handshake done sender: [6:7577789363246719145:2117] self: [6:7577789363246719150:0] peer: [3:7577789362679196167:0] socket: 211 qp: -1
2025-11-28T14:52:54.493101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789360708238493:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24356
2025-11-28T14:52:54.493389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363246719154:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:54.492669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789360257912707:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.493408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789362679196091:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.493171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789360257912630:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.493446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.493451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:54.493363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:54.493476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789362679196091:2048] [node 1] ICP22 created new session: [3:7577789362679196192:2048]
2025-11-28T14:52:54.493493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789360708238402:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.493548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789362679196192:2048] [node 1] ICS09 handshake done sender: [3:7577789362679196179:2120] self: [3:7577789362679196187:0] peer: [1:7577789360257912696:0] socket: 210 qp: -1
2025-11-28T14:52:54.493391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789360257912630:2048] [node 2] ICP22 created new session: [1:7577789360257912729:2048]
2025-11-28T14:52:54.493698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361105951318:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57646
2025-11-28T14:52:54.493425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789360257912729:2048] [node 2] ICS09 handshake done sender: [1:7577789360257912701:2112] self: [1:7577789360257912698:0] peer: [2:7577789361105951314:0] socket: 242 qp: -1
2025-11-28T14:52:54.493728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789362679196192:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:54.495718Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789362170311140:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:54.495309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789360708238493:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.493459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789360257912729:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:54.493477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:54.493745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:54.496088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361105951305:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:54.497678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789361105951216:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789361105951305:2119] held: no
2025-11-28T14:52:54.497820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789362679196171:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.497856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789362679196171:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.498107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789360257912707:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:54.498384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789362170311059:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.498810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:54.498408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789360257912694:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.498915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:54.499059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362170311059:2049] [node 2] ICP22 created new session: [5:7577789362170311152:2049]
2025-11-28T14:52:54.499111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789362170311152:2049] [node 2] ICS09 handshake done sender: [5:7577789362170311140:2118] self: [5:7577789362170311147:0] peer: [2:7577789361105951293:0] socket: 249 qp: -1
2025-11-28T14:52:54.499071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789360257912694:2109] [node 6] ICP22 created new session: [1:7577789360257912731:2109]
2025-11-28T14:52:54.499141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789362170311152:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:54.499111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719156:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.499273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:54.499207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719157:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.499220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789360257912731:2109] [node 6] ICS09 handshake done sender: [1:7577789360257912707:2117] self: [1:7577789360257912706:0] peer: [6:7577789363246719153:0] socket: 223 qp: -1
2025-11-28T14:52:54.499370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789361105951216:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.499452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:54.499504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789360257912731:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:54.499640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361105951216:2048] [node 1] ICP22 created new session: [2:7577789361105951319:2048]
2025-11-28T14:52:54.499535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:54.499666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789361105951319:2048] [node 1] ICS09 handshake done sender: [2:7577789361105951305:2119] self: [2:7577789361105951314:0] peer: [1:7577789360257912698:0] socket: 243 qp: -1
2025-11-28T14:52:54.499690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789361105951319:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:54.500291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363246719156:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.499718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:54.499972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362170311151:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48680
2025-11-28T14:52:54.500530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789363246719064:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789363246719148:2118] held: no
2025-11-28T14:52:54.500058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789362679196171:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.502248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789360708238499:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24356
2025-11-28T14:52:54.502596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789362679196156:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.502641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789360708238489:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:54.502648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362170311151:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.502702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:54.502760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789362679196156:2107] [node 4] ICP22 created new session: [3:7577789362679196195:2107]
2025-11-28T14:52:54.502996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789362679196195:2107] [node 4] ICS09 handshake done sender: [3:7577789362679196171:2116] self: [3:7577789362679196170:0] peer: [4:7577789360708238497:0] socket: 252 qp: -1
2025-11-28T14:52:54.503111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789362679196195:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:54.503147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:54.503162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363246719148:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:54.503195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789360708238402:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.503240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:54.503280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789360708238402:2050] [node 3] ICP22 created new session: [4:7577789360708238500:2050]
2025-11-28T14:52:54.503219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789362170311058:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.503460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789360708238500:2050] [node 3] ICS09 handshake done sender: [4:7577789360708238489:2120] self: [4:7577789360708238497:0] peer: [3:7577789362679196170:0] socket: 255 qp: -1
2025-11-28T14:52:54.503385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363246719151:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.503638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362170311154:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48680
2025-11-28T14:52:54.503692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789360708238500:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:54.503533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363246719125:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.503781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:54.503875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789360257912710:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.503878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363246719064:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.503908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789360257912710:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.504840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:54.504899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363246719064:2048] [node 1] ICP22 created new session: [6:7577789363246719160:2048]
2025-11-28T14:52:54.504917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363246719124:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.504961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363246719160:2048] [node 1] ICS09 handshake done sender: [6:7577789363246719148:2118] self: [6:7577789363246719153:0] peer: [1:7577789360257912706:0] socket: 224 qp: -1
2025-11-28T14:52:54.505015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363246719160:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:54.505057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:54.505117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789360257912710:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:54.505117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362170311135:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.505149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789362170311135:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.506474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789360708238480:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.506574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789362170311135:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:54.506511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789360708238480:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.506657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789360257912695:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.506820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:54.507165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362170311154:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.507182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789360257912695:2108] [node 5] ICP22 created new session: [1:7577789360257912734:2108]
2025-11-28T14:52:54.507536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362170311156:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48680
2025-11-28T14:52:54.507558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789362170311122:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.507615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789360257912734:2108] [node 5] ICS09 handshake done sender: [1:7577789360257912710:2119] self: [1:7577789360257912709:0] peer: [5:7577789362170311155:0] socket: 216 qp: -1
2025-11-28T14:52:54.507820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789360257912734:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:54.507894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:54.507871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:54.507934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362170311122:2109] [node 6] ICP22 created new session: [5:7577789362170311158:2109]
2025-11-28T14:52:54.507983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789362170311158:2109] [node 6] ICS09 handshake done sender: [5:7577789362170311135:2114] self: [5:7577789362170311133:0] peer: [6:7577789363246719161:0] socket: 260 qp: -1
2025-11-28T14:52:54.508018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789360708238480:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:54.508029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789362170311158:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.508053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:54.508175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789360708238465:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:54.508141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719163:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.509010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363246719164:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58477
2025-11-28T14:52:54.509026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789362170311058:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789362170311151:2121] held: no
2025-11-28T14:52:54.509110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:54.509169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789360708238465:2110] [node 6] ICP22 created new session: [4:7577789360708238503:2110]
2025-11-28T14:52:54.509379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789360708238503:2110] [node 6] ICS09 handshake done sender: [4:7577789360708238480:2116] self: [4:7577789360708238478:0] peer: [6:7577789363246719159:0] socket: 206 qp: -1
2025-11-28T14:52:54.509413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789360708238503:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:54.509434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:54.509497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363246719151:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:54.509669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363246719156:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:54.510030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363246719125:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.510047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363246719124:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.510099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:54.510109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:54.510135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363246719124:2106] [node 5] ICP22 created new session: [6:7577789363246719165:2106]
2025-11-28T14:52:54.510155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363246719125:2105] [node 4] ICP22 created new session: [6:7577789363246719166:2105]
2025-11-28T14:52:54.510168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363246719165:2106] [node 5] ICS09 handshake done sender: [6:7577789363246719151:2119] self: [6:7577789363246719161:0] peer: [5:7577789362170311133:0] socket: 261 qp: -1
2025-11-28T14:52:54.510291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789362170311151:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:54.510194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363246719165:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:54.510244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:54.510701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789362170311058:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.510248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363246719166:2105] [node 4] ICS09 handshake done sender: [6:7577789363246719156:2121] self: [6:7577789363246719159:0] peer: [4:7577789360708238478:0] socket: 265 qp: -1
2025-11-28T14:52:54.510743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:54.510273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363246719166:2105] [node 4] ICS10 traffic start
2025-11-28T14:52:54.510781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362170311058:2048] [node 1] ICP22 created new session: [5:7577789362170311160:2048]
2025-11-28T14:52:54.510300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:54.511030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789362170311160:2048] [node 1] ICS09 handshake done sender: [5:7577789362170311151:2121] self: [5:7577789362170311155:0] peer: [1:7577789360257912709:0] socket: 266 qp: -1
2025-11-28T14:52:54.511054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789362170311160:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:54.511134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:54.511135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789362170311122:2109] [node 6] ICP18 handshake (actor [5:7577789362170311154:2122]) is too old
2025-11-28T14:52:54.547570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:54.547815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789361105951319:2048] [node 1] ICS01 socket: 243 reason# EndOfStream
2025-11-28T14:52:54.547863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789361105951216:2048] [node 1] ICP30 unregister session Session# [2:7577789361105951319:2048] VirtualId# [2:7577789361105951314:0]
2025-11-28T14:52:54.547918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789361105951319:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.548026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.548081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.548582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:54.548743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789362679196192:2048] [node 1] ICS01 socket: 210 reason# EndOfStream
2025-11-28T14:52:54.548778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789362679196091:2048] [node 1] ICP30 unregister session Session# [3:7577789362679196192:2048] VirtualId# [3:7577789362679196187:0]
2025-11-28T14:52:54.548817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789362679196192:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.548917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.548963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.550331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:54.550532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789360708238495:2048] [node 1] ICS01 socket: 239 reason# EndOfStream
2025-11-28T14:52:54.550592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789360708238400:2048] [node 1] ICP30 unregister session Session# [4:7577789360708238495:2048] VirtualId# [4:7577789360708238486:0]
2025-11-28T14:52:54.550621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789360708238495:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.550704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.550760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.550964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:54.551099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789362170311160:2048] [node 1] ICS01 socket: 266 reason# EndOfStream
2025-11-28T14:52:54.551127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789362170311058:2048] [node 1] ICP30 unregister session Session# [5:7577789362170311160:2048] VirtualId# [5:7577789362170311155:0]
2025-11-28T14:52:54.551146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789362170311160:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.551212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.551258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.551575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:54.551707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363246719160:2048] [node 1] ICS01 socket: 224 reason# EndOfStream
2025-11-28T14:52:54.551740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363246719064:2048] [node 1] ICP30 unregister session Session# [6:7577789363246719160:2048] VirtualId# [6:7577789363246719153:0]
2025-11-28T14:52:54.551760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363246719160:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.551827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:54.551869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:54.585878Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:54.586050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789362679196183:2049] [node 2] ICS01 socket: 202 reason# EndOfStream
2025-11-28T14:52:54.586089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789362679196092:2049] [node 2] ICP30 unregister session Session# [3:7577789362679196183:2049] VirtualId# [3:7577789362679196174:0]
2025-11-28T14:52:54.586118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789362679196183:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.586214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:54.586278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:54.589193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:54.589391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789360708238482:2049] [node 2] ICS07 socket disconnect 203 reason# EndOfStream
2025-11-28T14:52:54.589425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789360708238482:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.589521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:54.589569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789360708238482:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:54.589879Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:54.590004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789362170311152:2049] [node 2] ICS01 socket: 249 reason# EndOfStream
2025-11-28T14:52:54.590035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789362170311059:2049] [node 2] ICP30 unregister session Session# [5:7577789362170311152:2049] VirtualId# [5:7577789362170311147:0]
2025-11-28T14:52:54.590059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789362170311152:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.590113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:54.590155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:54.590245Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789360708238401:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12001) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.590282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789360708238482:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.590308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789360708238401:2049] [node 2] ICP30 unregister session Session# [4:7577789360708238482:2049] VirtualId# [4:7577789360708238479:0]
2025-11-28T14:52:54.590404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:54.590427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789360708238401:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12001) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.590441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12001) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.590624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:54.590743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789363246719146:2049] [node 2] ICS07 socket disconnect 236 reason# EndOfStream
2025-11-28T14:52:54.590768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363246719146:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.590810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:54.590845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789363246719146:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:54.591300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789363246719065:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12001) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.591384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363246719146:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.591412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363246719065:2049] [node 2] ICP30 unregister session Session# [6:7577789363246719146:2049] VirtualId# [6:7577789363246719143:0]
2025-11-28T14:52:54.591440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:54.591456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789363246719065:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12001) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.591473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12001) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.600374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789360708238401:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:54.601375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789363246719065:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:54.629310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:54.629539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789360708238500:2050] [node 3] ICS07 socket disconnect 255 reason# EndOfStream
2025-11-28T14:52:54.629570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789360708238500:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.629663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:54.629698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789360708238500:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:54.630199Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:54.630341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789362170311143:2050] [node 3] ICS01 socket: 227 reason# EndOfStream
2025-11-28T14:52:54.630384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789362170311060:2050] [node 3] ICP30 unregister session Session# [5:7577789362170311143:2050] VirtualId# [5:7577789362170311137:0]
2025-11-28T14:52:54.630345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789360708238402:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16949) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.630408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789362170311143:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.630466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:54.630497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:54.630403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789360708238500:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.630422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789360708238402:2050] [node 3] ICP30 unregister session Session# [4:7577789360708238500:2050] VirtualId# [4:7577789360708238497:0]
2025-11-28T14:52:54.630464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:54.630511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789360708238402:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.630522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.630805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:54.630921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789363246719154:2050] [node 3] ICS07 socket disconnect 211 reason# EndOfStream
2025-11-28T14:52:54.630937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363246719154:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.630977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:54.631003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789363246719154:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:54.631503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789363246719066:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16949) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.631569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363246719154:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.631599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363246719066:2050] [node 3] ICP30 unregister session Session# [6:7577789363246719154:2050] VirtualId# [6:7577789363246719150:0]
2025-11-28T14:52:54.631671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:54.631696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789363246719066:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.631704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.640280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789360708238402:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:54.641286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789363246719066:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:54.689495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:54.689767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789362170311146:2108] [node 4] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-28T14:52:54.689836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789362170311146:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.689959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:54.690017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789362170311146:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:54.690273Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:54.690436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789363246719166:2105] [node 4] ICS07 socket disconnect 265 reason# EndOfStream
2025-11-28T14:52:54.690460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363246719166:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.690545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:54.690576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789363246719166:2105] [node 4] ICS15 start handshake
2025-11-28T14:52:54.690854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789362170311121:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9311) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.690966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789362170311146:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.691016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789362170311121:2108] [node 4] ICP30 unregister session Session# [5:7577789362170311146:2108] VirtualId# [5:7577789362170311141:0]
2025-11-28T14:52:54.691106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:54.691138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789362170311121:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9311) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.691147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9311) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.691177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789363246719125:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9311) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.691243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363246719166:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.691278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363246719125:2105] [node 4] ICP30 unregister session Session# [6:7577789363246719166:2105] VirtualId# [6:7577789363246719159:0]
2025-11-28T14:52:54.691361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:54.691398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789363246719125:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9311) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.691411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9311) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.701724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789363246719125:2105] [node 4] ICP33 wake up from error state
2025-11-28T14:52:54.712635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:54.712862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789363246719165:2106] [node 5] ICS07 socket disconnect 261 reason# EndOfStream
2025-11-28T14:52:54.712906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363246719165:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:54.713022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:54.713085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789363246719165:2106] [node 5] ICS15 start handshake
2025-11-28T14:52:54.713675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789363246719124:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:54.713732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363246719165:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:54.713778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363246719124:2106] [node 5] ICP30 unregister session Session# [6:7577789363246719165:2106] VirtualId# [6:7577789363246719161:0]
2025-11-28T14:52:54.713860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:54.713886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789363246719124:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.713896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:54.723228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789363246719124:2106] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:54.795756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789361558910758:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:54.795814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789361558910759:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:54.812650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361421040695:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.812902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361421040696:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:54.830873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789361813574857:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:54.830877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789361813574856:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.848679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789361052546928:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.848756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789361052546930:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:54.848802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789361052546929:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:54.865706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362330582595:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.865739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362330582597:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:54.865955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362330582596:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:54.882700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789361621478239:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:54.882739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789361621478241:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:54.882990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789361621478240:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:54.924101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789361621478301:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:54.967153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361421040758:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:54.967151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789361558910822:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:54.967227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789361813574919:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:54.967343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361421040759:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:54.968964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789361558910823:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:54.969051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789361621478303:2109] [node 4] ICP01 ready to work
2025-11-28T14:52:54.967359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362330582658:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:54.969049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789361052547000:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:54.969848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789361421040760:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:54.969917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789361558910824:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:54.970003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789361052547001:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:54.970051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789362330582659:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:54.970091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789361813574920:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:54.970109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789361813574921:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:54.973194Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789361558910758:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48756854ns) {Configure:104(48748547ns) {ProcessConfigured:135(100564ns) {StartInitialHandshake:157(98014ns) {DropHandshakes:470(1857ns) {DropIncomingHandshake:442(485ns) DropOutgoingHandshake:457(410ns)} GenerateSessionVirtualId:988(2737ns) PrepareNewSessionHandshake:477(978ns) {SwitchToState:179(423ns)}}}}}
2025-11-28T14:52:54.973197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789361558910759:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48901403ns) {Configure:104(48898553ns) {ProcessConfigured:135(96501ns) {StartInitialHandshake:157(93931ns) {DropHandshakes:470(3790ns) {DropIncomingHandshake:442(433ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(1081ns) PrepareNewSessionHandshake:477(1321ns) {SwitchToState:179(329ns)}}}}}
2025-11-28T14:52:54.977431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789361052546929:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(35546923ns) {Configure:104(35534807ns) {ProcessConfigured:135(304373ns) {StartInitialHandshake:157(302327ns) {DropHandshakes:470(1317ns) {DropIncomingHandshake:442(390ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:477(842ns) {SwitchToState:179(388ns)}}}}}
2025-11-28T14:52:54.977929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789361621478301:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(52870510ns) {Configure:104(52865929ns) {ProcessConfigured:135(325820ns) {StartInitialHandshake:157(324401ns) {DropHandshakes:470(861ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(367ns)} GenerateSessionVirtualId:988(143ns) PrepareNewSessionHandshake:477(509ns) {SwitchToState:179(93ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.982163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361421040767:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30801
2025-11-28T14:52:54.982169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362330582668:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20525
2025-11-28T14:52:54.982422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789361813574930:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37171
2025-11-28T14:52:54.983447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361421040767:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.983465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789361813574930:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.985272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362330582668:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.986010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362330582672:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20525
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.987280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789361813574935:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37171
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.988829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789361558910838:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58713
2025-11-28T14:52:54.987005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789362330582595:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.988794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361421040773:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30801
2025-11-28T14:52:54.989240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789361558910830:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.987931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789361052547015:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.988976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789361813574856:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.989221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361421040775:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30801
2025-11-28T14:52:54.987858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362330582673:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20525
2025-11-28T14:52:54.989284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789361558910830:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.992448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789361052547016:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.989525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789361421040695:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.993424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789361813574934:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37171
2025-11-28T14:52:54.993476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362330582672:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.991795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789361558910840:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58713
2025-11-28T14:52:54.996859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789362330582659:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.994204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361421040774:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30801
2025-11-28T14:52:54.996227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789361813574935:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.991848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789361558910839:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58713
2025-11-28T14:52:54.997683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362330582673:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.996811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789361621478315:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.996252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361421040774:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.997066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789361621478316:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
2025-11-28T14:52:54.997672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789361052547015:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.997716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789361813574934:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.991891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789361558910830:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:54.999137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789361621478315:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.993486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789361558910828:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.997389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361421040773:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.998075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789361052547005:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.999342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789361621478316:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:54.999131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361421040775:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:54.993525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789361558910828:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.999368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789361621478317:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
2025-11-28T14:52:54.998098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789361052547005:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:52:54.999655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789361621478241:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-28T14:52:54.994229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789361558910827:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.999116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789361052547016:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.999512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789361052546930:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.994264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789361558910827:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.999706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789361052546929:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.999715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362330582677:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20525
2025-11-28T14:52:54.996848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789361558910840:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.999730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789361421040780:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30801
2025-11-28T14:52:54.999683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789361813574926:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.999762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789361621478240:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:54.999988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789361052547005:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:55.000301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361421040765:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.997367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789361558910828:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:54.999717Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789361813574926:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.999936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789361621478317:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.000315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789361421040765:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.997370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789361558910827:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.000275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789361813574939:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37171
2025-11-28T14:52:55.001837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789361621478301:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.000560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361421040764:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.000548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789361813574928:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.997659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789361558910822:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.000585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789361421040764:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.000595Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789361813574928:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:54.997851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:55.002667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361421040767:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.002993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789362330582668:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:55.002680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789361813574930:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:54.999337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789361558910822:2109] [node 5] ICP22 created new session: [1:7577789361558910845:2109]
2025-11-28T14:52:55.002756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361421040764:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:55.003391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789362330582667:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.005383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789365916445619:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
2025-11-28T14:52:55.004347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789361052547000:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.002698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789361813574928:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:54.999479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789361558910838:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.003003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361421040765:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.003410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789362330582680:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20525
2025-11-28T14:52:55.005412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:54.999486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789361558910758:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.003417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789362330582667:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.003378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789361813574926:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:55.003772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789361421040695:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.005442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789361052547000:2108] [node 5] ICP22 created new session: [4:7577789365347514319:2108]
2025-11-28T14:52:55.004000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789362330582595:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.999512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789361558910759:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.003986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.006234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.006203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365347514319:2108] [node 5] ICS09 handshake done sender: [4:7577789361052547005:2119] self: [4:7577789361052547002:0] peer: [5:7577789362330582678:0] socket: 212 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.999560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.004357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789361813574856:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.004356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361421040695:2048] [node 1] ICP22 created new session: [2:7577789365716008081:2048]
2025-11-28T14:52:55.006496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365347514319:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:55.007875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789365916445620:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
2025-11-28T14:52:54.999575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.006309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362330582595:2048] [node 1] ICP22 created new session: [5:7577789366625549978:2048]
2025-11-28T14:52:55.005399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789361813574919:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.006521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789366625549978:2048] [node 1] ICS09 handshake done sender: [5:7577789362330582668:2114] self: [5:7577789362330582674:0] peer: [1:7577789361558910829:0] socket: 209 qp: -1
2025-11-28T14:52:55.005399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789361421040758:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:54.999598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789361558910839:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.006522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:55.005406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.009156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365347514316:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-11-28T14:52:54.999591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789361558910845:2109] [node 5] ICS09 handshake done sender: [1:7577789361558910830:2113] self: [1:7577789361558910829:0] peer: [5:7577789362330582674:0] socket: 206 qp: -1
2025-11-28T14:52:55.005439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789361421040759:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.006550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789362330582672:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:55.005415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789361813574920:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.010950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789365916445622:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
2025-11-28T14:52:55.005574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365716008081:2048] [node 1] ICS09 handshake done sender: [2:7577789361421040767:2112] self: [2:7577789361421040777:0] peer: [1:7577789361558910825:0] socket: 210 qp: -1
2025-11-28T14:52:55.006854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789366625549978:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:55.009536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365347514317:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-11-28T14:52:54.999608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789361558910759:2049] [node 3] ICP22 created new session: [1:7577789361558910846:2049]
2025-11-28T14:52:55.011577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789361621478241:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13201) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789361621478315:2114] held: no
2025-11-28T14:52:55.005439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:55.007191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789362330582667:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:55.005872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:55.013125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789365916445621:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
2025-11-28T14:52:54.999695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789361558910758:2048] [node 2] ICP22 created new session: [1:7577789361558910847:2048]
2025-11-28T14:52:55.010425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365347514318:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-11-28T14:52:55.007656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.013872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789365916445619:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.006219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365716008081:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:55.005447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789361813574856:2048] [node 1] ICP22 created new session: [3:7577789366108542240:2048]
2025-11-28T14:52:55.012256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789365347514316:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:54.999713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789361558910845:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:55.014772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789361621478240:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789361621478316:2115] held: no
2025-11-28T14:52:55.008740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789362330582659:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:54.999729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789361558910846:2049] [node 3] ICS09 handshake done sender: [1:7577789361558910828:2112] self: [1:7577789361558910826:0] peer: [3:7577789361813574938:0] socket: 204 qp: -1
2025-11-28T14:52:55.005449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:55.008790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:54.999740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789361558910847:2048] [node 2] ICS09 handshake done sender: [1:7577789361558910827:2111] self: [1:7577789361558910825:0] peer: [2:7577789361421040777:0] socket: 205 qp: -1
2025-11-28T14:52:55.005471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789361813574919:2109] [node 6] ICP22 created new session: [3:7577789366108542241:2109]
2025-11-28T14:52:55.006229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:55.013177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789361052547016:2125] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.009134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789362330582658:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.005616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789361813574920:2107] [node 4] ICP22 created new session: [3:7577789366108542242:2107]
2025-11-28T14:52:55.006487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361421040759:2109] [node 6] ICP22 created new session: [2:7577789365716008082:2109]
2025-11-28T14:52:55.009193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362330582659:2108] [node 4] ICP22 created new session: [5:7577789366625549980:2108]
2025-11-28T14:52:54.999763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789361558910846:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:55.014643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789361052547015:2124] [node 3] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:55.006250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.014874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789361052546929:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789361052547016:2125] held: no
2025-11-28T14:52:55.009575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789366625549980:2108] [node 4] ICS09 handshake done sender: [5:7577789362330582672:2115] self: [5:7577789362330582678:0] peer: [4:7577789361052547002:0] socket: 224 qp: -1
2025-11-28T14:52:55.006478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789366108542240:2048] [node 1] ICS09 handshake done sender: [3:7577789361813574930:2114] self: [3:7577789361813574938:0] peer: [1:7577789361558910826:0] socket: 208 qp: -1
2025-11-28T14:52:54.999792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.006523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361421040758:2107] [node 4] ICP22 created new session: [2:7577789365716008083:2107]
2025-11-28T14:52:55.007193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365716008083:2107] [node 4] ICS09 handshake done sender: [2:7577789361421040765:2111] self: [2:7577789361421040761:0] peer: [4:7577789361052547019:0] socket: 230 qp: -1
2025-11-28T14:52:55.009768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789366625549980:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:55.007645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365716008082:2109] [node 6] ICS09 handshake done sender: [2:7577789361421040764:2110] self: [2:7577789361421040762:0] peer: [6:7577789361621478321:0] socket: 221 qp: -1
2025-11-28T14:52:55.009784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:55.007677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365716008082:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:55.009801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.007926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365716008083:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:54.999942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:55.017285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789361621478317:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:55.006487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789366108542241:2109] [node 6] ICS09 handshake done sender: [3:7577789361813574928:2112] self: [3:7577789361813574924:0] peer: [6:7577789361621478320:0] socket: 213 qp: -1
2025-11-28T14:52:55.008505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:55.010052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362330582658:2109] [node 6] ICP22 created new session: [5:7577789366625549981:2109]
2025-11-28T14:52:55.016861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789361052546928:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.000422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789361558910847:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:55.017828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789361621478301:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11600) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789361621478317:2116] held: no
2025-11-28T14:52:55.006818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789366108542240:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:55.008519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.016914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789361052546930:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.012207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789366625549981:2109] [node 6] ICS09 handshake done sender: [5:7577789362330582667:2113] self: [5:7577789362330582662:0] peer: [6:7577789361621478322:0] socket: 225 qp: -1
2025-11-28T14:52:55.001826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.006844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.017253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:55.012252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789366625549981:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:55.018058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789361558910832:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.020444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789361621478315:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:55.007168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789366108542242:2107] [node 4] ICS09 handshake done sender: [3:7577789361813574926:2110] self: [3:7577789361813574923:0] peer: [4:7577789361052547017:0] socket: 214 qp: -1
2025-11-28T14:52:55.017306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789361052546930:2050] [node 3] ICP22 created new session: [4:7577789365347514323:2050]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:55.012281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:55.018091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789361558910832:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.017531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365347514323:2050] [node 3] ICS09 handshake done sender: [4:7577789361052547015:2124] self: [4:7577789361052547017:0] peer: [3:7577789361813574923:0] socket: 229 qp: -1
2025-11-28T14:52:55.007197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789366108542242:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:55.017781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365347514323:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:55.020823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789361558910832:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:55.018008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.007619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789366108542241:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:55.018276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789361052546929:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.007655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:55.021192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789361621478239:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.021167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789366625549985:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20525
2025-11-28T14:52:55.020431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.007676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.020767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789361052546929:2049] [node 2] ICP22 created new session: [4:7577789365347514326:2049]
2025-11-28T14:52:55.017136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789366108542247:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37171
2025-11-28T14:52:55.020825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365347514326:2049] [node 2] ICS09 handshake done sender: [4:7577789361052547016:2125] self: [4:7577789361052547019:0] peer: [2:7577789361421040761:0] socket: 235 qp: -1
2025-11-28T14:52:55.020973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365347514326:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:55.021230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.021304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789361621478316:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.021673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789361558910758:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(17734343ns) {HandleHandshakeStatus:329(17732198ns) {DropIncomingHandshake:442(122ns) DropOutgoingHandshake:457(294ns) SwitchToState:179(139ns) ProcessPendingSessionEvents:495(14602411ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14511172ns max# 1213189ns}}}
2025-11-28T14:52:55.021381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789361621478241:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.021409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:55.021442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789361621478241:2050] [node 3] ICP22 created new session: [6:7577789365916445626:2050]
2025-11-28T14:52:55.021953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789361558910823:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.021697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789365916445626:2050] [node 3] ICS09 handshake done sender: [6:7577789361621478315:2114] self: [6:7577789361621478320:0] peer: [3:7577789361813574924:0] socket: 241 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.021950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789361621478301:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.022187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789361558910822:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(19222693ns) {HandleHandshakeStatus:329(19221026ns) {DropIncomingHandshake:442(214ns) DropOutgoingHandshake:457(371ns) SwitchToState:179(75ns) ProcessPendingSessionEvents:495(14548542ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13254578ns max# 1040648ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.021967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789365916445626:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:55.022799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:55.022479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.023307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789365716008089:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30801
2025-11-28T14:52:55.022813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:55.022854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789361621478301:2107] [node 5] ICP22 created new session: [6:7577789365916445629:2107]
2025-11-28T14:52:55.023232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789365916445629:2107] [node 5] ICS09 handshake done sender: [6:7577789361621478317:2116] self: [6:7577789361621478322:0] peer: [5:7577789362330582662:0] socket: 244 qp: -1
2025-11-28T14:52:55.022835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789361558910758:2048] [node 2] ICP18 handshake (actor [1:7577789361558910838:2116]) is too old
2025-11-28T14:52:55.023868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789361621478240:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.023377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789366108542248:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37171
2025-11-28T14:52:55.024032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789365916445629:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:55.023169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789361558910837:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.024014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789366625549985:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.023918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789361813574856:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16277503ns) {HandleHandshakeStatus:329(16276007ns) {DropIncomingHandshake:442(567ns) DropOutgoingHandshake:457(16853ns) SwitchToState:179(321ns) ProcessPendingSessionEvents:495(11568350ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11221629ns max# 1006883ns}}}
2025-11-28T14:52:55.023180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789361558910823:2107] [node 4] ICP22 created new session: [1:7577789365853878151:2107]
2025-11-28T14:52:55.024071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:55.023184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789361558910837:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.024094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789361813574920:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:55.023215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789365853878151:2107] [node 4] ICS09 handshake done sender: [1:7577789361558910832:2114] self: [1:7577789361558910831:0] peer: [4:7577789365347514322:0] socket: 254 qp: -1
2025-11-28T14:52:55.023299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789365853878151:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:55.024024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789361558910822:2109] [node 5] ICP18 handshake (actor [1:7577789361558910839:2117]) is too old
2025-11-28T14:52:55.025788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789366108542248:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.024181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.026329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.026374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789361813574857:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.026393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789361621478240:2049] [node 2] ICP22 created new session: [6:7577789365916445631:2049]
2025-11-28T14:52:55.026510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789366625549986:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20525
2025-11-28T14:52:55.026753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789365916445631:2049] [node 2] ICS09 handshake done sender: [6:7577789361621478316:2115] self: [6:7577789361621478321:0] peer: [2:7577789361421040762:0] socket: 243 qp: -1
2025-11-28T14:52:55.026793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789365916445631:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.027019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789365716008089:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.027092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.026853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789362330582595:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16548102ns) {HandleHandshakeStatus:329(16545235ns) {DropIncomingHandshake:442(763ns) DropOutgoingHandshake:457(6889ns) SwitchToState:179(489ns) ProcessPendingSessionEvents:495(11880561ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11800385ns max# 1054433ns}}}
2025-11-28T14:52:55.027048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361421040769:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.027132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789365916445625:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
2025-11-28T14:52:55.027145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789362330582597:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.027074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789361421040769:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.028197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789366625549986:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.028387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789361813574937:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.029124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789362330582596:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.028412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789361813574937:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.030579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789361421040778:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.030616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789361052546928:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789365347514316:2126] held: no
2025-11-28T14:52:55.030659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365347514328:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-11-28T14:52:55.030606Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361421040769:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:55.030662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789365853878150:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58713
2025-11-28T14:52:55.030615Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789361421040778:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.031408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789361421040696:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.031983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.033043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789365916445625:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.032174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361421040696:2049] [node 3] ICP22 created new session: [2:7577789365716008092:2049]
2025-11-28T14:52:55.032166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789361558910837:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:55.032259Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789361421040778:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:55.033116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365716008092:2049] [node 3] ICS09 handshake done sender: [2:7577789361421040769:2114] self: [2:7577789361421040766:0] peer: [3:7577789366108542249:0] socket: 218 qp: -1
2025-11-28T14:52:55.033244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365716008092:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:55.033431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.033277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789361813574857:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789366108542248:2120] held: no
2025-11-28T14:52:55.033426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789366108542250:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37171
2025-11-28T14:52:55.033517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789361421040760:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.034131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789361813574937:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:55.034177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:55.034193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789361621478303:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.034188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789361558910824:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:55.034230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789365853878150:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.034259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789361421040760:2108] [node 5] ICP22 created new session: [2:7577789365716008095:2108]
2025-11-28T14:52:55.034401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365716008095:2108] [node 5] ICS09 handshake done sender: [2:7577789361421040778:2118] self: [2:7577789361421040776:0] peer: [5:7577789366625549988:0] socket: 233 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.034432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789366625549989:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20525
2025-11-28T14:52:55.034430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789365916445633:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
2025-11-28T14:52:55.034423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365716008095:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:55.034399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789365347514316:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:55.034432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:55.034750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789366625549990:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20525
2025-11-28T14:52:55.034503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:55.035322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789366108542248:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.034712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365347514329:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-11-28T14:52:55.034463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789361558910824:2110] [node 6] ICP22 created new session: [1:7577789365853878154:2110]
2025-11-28T14:52:55.035240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789365916445619:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:55.034727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789361052546928:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.035939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789361813574921:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.036001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789366625549986:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.034499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789365853878154:2110] [node 6] ICS09 handshake done sender: [1:7577789361558910837:2115] self: [1:7577789361558910836:0] peer: [6:7577789365916445624:0] socket: 255 qp: -1
2025-11-28T14:52:55.036295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789361621478239:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.035999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:55.034734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789361052547014:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.034587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789365853878154:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:55.034760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789361052547014:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.037908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.036840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789362330582596:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789366625549986:2121] held: no
2025-11-28T14:52:55.034756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:55.036191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789361813574857:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.035497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.038204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789361621478239:2048] [node 1] ICP22 created new session: [6:7577789365916445636:2048]
2025-11-28T14:52:55.036255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789361813574921:2108] [node 5] ICP22 created new session: [3:7577789366108542252:2108]
2025-11-28T14:52:55.035959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789365347514329:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.038217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789366625549985:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:55.036778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.038878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789365916445636:2048] [node 1] ICS09 handshake done sender: [6:7577789365916445619:2118] self: [6:7577789365916445624:0] peer: [1:7577789361558910836:0] socket: 256 qp: -1
2025-11-28T14:52:55.035970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789361052546928:2048] [node 1] ICP22 created new session: [4:7577789365347514330:2048]
2025-11-28T14:52:55.036793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789366108542252:2108] [node 5] ICS09 handshake done sender: [3:7577789361813574937:2117] self: [3:7577789361813574936:0] peer: [5:7577789366625549987:0] socket: 223 qp: -1
2025-11-28T14:52:55.038906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789362330582597:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13201) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789366625549985:2120] held: no
2025-11-28T14:52:55.039051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789365916445636:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.036075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365347514330:2048] [node 1] ICS09 handshake done sender: [4:7577789365347514316:2126] self: [4:7577789365347514322:0] peer: [1:7577789361558910831:0] socket: 262 qp: -1
2025-11-28T14:52:55.037948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789366108542252:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:55.039374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789361621478303:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63598) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789365916445625:2122] held: no
2025-11-28T14:52:55.038009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:55.038179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789361813574857:2049] [node 2] ICP22 created new session: [3:7577789366108542253:2049]
2025-11-28T14:52:55.039391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.038897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789366108542253:2049] [node 2] ICS09 handshake done sender: [3:7577789366108542248:2120] self: [3:7577789366108542249:0] peer: [2:7577789361421040766:0] socket: 222 qp: -1
2025-11-28T14:52:55.036202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789361052547014:2123] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:55.041117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789365916445635:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40802
2025-11-28T14:52:55.039055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789366108542253:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:55.036254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365347514330:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:55.039673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.036471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.036666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789361052547001:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.041473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789362330582596:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.037915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:55.038064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789361052547001:2109] [node 6] ICP22 created new session: [4:7577789365347514334:2109]
2025-11-28T14:52:55.038881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365347514334:2109] [node 6] ICS09 handshake done sender: [4:7577789361052547014:2123] self: [4:7577789361052547012:0] peer: [6:7577789365916445634:0] socket: 234 qp: -1
2025-11-28T14:52:55.039045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365347514334:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:55.039086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:55.041767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.041889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362330582596:2049] [node 2] ICP22 created new session: [5:7577789366625549991:2049]
2025-11-28T14:52:55.041924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789362330582597:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.041939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789366625549991:2049] [node 2] ICS09 handshake done sender: [5:7577789366625549986:2121] self: [5:7577789366625549988:0] peer: [2:7577789361421040776:0] socket: 268 qp: -1
2025-11-28T14:52:55.041946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789361558910824:2110] [node 6] ICP18 handshake (actor [1:7577789365853878150:2119]) is too old
2025-11-28T14:52:55.041979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789366625549991:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:55.041988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:55.042026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789362330582597:2050] [node 3] ICP22 created new session: [5:7577789366625549992:2050]
2025-11-28T14:52:55.042032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.042073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789366625549992:2050] [node 3] ICS09 handshake done sender: [5:7577789366625549985:2120] self: [5:7577789366625549987:0] peer: [3:7577789361813574936:0] socket: 215 qp: -1
2025-11-28T14:52:55.042090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789366625549992:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:55.042117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.042125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789365916445625:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:55.042248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789361621478303:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.042294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:55.042321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789361621478303:2109] [node 4] ICP22 created new session: [6:7577789365916445638:2109]
2025-11-28T14:52:55.042724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789365916445638:2109] [node 4] ICS09 handshake done sender: [6:7577789365916445625:2122] self: [6:7577789365916445634:0] peer: [4:7577789361052547012:0] socket: 270 qp: -1
2025-11-28T14:52:55.042841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789365916445638:2109] [node 4] ICS10 traffic start
2025-11-28T14:52:55.042933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.094531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:55.094777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789365716008081:2048] [node 1] ICS01 socket: 210 reason# EndOfStream
2025-11-28T14:52:55.094833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789361421040695:2048] [node 1] ICP30 unregister session Session# [2:7577789365716008081:2048] VirtualId# [2:7577789361421040777:0]
2025-11-28T14:52:55.094870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789365716008081:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.094963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.095023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.108341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:55.108628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789366108542240:2048] [node 1] ICS07 socket disconnect 208 reason# EndOfStream
2025-11-28T14:52:55.108668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789366108542240:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.108780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.108841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789366108542240:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:55.109697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789361813574856:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.110251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789366108542240:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.110300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789361813574856:2048] [node 1] ICP30 unregister session Session# [3:7577789366108542240:2048] VirtualId# [3:7577789361813574938:0]
2025-11-28T14:52:55.110402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.110449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789361813574856:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.110470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.110555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:55.110755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789365347514330:2048] [node 1] ICS01 socket: 262 reason# EndOfStream
2025-11-28T14:52:55.110817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789361052546928:2048] [node 1] ICP30 unregister session Session# [4:7577789365347514330:2048] VirtualId# [4:7577789365347514322:0]
2025-11-28T14:52:55.110849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789365347514330:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.110936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.110973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.111322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:55.111460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789366625549978:2048] [node 1] ICS07 socket disconnect 209 reason# EndOfStream
2025-11-28T14:52:55.111482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789366625549978:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.111549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.111601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789366625549978:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:55.112017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:55.112210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789365916445636:2048] [node 1] ICS07 socket disconnect 256 reason# EndOfStream
2025-11-28T14:52:55.112204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789362330582595:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.112245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789366625549978:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.112270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789362330582595:2048] [node 1] ICP30 unregister session Session# [5:7577789366625549978:2048] VirtualId# [5:7577789362330582674:0]
2025-11-28T14:52:55.112243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789365916445636:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.112336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.112405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789365916445636:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:55.112563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.112625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789362330582595:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.112641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.113062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789361621478239:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.113099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789365916445636:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.113122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789361621478239:2048] [node 1] ICP30 unregister session Session# [6:7577789365916445636:2048] VirtualId# [6:7577789365916445624:0]
2025-11-28T14:52:55.113222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.113244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789361621478239:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.113255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23013) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.120583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789361813574856:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:55.122560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789362330582595:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:55.122595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789361621478239:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:55.163778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:55.164014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789366108542253:2049] [node 2] ICS01 socket: 222 reason# EndOfStream
2025-11-28T14:52:55.164068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789361813574857:2049] [node 2] ICP30 unregister session Session# [3:7577789366108542253:2049] VirtualId# [3:7577789366108542249:0]
2025-11-28T14:52:55.164125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789366108542253:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.164226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:55.164275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:55.167049Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:55.167205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789365347514326:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-11-28T14:52:55.167241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789361052546929:2049] [node 2] ICP30 unregister session Session# [4:7577789365347514326:2049] VirtualId# [4:7577789361052547019:0]
2025-11-28T14:52:55.167264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789365347514326:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.167345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:55.167395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:55.167833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:55.167967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789366625549991:2049] [node 2] ICS01 socket: 268 reason# EndOfStream
2025-11-28T14:52:55.167996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789362330582596:2049] [node 2] ICP30 unregister session Session# [5:7577789366625549991:2049] VirtualId# [5:7577789366625549988:0]
2025-11-28T14:52:55.168016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789366625549991:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.168084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:55.168126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:55.168543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:55.168733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789365916445631:2049] [node 2] ICS01 socket: 243 reason# EndOfStream
2025-11-28T14:52:55.168784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789361621478240:2049] [node 2] ICP30 unregister session Session# [6:7577789365916445631:2049] VirtualId# [6:7577789361621478321:0]
2025-11-28T14:52:55.168825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789365916445631:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.168900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:55.168965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:55.229426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:55.229592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789365347514323:2050] [node 3] ICS07 socket disconnect 229 reason# EndOfStream
2025-11-28T14:52:55.229618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789365347514323:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.229695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:55.229735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789365347514323:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:55.230226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:55.230292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789361052546930:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13201) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.230390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789366625549992:2050] [node 3] ICS01 socket: 215 reason# EndOfStream
2025-11-28T14:52:55.230365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789365347514323:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.230449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789362330582597:2050] [node 3] ICP30 unregister session Session# [5:7577789366625549992:2050] VirtualId# [5:7577789366625549987:0]
2025-11-28T14:52:55.230397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789361052546930:2050] [node 3] ICP30 unregister session Session# [4:7577789365347514323:2050] VirtualId# [4:7577789361052547017:0]
2025-11-28T14:52:55.230475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:55.230487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789366625549992:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.230495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789361052546930:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13201) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.230574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:55.230532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13201) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.230644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:55.231041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:55.231223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789365916445626:2050] [node 3] ICS01 socket: 241 reason# EndOfStream
2025-11-28T14:52:55.231265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789361621478241:2050] [node 3] ICP30 unregister session Session# [6:7577789365916445626:2050] VirtualId# [6:7577789361621478320:0]
2025-11-28T14:52:55.231295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789365916445626:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.231357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:55.231410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:55.279423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:55.279592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789366625549980:2108] [node 4] ICS07 socket disconnect 224 reason# EndOfStream
2025-11-28T14:52:55.279615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789366625549980:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.279677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:55.279712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789366625549980:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:55.280336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789362330582659:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63598) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.280386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789366625549980:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.280463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789362330582659:2108] [node 4] ICP30 unregister session Session# [5:7577789366625549980:2108] VirtualId# [5:7577789362330582678:0]
2025-11-28T14:52:55.280344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:55.280522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:55.280548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789362330582659:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63598) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.280569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63598) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.280577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789365916445638:2109] [node 4] ICS01 socket: 270 reason# EndOfStream
2025-11-28T14:52:55.280638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789361621478303:2109] [node 4] ICP30 unregister session Session# [6:7577789365916445638:2109] VirtualId# [6:7577789365916445634:0]
2025-11-28T14:52:55.280680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789365916445638:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.280747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:55.280798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:55.290651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789362330582659:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:55.380639Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:55.380874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789365916445629:2107] [node 5] ICS01 socket: 244 reason# EndOfStream
2025-11-28T14:52:55.380926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789361621478301:2107] [node 5] ICP30 unregister session Session# [6:7577789365916445629:2107] VirtualId# [6:7577789361621478322:0]
2025-11-28T14:52:55.380967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789365916445629:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.381090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:55.381146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:55.501899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789367008272706:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:55.501940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789367008272707:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:55.519506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789365205417270:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:55.519675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789365205417271:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:55.537579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789365110654777:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:55.537683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789365110654778:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:55.554855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789365018790549:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:55.554857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789365018790547:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:55.554862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789365018790548:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:55.571976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789364556283491:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:55.572037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789364556283493:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:55.572038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789364556283492:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:55.589800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363723778406:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:55.589828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363723778408:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:55.589962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363723778407:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:55.677314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363723778467:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:55.677075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789367008272769:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:55.680047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789365205417333:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:55.681899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789365205417334:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:55.681666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789365018790611:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:55.681042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789365110654840:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:55.681906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789367008272770:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:55.682308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789364556283554:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:55.681851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789363723778468:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:55.681591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789365110654841:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:55.681909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789365205417335:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:55.682632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789367008272771:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:55.682784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789365018790612:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:55.682827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789364556283555:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:55.682506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789365110654842:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.692826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789367008272783:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19960
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.698862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789365110654856:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44910
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.699648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789365205417347:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2094
2025-11-28T14:52:55.699505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789367008272783:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.700059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789364556283566:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34158
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.699486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789365110654857:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44910
2025-11-28T14:52:55.700062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778477:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.699947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789365110654856:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:55.700514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789365110654777:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.700936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789365205417350:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2094
2025-11-28T14:52:55.700960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778482:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.701354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789364556283566:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.701358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789365205417347:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.701036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363723778477:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.702518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789364556283569:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34158
2025-11-28T14:52:55.702322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789367008272786:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19960
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.701391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363723778406:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.702827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789365110654858:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44910
2025-11-28T14:52:55.702475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789367008272779:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.702858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789365205417270:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.703194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789364556283493:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.703724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365018790626:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16203
2025-11-28T14:52:55.703537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789365110654859:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44910
2025-11-28T14:52:55.702491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789367008272779:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.703529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789365205417351:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2094
2025-11-28T14:52:55.704477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789364556283569:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.702502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778483:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
2025-11-28T14:52:55.704482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789365110654848:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.703417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789367008272773:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.702534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363723778482:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.703451Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789367008272773:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.702947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363723778408:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.704513Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789365110654848:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.703553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789367008272775:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.703429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778487:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
2025-11-28T14:52:55.708259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789365110654850:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.709325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789364556283492:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.703575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789367008272775:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.704472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363723778483:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.708286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789365110654850:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.703693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789367008272787:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19960
2025-11-28T14:52:55.708635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363723778407:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.706320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789367008272788:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19960
2025-11-28T14:52:55.709297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789365110654848:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:55.709983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789365205417350:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.708336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789367008272786:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.709336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363723778487:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.710116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789365205417337:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.710164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789364556283565:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.709518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789365110654858:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.709469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789367008272788:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.709609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363723778467:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.710588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789365110654850:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:55.710140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789365205417337:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.710205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789364556283565:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.710620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789365110654841:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.710152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789365205417343:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.709504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789367008272787:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.711969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789365110654857:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.713069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789364556283573:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34158
2025-11-28T14:52:55.710189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789365205417343:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.709982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789367008272779:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.711977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:55.713120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789365205417337:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:55.712643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789365018790626:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.712613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789365110654841:2108] [node 5] ICP22 created new session: [3:7577789365110654864:2108]
2025-11-28T14:52:55.710005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789367008272773:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:55.713500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789365205417351:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.713807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789365110654842:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.713165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789364556283565:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:55.710167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789367008272775:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.713931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789365205417353:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2094
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.713841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:55.714099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789364556283493:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28335) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789364556283566:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.713172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789367008272707:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.715073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778493:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
2025-11-28T14:52:55.713128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789365110654864:2108] [node 5] ICS09 handshake done sender: [3:7577789365110654848:2111] self: [3:7577789365110654846:0] peer: [5:7577789364556283571:0] socket: 212 qp: -1
2025-11-28T14:52:55.713989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:55.715383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789365205417347:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:55.716141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365018790627:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16203
2025-11-28T14:52:55.715180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778492:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
2025-11-28T14:52:55.713926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789365110654864:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:55.715413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789365205417343:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:55.714653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789367008272707:2049] [node 3] ICP22 created new session: [1:7577789367008272796:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.716823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778494:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
2025-11-28T14:52:55.713957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:55.715979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789365205417334:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.714930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789367008272706:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.719307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789364556283555:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.714136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789365110654842:2109] [node 6] ICP22 created new session: [3:7577789365110654867:2109]
2025-11-28T14:52:55.718963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.720801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778495:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
2025-11-28T14:52:55.719348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:55.715095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789367008272796:2049] [node 3] ICS09 handshake done sender: [1:7577789367008272779:2112] self: [1:7577789367008272776:0] peer: [3:7577789365110654860:0] socket: 218 qp: -1
2025-11-28T14:52:55.714936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789365110654859:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.720895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789363723778407:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11784) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789363723778483:2117] held: no
2025-11-28T14:52:55.719006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789365205417270:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.720096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789364556283555:2109] [node 6] ICP22 created new session: [5:7577789364556283576:2109]
2025-11-28T14:52:55.715173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789367008272771:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.715064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789365110654867:2109] [node 6] ICS09 handshake done sender: [3:7577789365110654850:2112] self: [3:7577789365110654847:0] peer: [6:7577789363723778489:0] socket: 215 qp: -1
2025-11-28T14:52:55.719015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789365205417335:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.721885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789364556283576:2109] [node 6] ICS09 handshake done sender: [5:7577789364556283565:2114] self: [5:7577789364556283558:0] peer: [6:7577789363723778491:0] socket: 242 qp: -1
2025-11-28T14:52:55.721976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363723778482:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:55.715221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789367008272796:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:55.724951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789365018790627:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.715795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789365110654867:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:55.715255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.719403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:55.722512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789364556283576:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.725603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789365018790549:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.722223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363723778477:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:55.715842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.719454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789365205417335:2108] [node 5] ICP22 created new session: [2:7577789365205417359:2108]
2025-11-28T14:52:55.722747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.723323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363723778408:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.715960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.716477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789365110654865:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44910
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.723499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:55.723488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789364556283566:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:55.719550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.730238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365018790629:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16203
2025-11-28T14:52:55.723934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363723778408:2050] [node 3] ICP22 created new session: [6:7577789363723778496:2050]
2025-11-28T14:52:55.716178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789367008272706:2048] [node 2] ICP22 created new session: [1:7577789367008272798:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.724452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789364556283575:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34158
2025-11-28T14:52:55.731506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365018790630:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16203
2025-11-28T14:52:55.716806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789365110654777:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789365110654856:2115] held: no
2025-11-28T14:52:55.724862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363723778406:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.716817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: 2025-11-28T14:52:55.725318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789364556283493:2050] [node 3] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:55.725312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789365110654856:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:55.716848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789367008272798:2048] [node 2] ICS09 handshake done sender: [1:7577789367008272775:2111] self: [1:7577789367008272774:0] peer: [2:7577789365205417352:0] socket: 216 qp: -1
2025-11-28T14:52:55.719586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789365205417270:2048] [node 1] ICP22 created new session: [2:7577789365205417360:2048]
2025-11-28T14:52:55.725243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363723778496:2050] [node 3] ICS09 handshake done sender: [6:7577789363723778482:2116] self: [6:7577789363723778489:0] peer: [3:7577789365110654847:0] socket: 223 qp: -1
2025-11-28T14:52:55.725606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:55.727857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789365110654777:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.725532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363723778496:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:55.718977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789367008272771:2109] [node 6] ICP22 created new session: [1:7577789367008272800:2109]
2025-11-28T14:52:55.728041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.726120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.719356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789367008272798:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:55.728423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789365110654777:2048] [node 1] ICP22 created new session: [3:7577789365110654873:2048]
2025-11-28T14:52:55.727217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.719738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789365205417334:2109] [node 6] ICP22 created new session: [2:7577789365205417358:2109]
2025-11-28T14:52:55.719379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789367008272800:2109] [node 6] ICS09 handshake done sender: [1:7577789367008272773:2110] self: [1:7577789367008272772:0] peer: [6:7577789363723778488:0] socket: 213 qp: -1
2025-11-28T14:52:55.725901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789364556283493:2050] [node 3] ICP22 created new session: [5:7577789364556283581:2050]
2025-11-28T14:52:55.728614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789365110654852:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.727240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363723778406:2048] [node 1] ICP22 created new session: [6:7577789363723778497:2048]
2025-11-28T14:52:55.720126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365205417358:2109] [node 6] ICS09 handshake done sender: [2:7577789365205417337:2110] self: [2:7577789365205417336:0] peer: [6:7577789363723778490:0] socket: 224 qp: -1
2025-11-28T14:52:55.732153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365018790631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16203
2025-11-28T14:52:55.728646Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789365110654852:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.719742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789367008272800:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:55.728071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363723778487:2118] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:52:55.727149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789364556283569:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.720167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365205417358:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:55.719773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:55.730278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789365110654873:2048] [node 1] ICS09 handshake done sender: [3:7577789365110654856:2115] self: [3:7577789365110654860:0] peer: [1:7577789367008272776:0] socket: 219 qp: -1
2025-11-28T14:52:55.732383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789365018790629:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.728226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363723778497:2048] [node 1] ICS09 handshake done sender: [6:7577789363723778477:2113] self: [6:7577789363723778488:0] peer: [1:7577789367008272772:0] socket: 221 qp: -1
2025-11-28T14:52:55.721203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365205417359:2108] [node 5] ICS09 handshake done sender: [2:7577789365205417343:2112] self: [2:7577789365205417339:0] peer: [5:7577789364556283572:0] socket: 228 qp: -1
2025-11-28T14:52:55.728264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363723778497:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:55.730316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789365110654873:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:55.720090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.727500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789364556283581:2050] [node 3] ICS09 handshake done sender: [5:7577789364556283566:2115] self: [5:7577789364556283571:0] peer: [3:7577789365110654846:0] socket: 222 qp: -1
2025-11-28T14:52:55.732840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365018790633:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16203
2025-11-28T14:52:55.721217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365205417360:2048] [node 1] ICS09 handshake done sender: [2:7577789365205417347:2115] self: [2:7577789365205417352:0] peer: [1:7577789367008272774:0] socket: 220 qp: -1
2025-11-28T14:52:55.729209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.733642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789367008272781:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.730348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.721881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365205417359:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:55.730308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363723778483:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.731902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789365110654852:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:55.732891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789365018790630:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.727545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789364556283581:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:55.733668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789367008272781:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.722172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365205417360:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:55.731574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363723778467:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.722190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:55.734004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789367008272780:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.732008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789365110654872:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44910
2025-11-28T14:52:55.731583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363723778407:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.734031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789367008272780:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.727611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.722201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:55.733150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789365018790547:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.731617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.732430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789365110654840:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.722765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.729377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789364556283492:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.733129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:55.732088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789365205417367:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2094
2025-11-28T14:52:55.737300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789365205417366:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2094
2025-11-28T14:52:55.733716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789365110654840:2107] [node 4] ICP22 created new session: [3:7577789365110654876:2107]
2025-11-28T14:52:55.731690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363723778407:2049] [node 2] ICP22 created new session: [6:7577789363723778501:2049]
2025-11-28T14:52:55.733878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789365018790549:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28335) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789365018790627:2117] held: no
2025-11-28T14:52:55.734162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789365110654876:2107] [node 4] ICS09 handshake done sender: [3:7577789365110654852:2113] self: [3:7577789365110654849:0] peer: [4:7577789365018790632:0] socket: 267 qp: -1
2025-11-28T14:52:55.730293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.738248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789367008272781:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:55.731733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363723778501:2049] [node 2] ICS09 handshake done sender: [6:7577789363723778483:2117] self: [6:7577789363723778490:0] peer: [2:7577789365205417336:0] socket: 236 qp: -1
2025-11-28T14:52:55.734194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789365110654876:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:55.738259Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789367008272780:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:55.731772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363723778501:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:55.731152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789364556283580:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34158
2025-11-28T14:52:55.735390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.731961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.732002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:55.731539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789364556283492:2049] [node 2] ICP22 created new session: [5:7577789364556283583:2049]
2025-11-28T14:52:55.732057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363723778467:2106] [node 5] ICP22 created new session: [6:7577789363723778503:2106]
2025-11-28T14:52:55.738282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789365018790631:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.738417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789365205417366:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.732130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363723778503:2106] [node 5] ICS09 handshake done sender: [6:7577789363723778487:2118] self: [6:7577789363723778491:0] peer: [5:7577789364556283558:0] socket: 247 qp: -1
2025-11-28T14:52:55.731593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789364556283583:2049] [node 2] ICS09 handshake done sender: [5:7577789364556283569:2116] self: [5:7577789364556283572:0] peer: [2:7577789365205417339:0] socket: 235 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.738806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789365018790548:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.732161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363723778503:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:55.731796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789364556283583:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:55.732340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.731841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.739368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789367008272706:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(17499603ns) {HandleHandshakeStatus:329(17496769ns) {DropIncomingHandshake:442(9490ns) DropOutgoingHandshake:457(307ns) SwitchToState:179(145ns) ProcessPendingSessionEvents:495(9624404ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9539825ns max# 1325312ns}}}
2025-11-28T14:52:55.739456Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789365110654841:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21797364ns) {HandleHandshakeStatus:329(21795148ns) {DropIncomingHandshake:442(12335ns) DropOutgoingHandshake:457(531ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(17434735ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16782362ns max# 1537169ns}}}
2025-11-28T14:52:55.739367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789365205417345:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.739495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789367008272770:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.731980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789364556283580:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.739678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789365110654872:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.740967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789367008272769:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.739397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789365205417345:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.739824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365018790637:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16203
2025-11-28T14:52:55.732287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789364556283491:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.741190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789365110654778:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.740980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:55.741346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778506:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
2025-11-28T14:52:55.741294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789365018790627:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:55.741209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:55.739326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789364556283586:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34158
2025-11-28T14:52:55.742007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789365205417345:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:55.741260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789367008272769:2107] [node 4] ICP22 created new session: [1:7577789367008272807:2107]
2025-11-28T14:52:55.741240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789364556283580:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:55.742275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789365110654842:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23992704ns) {HandleHandshakeStatus:329(23990870ns) {DropIncomingHandshake:442(139ns) DropOutgoingHandshake:457(334ns) SwitchToState:179(193ns) ProcessPendingSessionEvents:495(18366957ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18127424ns max# 1446003ns}}}
2025-11-28T14:52:55.741663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789365018790629:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:55.741313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789367008272807:2107] [node 4] ICS09 handshake done sender: [1:7577789367008272780:2113] self: [1:7577789367008272778:0] peer: [4:7577789365018790634:0] socket: 211 qp: -1
2025-11-28T14:52:55.742196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789365205417344:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.745940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789363723778506:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.741412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789364556283491:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.741754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789365018790549:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.741324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789367008272770:2108] [node 5] ICP22 created new session: [1:7577789367008272806:2108]
2025-11-28T14:52:55.742651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789365110654842:2109] [node 6] ICP18 handshake (actor [3:7577789365110654859:2118]) is too old
2025-11-28T14:52:55.742665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.741351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789367008272807:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:55.742230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789365205417344:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.741389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.744818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789364556283491:2048] [node 1] ICP22 created new session: [5:7577789364556283587:2048]
2025-11-28T14:52:55.741787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:55.741832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789365018790549:2050] [node 3] ICP22 created new session: [4:7577789365018790639:2050]
2025-11-28T14:52:55.741784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789367008272771:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18983498ns) {HandleHandshakeStatus:329(18981819ns) {DropIncomingHandshake:442(86ns) DropOutgoingHandshake:457(202ns) SwitchToState:179(531ns) ProcessPendingSessionEvents:495(11938854ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11853871ns max# 1607591ns}}}
2025-11-28T14:52:55.745219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789364556283555:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18019499ns) {HandleHandshakeStatus:329(18018174ns) {DropIncomingHandshake:442(205ns) DropOutgoingHandshake:457(232ns) SwitchToState:179(304ns) ProcessPendingSessionEvents:495(12872488ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12230803ns max# 1669092ns}}}
2025-11-28T14:52:55.742704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789365205417335:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18945129ns) {HandleHandshakeStatus:329(18825969ns) {DropIncomingHandshake:442(658ns) DropOutgoingHandshake:457(406ns) SwitchToState:179(174ns) ProcessPendingSessionEvents:495(13943316ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13842372ns max# 1207154ns}}}
2025-11-28T14:52:55.742351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365018790639:2050] [node 3] ICS09 handshake done sender: [4:7577789365018790627:2117] self: [4:7577789365018790632:0] peer: [3:7577789365110654849:0] socket: 268 qp: -1
2025-11-28T14:52:55.742070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789367008272707:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(20014795ns) {HandleHandshakeStatus:329(20012547ns) {DropIncomingHandshake:442(18347ns) DropOutgoingHandshake:457(1457ns) SwitchToState:179(157ns) ProcessPendingSessionEvents:495(14810501ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14365419ns max# 1260952ns}}}
2025-11-28T14:52:55.746098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789364556283587:2048] [node 1] ICS09 handshake done sender: [5:7577789364556283580:2119] self: [5:7577789364556283585:0] peer: [1:7577789367008272777:0] socket: 231 qp: -1
2025-11-28T14:52:55.742171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789367008272806:2108] [node 5] ICS09 handshake done sender: [1:7577789367008272781:2114] self: [1:7577789367008272777:0] peer: [5:7577789364556283585:0] socket: 210 qp: -1
2025-11-28T14:52:55.745101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789365205417333:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.747001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789363723778468:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.746480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789364556283587:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:55.742697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365018790639:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:55.742200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789367008272806:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:55.742636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:55.746823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.746512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.746511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789365205417334:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(21776408ns) {HandleHandshakeStatus:329(21774001ns) {DropIncomingHandshake:442(7617ns) DropOutgoingHandshake:457(2737ns) SwitchToState:179(126ns) ProcessPendingSessionEvents:495(13871344ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13608499ns max# 2001031ns}}}
2025-11-28T14:52:55.742712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789365018790547:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.746563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789365205417333:2107] [node 4] ICP22 created new session: [2:7577789365205417370:2107]
2025-11-28T14:52:55.742724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.750358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789365110654879:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44910
2025-11-28T14:52:55.746834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365205417370:2107] [node 4] ICS09 handshake done sender: [2:7577789365205417345:2114] self: [2:7577789365205417342:0] peer: [4:7577789365018790638:0] socket: 243 qp: -1
2025-11-28T14:52:55.744828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:55.746873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365205417370:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:55.746980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.745087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789365018790547:2048] [node 1] ICP22 created new session: [4:7577789365018790642:2048]
2025-11-28T14:52:55.750873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789365110654872:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.747838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789365205417344:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:55.746504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365018790642:2048] [node 1] ICS09 handshake done sender: [4:7577789365018790629:2118] self: [4:7577789365018790634:0] peer: [1:7577789367008272778:0] socket: 232 qp: -1
2025-11-28T14:52:55.751769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789365110654778:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.748955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789365205417270:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20607478ns) {HandleHandshakeStatus:329(20605082ns) {DropIncomingHandshake:442(452ns) DropOutgoingHandshake:457(4496ns) SwitchToState:179(466ns) ProcessPendingSessionEvents:495(14076800ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13968561ns max# 1312719ns}}}
2025-11-28T14:52:55.751800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.746647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365018790642:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:55.749263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789365205417270:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:55.751842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789365110654778:2049] [node 2] ICP22 created new session: [3:7577789365110654880:2049]
2025-11-28T14:52:55.751887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789365110654880:2049] [node 2] ICS09 handshake done sender: [3:7577789365110654872:2120] self: [3:7577789365110654878:0] peer: [2:7577789365205417341:0] socket: 275 qp: -1
2025-11-28T14:52:55.752047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789365110654880:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:55.746998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:55.750347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789365205417271:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.752321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.747166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789365018790640:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16203
2025-11-28T14:52:55.750589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789365205417335:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:55.750707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:55.750876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789365205417271:2049] [node 3] ICP22 created new session: [2:7577789365205417373:2049]
No RDMA context found for GID: 2025-11-28T14:52:55.751698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789365205417373:2049] [node 3] ICS09 handshake done sender: [2:7577789365205417344:2113] self: [2:7577789365205417341:0] peer: [3:7577789365110654878:0] socket: 274 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:55.749236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789365018790548:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11784) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789365018790631:2120] held: no
2025-11-28T14:52:55.752843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789365110654840:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16859076ns) {HandleHandshakeStatus:329(16856258ns) {DropIncomingHandshake:442(12694ns) DropOutgoingHandshake:457(1182ns) SwitchToState:179(281ns) ProcessPendingSessionEvents:495(11067176ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9692525ns max# 1293844ns}}}
2025-11-28T14:52:55.751864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789365205417373:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:55.752026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:55.749246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789365018790620:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.753189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789364556283589:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34158
2025-11-28T14:52:55.749267Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789365018790620:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:55.752937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789363723778468:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789363723778506:2123] held: no
2025-11-28T14:52:55.751392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789365018790620:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:55.751562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789365018790611:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.751840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789365018790631:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:55.754945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789364556283589:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.751887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:55.755399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789363723778508:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14869
2025-11-28T14:52:55.755574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789364556283554:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:55.751939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789365018790548:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.752089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789365018790611:2110] [node 6] ICP22 created new session: [4:7577789365018790646:2110]
2025-11-28T14:52:55.752566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365018790646:2110] [node 6] ICS09 handshake done sender: [4:7577789365018790620:2113] self: [4:7577789365018790618:0] peer: [6:7577789363723778507:0] socket: 277 qp: -1
2025-11-28T14:52:55.752597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365018790646:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:55.752630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:55.752643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:55.752660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789365018790548:2049] [node 2] ICP22 created new session: [4:7577789365018790648:2049]
2025-11-28T14:52:55.752836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365018790648:2049] [node 2] ICS09 handshake done sender: [4:7577789365018790631:2120] self: [4:7577789365018790638:0] peer: [2:7577789365205417342:0] socket: 271 qp: -1
2025-11-28T14:52:55.752856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365018790648:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:55.752880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:55.755757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789365018790622:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:55.756927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789363723778506:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:55.755775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789365018790622:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:55.757162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789364556283591:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34158
2025-11-28T14:52:55.757164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789363723778468:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.757214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:55.757258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789363723778468:2107] [node 4] ICP22 created new session: [6:7577789363723778509:2107]
2025-11-28T14:52:55.757313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789363723778509:2107] [node 4] ICS09 handshake done sender: [6:7577789363723778506:2123] self: [6:7577789363723778507:0] peer: [4:7577789365018790618:0] socket: 278 qp: -1
2025-11-28T14:52:55.757369Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789365018790622:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:55.757368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789363723778509:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:55.757474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.757639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789364556283589:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:55.757607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789365018790612:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:55.757673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:55.757976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789364556283554:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:55.758154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:55.757815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789365018790612:2108] [node 5] ICP22 created new session: [4:7577789365018790651:2108]
2025-11-28T14:52:55.758188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789364556283554:2108] [node 4] ICP22 created new session: [5:7577789364556283592:2108]
2025-11-28T14:52:55.757865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789365018790651:2108] [node 5] ICS09 handshake done sender: [4:7577789365018790622:2115] self: [4:7577789365018790615:0] peer: [5:7577789364556283590:0] socket: 238 qp: -1
2025-11-28T14:52:55.758327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789364556283592:2108] [node 4] ICS09 handshake done sender: [5:7577789364556283589:2121] self: [5:7577789364556283590:0] peer: [4:7577789365018790615:0] socket: 239 qp: -1
2025-11-28T14:52:55.757969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789365018790651:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:55.758356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789364556283592:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:55.758261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:55.758390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:55.759106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789364556283554:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:55.816009Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:55.816172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789365205417360:2048] [node 1] ICS07 socket disconnect 220 reason# EndOfStream
2025-11-28T14:52:55.816195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789365205417360:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.816268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.816310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789365205417360:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:55.817149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:55.817309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789365110654873:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-28T14:52:55.817367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789365110654777:2048] [node 1] ICP30 unregister session Session# [3:7577789365110654873:2048] VirtualId# [3:7577789365110654860:0]
2025-11-28T14:52:55.817401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789365110654873:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.817486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.817544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.818525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789365205417270:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:55.818592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789365205417360:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:55.819946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:55.820153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789365018790642:2048] [node 1] ICS07 socket disconnect 232 reason# EndOfStream
2025-11-28T14:52:55.820187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789365018790642:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.820270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.820254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789365205417270:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.820299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789365205417360:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.820381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789365018790642:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:55.820317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789365205417270:2048] [node 1] ICP30 unregister session Session# [2:7577789365205417360:2048] VirtualId# [2:7577789365205417352:0]
2025-11-28T14:52:55.820484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.820505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789365205417270:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.820514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.820779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:55.820964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789364556283587:2048] [node 1] ICS07 socket disconnect 231 reason# EndOfStream
2025-11-28T14:52:55.821005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789364556283587:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.821068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.821115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789365018790547:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.821114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789364556283587:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:55.821285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789365018790642:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.821353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789365018790547:2048] [node 1] ICP30 unregister session Session# [4:7577789365018790642:2048] VirtualId# [4:7577789365018790634:0]
2025-11-28T14:52:55.821408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.821449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789365018790547:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.821461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.821888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789364556283491:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.821963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789364556283587:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.821884Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:55.821987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789364556283491:2048] [node 1] ICP30 unregister session Session# [5:7577789364556283587:2048] VirtualId# [5:7577789364556283585:0]
2025-11-28T14:52:55.822101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.822132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789364556283491:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.822140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.822174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363723778497:2048] [node 1] ICS01 socket: 221 reason# EndOfStream
2025-11-28T14:52:55.822228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363723778406:2048] [node 1] ICP30 unregister session Session# [6:7577789363723778497:2048] VirtualId# [6:7577789363723778488:0]
2025-11-28T14:52:55.822276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363723778497:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.822367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:55.822431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:55.831192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789365018790547:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:55.832217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789364556283491:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:55.905585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:55.905817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789365110654880:2049] [node 2] ICS01 socket: 275 reason# EndOfStream
2025-11-28T14:52:55.905917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789365110654778:2049] [node 2] ICP30 unregister session Session# [3:7577789365110654880:2049] VirtualId# [3:7577789365110654878:0]
2025-11-28T14:52:55.905958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789365110654880:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.906074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:55.906142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:55.907772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:55.907930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789365018790648:2049] [node 2] ICS01 socket: 271 reason# EndOfStream
2025-11-28T14:52:55.907961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789365018790548:2049] [node 2] ICP30 unregister session Session# [4:7577789365018790648:2049] VirtualId# [4:7577789365018790638:0]
2025-11-28T14:52:55.908006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789365018790648:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.908120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:55.908173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:55.908531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:55.908728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789364556283583:2049] [node 2] ICS07 socket disconnect 235 reason# EndOfStream
2025-11-28T14:52:55.908760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789364556283583:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.908853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:55.908907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789364556283583:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:55.909305Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:55.909498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363723778501:2049] [node 2] ICS01 socket: 236 reason# EndOfStream
2025-11-28T14:52:55.909478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789364556283492:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11784) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.909519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789364556283583:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.909673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363723778407:2049] [node 2] ICP30 unregister session Session# [6:7577789363723778501:2049] VirtualId# [6:7577789363723778490:0]
2025-11-28T14:52:55.909618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789364556283492:2049] [node 2] ICP30 unregister session Session# [5:7577789364556283583:2049] VirtualId# [5:7577789364556283572:0]
2025-11-28T14:52:55.909657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:55.909730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363723778501:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.909700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789364556283492:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.909805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:55.909713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.909864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:55.919246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789364556283492:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:55.964961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:55.965191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789365018790639:2050] [node 3] ICS01 socket: 268 reason# EndOfStream
2025-11-28T14:52:55.965237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789365018790549:2050] [node 3] ICP30 unregister session Session# [4:7577789365018790639:2050] VirtualId# [4:7577789365018790632:0]
2025-11-28T14:52:55.965275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789365018790639:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.965366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:55.965422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:55.966166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:55.966335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789364556283581:2050] [node 3] ICS01 socket: 222 reason# EndOfStream
2025-11-28T14:52:55.966365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789364556283493:2050] [node 3] ICP30 unregister session Session# [5:7577789364556283581:2050] VirtualId# [5:7577789364556283571:0]
2025-11-28T14:52:55.966389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789364556283581:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.966484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:55.966528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:55.966971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:55.967183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789363723778496:2050] [node 3] ICS07 socket disconnect 223 reason# EndOfStream
2025-11-28T14:52:55.967238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363723778496:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:55.967333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:55.967387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789363723778496:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:55.968157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789363723778408:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28335) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:55.968205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363723778496:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:55.968267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363723778408:2050] [node 3] ICP30 unregister session Session# [6:7577789363723778496:2050] VirtualId# [6:7577789363723778489:0]
2025-11-28T14:52:55.968306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:55.968332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789363723778408:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28335) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.968344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28335) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:55.978719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789363723778408:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:56.037986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:56.038145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789364556283592:2108] [node 4] ICS07 socket disconnect 239 reason# EndOfStream
2025-11-28T14:52:56.038174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789364556283592:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.038255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:56.038300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789364556283592:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:56.038933Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:56.038945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789364556283554:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19263) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.039028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789364556283592:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.039068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789364556283554:2108] [node 4] ICP30 unregister session Session# [5:7577789364556283592:2108] VirtualId# [5:7577789364556283590:0]
2025-11-28T14:52:56.039156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:56.039156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363723778509:2107] [node 4] ICS01 socket: 278 reason# EndOfStream
2025-11-28T14:52:56.039189Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789364556283554:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.039204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19263) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.039212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363723778468:2107] [node 4] ICP30 unregister session Session# [6:7577789363723778509:2107] VirtualId# [6:7577789363723778507:0]
2025-11-28T14:52:56.039249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363723778509:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.039334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:56.039401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:56.049307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789364556283554:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:56.119274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:56.119552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789363723778503:2106] [node 5] ICS01 socket: 247 reason# EndOfStream
2025-11-28T14:52:56.119616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789363723778467:2106] [node 5] ICP30 unregister session Session# [6:7577789363723778503:2106] VirtualId# [6:7577789363723778491:0]
2025-11-28T14:52:56.119656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789363723778503:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.119774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:56.119836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:56.219689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789371432825985:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:56.219728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789371432825986:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:56.236168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370906009389:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.236205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370906009390:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:56.253146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789371119409180:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.253188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789371119409181:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:56.271480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371924468813:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.271517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371924468814:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:56.271541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371924468815:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:56.288081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789372022749581:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.288119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789372022749582:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:56.288149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789372022749583:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:56.304433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368558597872:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.304476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368558597873:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:56.304503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368558597874:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:56.377889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368558597933:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:56.382797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789371119409243:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:56.382770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370906009452:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:56.383669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370906009453:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:56.383403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789371432826048:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:56.383703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789371119409244:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:56.384298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789372022749644:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:56.384342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371924468878:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:56.384468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789371432826049:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:56.384502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370906009454:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:56.384666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789371119409245:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:56.384722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368558597937:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:56.384767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789372022749645:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:56.384807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371924468879:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:56.384931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789371432826050:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.393690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789372022749656:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47630
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.393827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370906009465:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38680
2025-11-28T14:52:56.393909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370906009466:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38680
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.393981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789371432826061:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27910
2025-11-28T14:52:56.394651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789371432826062:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27910
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.395590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789371432826066:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27910
2025-11-28T14:52:56.395651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789371432826067:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27910
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.395784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370906009470:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38680
2025-11-28T14:52:56.395862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370906009471:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38680
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.403036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371924468893:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37697
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.403033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789371119409256:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6002
2025-11-28T14:52:56.403119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371924468894:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37697
2025-11-28T14:52:56.403154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789371119409257:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6002
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.403118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789372022749656:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.403232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789371119409258:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6002
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.403312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789371119409259:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6002
2025-11-28T14:52:56.403820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789371119409256:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.403886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789371119409258:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.403847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826061:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789371119409257:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826066:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:56.404232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371924468894:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826062:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371924468895:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37697
2025-11-28T14:52:56.404469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789371924468813:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.404623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826067:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370906009465:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371924468893:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370906009466:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789371924468814:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.404758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826056:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789371432826056:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.404783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370906009471:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.404994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370906009470:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.405258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370906009460:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.405278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789370906009460:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.405330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789371432826068:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27910
2025-11-28T14:52:56.405722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789371119409259:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.405864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789371432826056:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.406007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371924468898:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37697
2025-11-28T14:52:56.405993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370906009472:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38680
2025-11-28T14:52:56.406162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789371432826048:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.406209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.406248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789371432826048:2107] [node 4] ICP22 created new session: [1:7577789371432826074:2107]
2025-11-28T14:52:56.406218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789370906009389:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.406282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789371432826074:2107] [node 4] ICS09 handshake done sender: [1:7577789371432826056:2112] self: [1:7577789371432826055:0] peer: [4:7577789371924468896:0] socket: 239 qp: -1
2025-11-28T14:52:56.406310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789371432826074:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:56.406332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.406617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371924468894:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:56.406611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370906009460:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:56.407040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789370906009452:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.407052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789371924468813:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.407099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.407144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:56.407152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370906009452:2107] [node 4] ICP22 created new session: [2:7577789370906009478:2107]
2025-11-28T14:52:56.407189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371924468813:2048] [node 1] ICP22 created new session: [4:7577789371924468900:2048]
2025-11-28T14:52:56.407200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370906009478:2107] [node 4] ICS09 handshake done sender: [2:7577789370906009460:2112] self: [2:7577789370906009459:0] peer: [4:7577789371924468897:0] socket: 238 qp: -1
2025-11-28T14:52:56.407234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371924468900:2048] [node 1] ICS09 handshake done sender: [4:7577789371924468894:2118] self: [4:7577789371924468896:0] peer: [1:7577789371432826055:0] socket: 250 qp: -1
2025-11-28T14:52:56.407223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370906009478:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:56.407259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.407299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371924468900:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.407334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:56.407966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826052:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.407987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789371432826052:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.408983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789371119409180:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.409325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789371119409181:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.409412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370906009472:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.409555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370906009458:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.409583Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789370906009458:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.409685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789372022749661:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47630
2025-11-28T14:52:56.410870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826068:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.410957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826054:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.410977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789371432826054:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.412026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789371119409267:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6002
2025-11-28T14:52:56.412253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789371432826052:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.412290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370906009458:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.412351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789372022749662:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47630
2025-11-28T14:52:56.413074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789371432825985:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.413412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789372022749661:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.413418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.413448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789371432825985:2048] [node 2] ICP22 created new session: [1:7577789371432826080:2048]
2025-11-28T14:52:56.413452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789370906009390:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.413528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.413493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789371432826080:2048] [node 2] ICS09 handshake done sender: [1:7577789371432826052:2110] self: [1:7577789371432826051:0] peer: [2:7577789370906009477:0] socket: 230 qp: -1
2025-11-28T14:52:56.413612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370906009390:2049] [node 3] ICP22 created new session: [2:7577789370906009484:2049]
2025-11-28T14:52:56.413587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789371432826080:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:56.413608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789372022749582:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.413608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789371119409259:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:56.413670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370906009484:2049] [node 3] ICS09 handshake done sender: [2:7577789370906009458:2111] self: [2:7577789370906009457:0] peer: [3:7577789371119409266:0] socket: 235 qp: -1
2025-11-28T14:52:56.413644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:56.413705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370906009484:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:56.413753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.413875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789371119409268:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6002
2025-11-28T14:52:56.413909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789371119409181:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.413902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371924468899:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37697
2025-11-28T14:52:56.413971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.413998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789371119409181:2049] [node 2] ICP22 created new session: [3:7577789371119409269:2049]
2025-11-28T14:52:56.414020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789371119409269:2049] [node 2] ICS09 handshake done sender: [3:7577789371119409259:2118] self: [3:7577789371119409266:0] peer: [2:7577789370906009457:0] socket: 248 qp: -1
2025-11-28T14:52:56.414038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789371119409269:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:56.414061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:56.414546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789371924468814:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789371924468893:2117] held: no
2025-11-28T14:52:56.414737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789371432826054:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.415729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371924468893:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:56.416274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789371432825986:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.416337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.416417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789371432825986:2049] [node 3] ICP22 created new session: [1:7577789371432826082:2049]
2025-11-28T14:52:56.416457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371924468895:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.416500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789371924468814:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.416452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789371432826082:2049] [node 3] ICS09 handshake done sender: [1:7577789371432826054:2111] self: [1:7577789371432826053:0] peer: [3:7577789371119409265:0] socket: 231 qp: -1
2025-11-28T14:52:56.416474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789371432826082:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:56.416643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.416490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:56.416653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370906009462:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.416698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371924468814:2049] [node 2] ICP22 created new session: [4:7577789371924468904:2049]
2025-11-28T14:52:56.416689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789370906009462:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.416750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371924468904:2049] [node 2] ICS09 handshake done sender: [4:7577789371924468893:2117] self: [4:7577789371924468897:0] peer: [2:7577789370906009459:0] socket: 249 qp: -1
2025-11-28T14:52:56.416783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371924468904:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:56.416814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:56.417256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789371119409257:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:56.417430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789371119409180:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.417497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:56.417522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789371119409180:2048] [node 1] ICP22 created new session: [3:7577789371119409272:2048]
2025-11-28T14:52:56.417550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789371119409272:2048] [node 1] ICS09 handshake done sender: [3:7577789371119409257:2116] self: [3:7577789371119409265:0] peer: [1:7577789371432826053:0] socket: 246 qp: -1
2025-11-28T14:52:56.417569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789371119409272:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.417592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:56.417962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789372022749662:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.418791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789372022749581:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.418902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789372022749664:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47630
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.419016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789371924468815:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.419015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597951:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
2025-11-28T14:52:56.419084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597952:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.419227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370906009483:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38680
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.420395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789370906009389:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1645) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789370906009470:2117] held: no
2025-11-28T14:52:56.420794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789372022749666:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47630
2025-11-28T14:52:56.421082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789371119409251:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.421104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789371119409251:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.421355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826058:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.421378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789371432826058:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.422448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368558597951:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.422449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789372022749667:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47630
2025-11-28T14:52:56.422466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370906009470:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.422627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789372022749664:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.422689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597953:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
2025-11-28T14:52:56.422735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789371432826058:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.422770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789370906009389:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.422820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.422843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368558597952:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.422870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370906009389:2048] [node 1] ICP22 created new session: [2:7577789370906009488:2048]
2025-11-28T14:52:56.422979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370906009488:2048] [node 1] ICS09 handshake done sender: [2:7577789370906009470:2117] self: [2:7577789370906009477:0] peer: [1:7577789371432826051:0] socket: 241 qp: -1
2025-11-28T14:52:56.422985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368558597872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.423021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370906009488:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.422975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371924468907:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37697
2025-11-28T14:52:56.423068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:56.423076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789371432826049:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.423073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789372022749644:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.423115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: 2025-11-28T14:52:56.423147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789371432826049:2108] [node 5] ICP22 created new session: [1:7577789371432826086:2108]
::ffff:127.1.0.0
2025-11-28T14:52:56.423172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789371432826086:2108] [node 5] ICS09 handshake done sender: [1:7577789371432826058:2113] self: [1:7577789371432826057:0] peer: [5:7577789372022749665:0] socket: 217 qp: -1
2025-11-28T14:52:56.423259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368558597937:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.423259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789371432826086:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:56.423371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371924468887:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.423372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789372022749668:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47630
2025-11-28T14:52:56.423301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:56.423409Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789371924468887:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.424049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789371119409251:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:56.424169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371924468889:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.424195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789371924468889:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.424982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597956:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
2025-11-28T14:52:56.425028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789371119409243:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.424949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789372022749662:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:56.425163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.425209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789371119409243:2107] [node 4] ICP22 created new session: [3:7577789371119409276:2107]
2025-11-28T14:52:56.425247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789371119409276:2107] [node 4] ICS09 handshake done sender: [3:7577789371119409251:2112] self: [3:7577789371119409250:0] peer: [4:7577789371924468906:0] socket: 253 qp: -1
2025-11-28T14:52:56.425254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789372022749581:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.425270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789371119409276:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:56.425316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.425304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:56.425341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789372022749581:2048] [node 1] ICP22 created new session: [5:7577789372022749670:2048]
2025-11-28T14:52:56.425376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789372022749670:2048] [node 1] ICS09 handshake done sender: [5:7577789372022749662:2117] self: [5:7577789372022749665:0] peer: [1:7577789371432826057:0] socket: 265 qp: -1
2025-11-28T14:52:56.425401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789372022749670:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.425428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:56.425759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370906009462:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.425811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371924468895:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.425915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789371432826060:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.425938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789371432826060:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.426693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371924468887:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.426761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597957:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
2025-11-28T14:52:56.427095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371924468889:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:56.427165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789371432826060:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:56.427238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789370906009453:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.427369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:56.427429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789371924468815:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.427542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370906009453:2108] [node 5] ICP22 created new session: [2:7577789370906009490:2108]
2025-11-28T14:52:56.427477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.427630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370906009490:2108] [node 5] ICS09 handshake done sender: [2:7577789370906009462:2113] self: [2:7577789370906009461:0] peer: [5:7577789372022749663:0] socket: 225 qp: -1
2025-11-28T14:52:56.427684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370906009490:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:56.427571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371924468815:2050] [node 3] ICP22 created new session: [4:7577789371924468910:2050]
2025-11-28T14:52:56.427766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:56.427611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371924468910:2050] [node 3] ICS09 handshake done sender: [4:7577789371924468895:2119] self: [4:7577789371924468906:0] peer: [3:7577789371119409250:0] socket: 254 qp: -1
2025-11-28T14:52:56.427646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371924468910:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:56.427683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:56.428046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368558597952:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:56.428283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368558597937:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.428358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.428452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368558597937:2106] [node 4] ICP22 created new session: [6:7577789368558597959:2106]
2025-11-28T14:52:56.428485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789371432826050:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.428505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368558597959:2106] [node 4] ICS09 handshake done sender: [6:7577789368558597952:2117] self: [6:7577789368558597955:0] peer: [4:7577789371924468888:0] socket: 269 qp: -1
2025-11-28T14:52:56.428524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.428537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368558597959:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:56.428582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.428582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789371432826050:2109] [node 6] ICP22 created new session: [1:7577789371432826089:2109]
2025-11-28T14:52:56.428637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789371432826089:2109] [node 6] ICS09 handshake done sender: [1:7577789371432826060:2114] self: [1:7577789371432826059:0] peer: [6:7577789368558597954:0] socket: 237 qp: -1
2025-11-28T14:52:56.428659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789371432826089:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:56.428684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.430469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597958:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
2025-11-28T14:52:56.430491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789372022749666:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.430622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789371924468878:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.430718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:56.430781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371924468878:2108] [node 5] ICP22 created new session: [4:7577789371924468912:2108]
2025-11-28T14:52:56.430842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371924468912:2108] [node 5] ICS09 handshake done sender: [4:7577789371924468887:2115] self: [4:7577789371924468886:0] peer: [5:7577789372022749669:0] socket: 219 qp: -1
2025-11-28T14:52:56.430969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371924468912:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:56.431002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:56.432049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789372022749661:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:56.432301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789372022749582:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789372022749661:2116] held: no
2025-11-28T14:52:56.432771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789372022749582:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.432804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.432768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789371924468879:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.432887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789372022749582:2049] [node 2] ICP22 created new session: [5:7577789372022749673:2049]
2025-11-28T14:52:56.432796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.432925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789372022749673:2049] [node 2] ICS09 handshake done sender: [5:7577789372022749661:2116] self: [5:7577789372022749663:0] peer: [2:7577789370906009461:0] socket: 226 qp: -1
2025-11-28T14:52:56.432823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371924468879:2110] [node 6] ICP22 created new session: [4:7577789371924468914:2110]
2025-11-28T14:52:56.432954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789372022749673:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:56.432905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371924468914:2110] [node 6] ICS09 handshake done sender: [4:7577789371924468889:2116] self: [4:7577789371924468888:0] peer: [6:7577789368558597955:0] socket: 252 qp: -1
2025-11-28T14:52:56.432999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:56.432934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371924468914:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:56.432962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:56.433125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368558597951:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:56.433355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789368558597872:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1645) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789368558597951:2116] held: no
2025-11-28T14:52:56.433732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368558597872:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.433760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: 2025-11-28T14:52:56.433785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368558597872:2048] [node 1] ICP22 created new session: [6:7577789368558597961:2048]
::ffff:127.1.0.0
2025-11-28T14:52:56.433808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368558597961:2048] [node 1] ICS09 handshake done sender: [6:7577789368558597951:2116] self: [6:7577789368558597954:0] peer: [1:7577789371432826059:0] socket: 256 qp: -1
2025-11-28T14:52:56.433826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368558597961:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.433850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:56.433845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789372022749672:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47630
2025-11-28T14:52:56.434678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368558597956:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.434744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789372022749583:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.435111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789372022749664:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:56.435183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789371119409253:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.435210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789371119409253:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.435829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789371119409253:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.435861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789372022749644:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.435899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.435864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368558597953:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.435928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789372022749644:2108] [node 4] ICP22 created new session: [5:7577789372022749678:2108]
2025-11-28T14:52:56.436002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789372022749678:2108] [node 4] ICS09 handshake done sender: [5:7577789372022749664:2118] self: [5:7577789372022749669:0] peer: [4:7577789371924468886:0] socket: 255 qp: -1
2025-11-28T14:52:56.436033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789372022749678:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:56.436060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.436072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789371119409244:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.436137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:56.436169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789371119409244:2108] [node 5] ICP22 created new session: [3:7577789371119409279:2108]
2025-11-28T14:52:56.436211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789371119409279:2108] [node 5] ICS09 handshake done sender: [3:7577789371119409253:2113] self: [3:7577789371119409252:0] peer: [5:7577789372022749676:0] socket: 234 qp: -1
2025-11-28T14:52:56.436235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789371119409279:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:56.436277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.436584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371924468916:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37697
2025-11-28T14:52:56.437113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368558597874:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.437182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368558597873:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.437400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597964:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
2025-11-28T14:52:56.437546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370906009464:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.437588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789370906009464:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.438256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789371119409255:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.438285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789371119409255:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.438356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370906009464:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:56.439147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789370906009454:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.439244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.439325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370906009454:2109] [node 6] ICP22 created new session: [2:7577789370906009493:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.439388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370906009493:2109] [node 6] ICS09 handshake done sender: [2:7577789370906009464:2114] self: [2:7577789370906009463:0] peer: [6:7577789368558597966:0] socket: 271 qp: -1
2025-11-28T14:52:56.439425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370906009493:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:56.439413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597967:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
2025-11-28T14:52:56.439444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789371119409281:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6002
2025-11-28T14:52:56.439487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:56.439588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789371119409255:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:56.439918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789371119409245:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.439964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.439999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789371119409245:2109] [node 6] ICP22 created new session: [3:7577789371119409283:2109]
2025-11-28T14:52:56.440031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789371119409283:2109] [node 6] ICS09 handshake done sender: [3:7577789371119409255:2114] self: [3:7577789371119409254:0] peer: [6:7577789368558597965:0] socket: 278 qp: -1
2025-11-28T14:52:56.440059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789371119409283:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:56.440102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:56.440187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368558597953:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:56.440679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368558597873:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.440734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.440810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368558597873:2049] [node 2] ICP22 created new session: [6:7577789368558597969:2049]
2025-11-28T14:52:56.440883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368558597969:2049] [node 2] ICS09 handshake done sender: [6:7577789368558597953:2118] self: [6:7577789368558597966:0] peer: [2:7577789370906009463:0] socket: 273 qp: -1
2025-11-28T14:52:56.440918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368558597969:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:56.440966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:56.441082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789372022749583:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789372022749666:2119] held: no
2025-11-28T14:52:56.441284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789371119409281:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.441408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789371119409245:2109] [node 6] ICP18 handshake (actor [3:7577789371119409281:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.441783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789372022749680:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47630
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.442353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597968:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
2025-11-28T14:52:56.443405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789372022749666:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.443586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789372022749583:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.443634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.443678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789372022749583:2050] [node 3] ICP22 created new session: [5:7577789372022749681:2050]
2025-11-28T14:52:56.443717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789372022749681:2050] [node 3] ICS09 handshake done sender: [5:7577789372022749666:2119] self: [5:7577789372022749676:0] peer: [3:7577789371119409252:0] socket: 270 qp: -1
2025-11-28T14:52:56.443744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789372022749681:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:56.443780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:56.444372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789368558597874:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789368558597956:2119] held: no
2025-11-28T14:52:56.444906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368558597956:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.445295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368558597964:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.445323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368558597874:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.445361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.445390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368558597874:2050] [node 3] ICP22 created new session: [6:7577789368558597971:2050]
2025-11-28T14:52:56.445419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368558597971:2050] [node 3] ICS09 handshake done sender: [6:7577789368558597956:2119] self: [6:7577789368558597965:0] peer: [3:7577789371119409254:0] socket: 279 qp: -1
2025-11-28T14:52:56.445455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368558597971:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:56.445479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:56.446343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368558597933:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.446514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789372022749655:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.446536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789372022749655:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.447094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789372022749655:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.447268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368558597974:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27916
2025-11-28T14:52:56.447337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789372022749645:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.447376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.447409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789372022749645:2109] [node 6] ICP22 created new session: [5:7577789372022749684:2109]
2025-11-28T14:52:56.447448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789372022749684:2109] [node 6] ICS09 handshake done sender: [5:7577789372022749655:2114] self: [5:7577789372022749654:0] peer: [6:7577789368558597973:0] socket: 227 qp: -1
2025-11-28T14:52:56.447476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789372022749684:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:56.447518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:56.447593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368558597964:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.447753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368558597933:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.447780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:56.447803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368558597933:2105] [node 5] ICP22 created new session: [6:7577789368558597975:2105]
2025-11-28T14:52:56.447834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368558597975:2105] [node 5] ICS09 handshake done sender: [6:7577789368558597964:2122] self: [6:7577789368558597973:0] peer: [5:7577789372022749654:0] socket: 242 qp: -1
2025-11-28T14:52:56.447860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368558597975:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:56.447882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:56.464700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:56.464886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789370906009488:2048] [node 1] ICS01 socket: 241 reason# EndOfStream
2025-11-28T14:52:56.464948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789370906009389:2048] [node 1] ICP30 unregister session Session# [2:7577789370906009488:2048] VirtualId# [2:7577789370906009477:0]
2025-11-28T14:52:56.464985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789370906009488:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.465066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.465110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.466012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:56.466159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789371119409272:2048] [node 1] ICS07 socket disconnect 246 reason# EndOfStream
2025-11-28T14:52:56.466182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789371119409272:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.466256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.466292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789371119409272:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:56.468253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789371119409180:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1645) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:56.468283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789371119409272:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:56.469121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789371119409180:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1645) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.469163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789371119409272:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.469188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789371119409180:2048] [node 1] ICP30 unregister session Session# [3:7577789371119409272:2048] VirtualId# [3:7577789371119409265:0]
2025-11-28T14:52:56.469271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.469298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789371119409180:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1645) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.469308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1645) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.469316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:56.469531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789371924468900:2048] [node 1] ICS01 socket: 250 reason# EndOfStream
2025-11-28T14:52:56.469589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789371924468813:2048] [node 1] ICP30 unregister session Session# [4:7577789371924468900:2048] VirtualId# [4:7577789371924468896:0]
2025-11-28T14:52:56.469620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789371924468900:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.469707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.469761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.470092Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:56.470228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789372022749670:2048] [node 1] ICS07 socket disconnect 265 reason# EndOfStream
2025-11-28T14:52:56.470254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789372022749670:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.470303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.470343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789372022749670:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:56.470830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:56.470875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789372022749581:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1645) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.470917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789372022749670:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.470946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789372022749581:2048] [node 1] ICP30 unregister session Session# [5:7577789372022749670:2048] VirtualId# [5:7577789372022749665:0]
2025-11-28T14:52:56.471031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.471060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789372022749581:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1645) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.471069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1645) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.471133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368558597961:2048] [node 1] ICS01 socket: 256 reason# EndOfStream
2025-11-28T14:52:56.471178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368558597872:2048] [node 1] ICP30 unregister session Session# [6:7577789368558597961:2048] VirtualId# [6:7577789368558597954:0]
2025-11-28T14:52:56.471214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368558597961:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.471332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.471389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.479388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789371119409180:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:56.480447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789372022749581:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:56.503999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:56.504179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789371119409269:2049] [node 2] ICS07 socket disconnect 248 reason# EndOfStream
2025-11-28T14:52:56.504206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789371119409269:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.504295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:56.504347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789371119409269:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:56.506445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:56.506670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789371924468904:2049] [node 2] ICS01 socket: 249 reason# EndOfStream
2025-11-28T14:52:56.506726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789371924468814:2049] [node 2] ICP30 unregister session Session# [4:7577789371924468904:2049] VirtualId# [4:7577789371924468897:0]
2025-11-28T14:52:56.506768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789371924468904:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.506845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:56.506894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:56.507205Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789371119409181:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6295) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:56.507237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789371119409269:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:56.507502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:56.507671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789372022749673:2049] [node 2] ICS01 socket: 226 reason# EndOfStream
2025-11-28T14:52:56.507710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789372022749582:2049] [node 2] ICP30 unregister session Session# [5:7577789372022749673:2049] VirtualId# [5:7577789372022749663:0]
2025-11-28T14:52:56.507735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789372022749673:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.507726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789371119409181:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6295) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.507790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:56.507760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789371119409269:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.507835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:56.507779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789371119409181:2049] [node 2] ICP30 unregister session Session# [3:7577789371119409269:2049] VirtualId# [3:7577789371119409266:0]
2025-11-28T14:52:56.507816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:56.507835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789371119409181:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.507844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.508411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:56.508659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789368558597969:2049] [node 2] ICS07 socket disconnect 273 reason# EndOfStream
2025-11-28T14:52:56.508718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368558597969:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.508802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:56.508846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789368558597969:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:56.509508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789368558597873:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6295) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.509558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368558597969:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.509595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368558597873:2049] [node 2] ICP30 unregister session Session# [6:7577789368558597969:2049] VirtualId# [6:7577789368558597966:0]
2025-11-28T14:52:56.509685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:56.509723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789368558597873:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.509743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.519334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789368558597873:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:56.528404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:56.528577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789371924468910:2050] [node 3] ICS07 socket disconnect 254 reason# EndOfStream
2025-11-28T14:52:56.528604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789371924468910:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.528677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:56.528709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789371924468910:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:56.529208Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:56.529349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789372022749681:2050] [node 3] ICS01 socket: 270 reason# EndOfStream
2025-11-28T14:52:56.529354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789371924468815:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29207) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.529383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789372022749583:2050] [node 3] ICP30 unregister session Session# [5:7577789372022749681:2050] VirtualId# [5:7577789372022749676:0]
2025-11-28T14:52:56.529394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789371924468910:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.529404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789372022749681:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.529414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789371924468815:2050] [node 3] ICP30 unregister session Session# [4:7577789371924468910:2050] VirtualId# [4:7577789371924468906:0]
2025-11-28T14:52:56.529457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:56.529485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:56.529498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:56.529518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789371924468815:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.529531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.529989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:56.530144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368558597971:2050] [node 3] ICS01 socket: 279 reason# EndOfStream
2025-11-28T14:52:56.530180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368558597874:2050] [node 3] ICP30 unregister session Session# [6:7577789368558597971:2050] VirtualId# [6:7577789368558597965:0]
2025-11-28T14:52:56.530200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368558597971:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.530284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:56.530318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:56.539851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789371924468815:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:56.566488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:56.566668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789372022749678:2108] [node 4] ICS07 socket disconnect 255 reason# EndOfStream
2025-11-28T14:52:56.566700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789372022749678:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.566797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:56.566837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789372022749678:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:56.567243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:56.567408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789368558597959:2106] [node 4] ICS07 socket disconnect 269 reason# EndOfStream
2025-11-28T14:52:56.567439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368558597959:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.567513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:56.567491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789372022749644:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16787) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.567534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789372022749678:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.567548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789368558597959:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:56.567560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789372022749644:2108] [node 4] ICP30 unregister session Session# [5:7577789372022749678:2108] VirtualId# [5:7577789372022749669:0]
2025-11-28T14:52:56.567668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:56.567686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789372022749644:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16787) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.567700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16787) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.568169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789368558597937:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16787) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.568205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368558597959:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.568237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368558597937:2106] [node 4] ICP30 unregister session Session# [6:7577789368558597959:2106] VirtualId# [6:7577789368558597955:0]
2025-11-28T14:52:56.568268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:56.568288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789368558597937:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16787) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.568298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16787) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.577764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789368558597937:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:56.589606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:56.589868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789368558597975:2105] [node 5] ICS07 socket disconnect 242 reason# EndOfStream
2025-11-28T14:52:56.589910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368558597975:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.590011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:56.590061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789368558597975:2105] [node 5] ICS15 start handshake
2025-11-28T14:52:56.590811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789368558597933:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3770) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.590868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368558597975:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.590906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368558597933:2105] [node 5] ICP30 unregister session Session# [6:7577789368558597975:2105] VirtualId# [6:7577789368558597973:0]
2025-11-28T14:52:56.591027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:56.591066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789368558597933:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3770) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.591080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3770) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.601286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789368558597933:2105] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:56.653758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789369538487146:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:56.653764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789369538487147:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:56.669914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370804104439:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.669989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370804104440:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:56.687630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789370269311440:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:56.687629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789370269311439:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.707647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371564410599:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:56.707655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371564410598:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.707690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371564410600:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:56.727333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789369621695394:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.727372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789369621695396:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:56.727442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789369621695395:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:56.747455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368407093486:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:56.747489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368407093488:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:56.747563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368407093487:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:56.822461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368407093547:2106] [node 5] ICP01 ready to work
2025-11-28T14:52:56.822335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789369538487210:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:56.823873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370804104503:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:56.824776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371564410662:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:56.824923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789369538487211:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:56.824822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789370269311502:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:56.824526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370804104504:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:56.825083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789369621695457:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:56.825453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789370269311503:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:56.825463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789369538487212:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:56.825544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789371564410663:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:56.825534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789370804104505:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:56.825295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789369621695458:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:56.825506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789370269311504:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:56.826928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789368407093550:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.836434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093564:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.836791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789370269311518:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.836890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789370269311520:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.836953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370804104519:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65275
2025-11-28T14:52:56.836898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789370269311519:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
2025-11-28T14:52:56.837772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370804104520:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65275
2025-11-28T14:52:56.839083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370804104521:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65275
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.839304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370804104522:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65275
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.839374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789369538487226:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.839449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789369538487227:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
2025-11-28T14:52:56.839541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789369538487228:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
2025-11-28T14:52:56.839991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789369538487229:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.843278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789369621695467:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54062
2025-11-28T14:52:56.844489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789369621695468:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54062
2025-11-28T14:52:56.844663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789369621695469:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54062
2025-11-28T14:52:56.845358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789369621695467:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789369621695468:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368407093564:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370804104522:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487227:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789369621695395:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.845729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487226:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370804104521:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370804104519:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789370269311520:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487229:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.845803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487228:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789369621695469:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789370269311518:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:56.845873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789369538487230:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
::ffff:127.1.0.0
2025-11-28T14:52:56.845866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370804104520:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789370269311519:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789369621695396:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.845919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487221:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:56.845942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789370269311521:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
2025-11-28T14:52:56.845956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370804104511:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.845946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789369538487221:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.845643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368407093486:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.845981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789370804104511:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.846067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789370269311439:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.846120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370804104523:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65275
2025-11-28T14:52:56.846219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789370269311440:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.847281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487214:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.846616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789370269311510:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.846647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789370269311510:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.847317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789369538487214:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.847705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789370804104439:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.847717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370804104509:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.848020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789370269311510:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.847735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789370804104509:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.848344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789369538487221:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:56.848870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370804104511:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.849327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487216:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.849356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789370269311503:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.849366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789369538487216:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.849506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:56.849681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370804104509:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.850589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789369538487211:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.849922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789370269311503:2108] [node 5] ICP22 created new session: [3:7577789370269311527:2108]
2025-11-28T14:52:56.850646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093566:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
2025-11-28T14:52:56.850456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789370269311527:2108] [node 5] ICS09 handshake done sender: [3:7577789370269311510:2112] self: [3:7577789370269311509:0] peer: [5:7577789369621695477:0] socket: 249 qp: -1
2025-11-28T14:52:56.850498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789370269311527:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:56.850894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789369538487214:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.850777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:56.850914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.851096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789370804104503:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.851153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789369538487211:2110] [node 6] ICP22 created new session: [1:7577789369538487238:2110]
2025-11-28T14:52:56.851147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.851195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370804104503:2108] [node 5] ICP22 created new session: [2:7577789370804104530:2108]
2025-11-28T14:52:56.851244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789369538487238:2110] [node 6] ICS09 handshake done sender: [1:7577789369538487221:2114] self: [1:7577789369538487219:0] peer: [6:7577789368407093565:0] socket: 224 qp: -1
2025-11-28T14:52:56.851287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789369538487238:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:56.851377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:56.851245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370804104530:2108] [node 5] ICS09 handshake done sender: [2:7577789370804104511:2113] self: [2:7577789370804104510:0] peer: [5:7577789369621695476:0] socket: 240 qp: -1
2025-11-28T14:52:56.851283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370804104530:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:56.851313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:56.851423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368407093564:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:56.851371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789370804104440:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.851447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789369621695478:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54062
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.851455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.851538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370804104440:2049] [node 3] ICP22 created new session: [2:7577789370804104532:2049]
2025-11-28T14:52:56.851793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368407093486:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.851568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370804104532:2049] [node 3] ICS09 handshake done sender: [2:7577789370804104509:2112] self: [2:7577789370804104508:0] peer: [3:7577789370269311524:0] socket: 235 qp: -1
2025-11-28T14:52:56.851914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789369621695479:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54062
2025-11-28T14:52:56.851656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370804104532:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:56.851907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.851675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:56.851937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368407093486:2048] [node 1] ICP22 created new session: [6:7577789368407093567:2048]
2025-11-28T14:52:56.852011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368407093567:2048] [node 1] ICS09 handshake done sender: [6:7577789368407093564:2115] self: [6:7577789368407093565:0] peer: [1:7577789369538487219:0] socket: 237 qp: -1
2025-11-28T14:52:56.852294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789370269311526:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
2025-11-28T14:52:56.852030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368407093567:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.852935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789369621695469:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.852119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:56.853385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789369621695468:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:56.853370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487230:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.853402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789369538487147:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.853697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789369621695396:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.853618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.853706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789370269311521:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.853703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789369538487147:2049] [node 3] ICP22 created new session: [1:7577789369538487240:2049]
2025-11-28T14:52:56.853781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789369538487240:2049] [node 3] ICS09 handshake done sender: [1:7577789369538487214:2111] self: [1:7577789369538487213:0] peer: [3:7577789370269311523:0] socket: 222 qp: -1
No RDMA context found for GID: 2025-11-28T14:52:56.853926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.853907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789369538487240:2049] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:56.853992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789369621695396:2050] [node 3] ICP22 created new session: [5:7577789369621695480:2050]
2025-11-28T14:52:56.853973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:56.854102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789369621695480:2050] [node 3] ICS09 handshake done sender: [5:7577789369621695469:2116] self: [5:7577789369621695477:0] peer: [3:7577789370269311509:0] socket: 254 qp: -1
2025-11-28T14:52:56.854116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789370269311529:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
2025-11-28T14:52:56.854145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789369621695480:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:56.854237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.854802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371564410677:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53268
2025-11-28T14:52:56.854936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789370269311503:2108] [node 5] ICP18 handshake (actor [3:7577789370269311521:2118]) is too old
2025-11-28T14:52:56.855088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789370269311518:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:56.855151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789369621695395:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.855213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.855283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789369621695395:2049] [node 2] ICP22 created new session: [5:7577789369621695482:2049]
2025-11-28T14:52:56.855310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789369621695482:2049] [node 2] ICS09 handshake done sender: [5:7577789369621695468:2115] self: [5:7577789369621695476:0] peer: [2:7577789370804104510:0] socket: 253 qp: -1
2025-11-28T14:52:56.855333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789369621695482:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:56.855378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:56.855361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370804104523:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.855401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789370269311439:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.855504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:56.855585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789370269311439:2048] [node 1] ICP22 created new session: [3:7577789370269311532:2048]
2025-11-28T14:52:56.855631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789370269311532:2048] [node 1] ICS09 handshake done sender: [3:7577789370269311518:2115] self: [3:7577789370269311523:0] peer: [1:7577789369538487213:0] socket: 227 qp: -1
2025-11-28T14:52:56.855828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789370269311532:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.856060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:56.856309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789370269311519:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:56.856522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789370804104503:2108] [node 5] ICP18 handshake (actor [2:7577789370804104523:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.856722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789370804104534:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65275
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.856912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789369538487216:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:56.856923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371564410678:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53268
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.857969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093570:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
2025-11-28T14:52:56.858059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789369538487146:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.858129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.858167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789369538487146:2048] [node 2] ICP22 created new session: [1:7577789369538487244:2048]
2025-11-28T14:52:56.858195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789369538487244:2048] [node 2] ICS09 handshake done sender: [1:7577789369538487216:2112] self: [1:7577789369538487215:0] peer: [2:7577789370804104526:0] socket: 223 qp: -1
2025-11-28T14:52:56.858215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789369538487244:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:56.858272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.859190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371564410679:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53268
2025-11-28T14:52:56.859148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370804104519:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:56.859239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789370269311440:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.859367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.859389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789370269311440:2049] [node 2] ICP22 created new session: [3:7577789370269311534:2049]
2025-11-28T14:52:56.859495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789370269311534:2049] [node 2] ICS09 handshake done sender: [3:7577789370269311519:2116] self: [3:7577789370269311524:0] peer: [2:7577789370804104508:0] socket: 236 qp: -1
2025-11-28T14:52:56.859514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789370269311534:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:56.859531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:56.860130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371564410678:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.860356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789371564410599:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.860526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789370804104439:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.860610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:56.860653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370804104439:2048] [node 1] ICP22 created new session: [2:7577789370804104537:2048]
2025-11-28T14:52:56.860710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370804104537:2048] [node 1] ICS09 handshake done sender: [2:7577789370804104519:2116] self: [2:7577789370804104526:0] peer: [1:7577789369538487215:0] socket: 238 qp: -1
2025-11-28T14:52:56.860743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370804104537:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.860754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368407093570:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.860812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:56.860911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368407093487:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.860881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789370269311440:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: 2025-11-28T14:52:56.860907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370804104513:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:56.860939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789370804104513:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.861079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789369621695484:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54062
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:56.861449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093572:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
2025-11-28T14:52:56.861450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371564410681:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53268
2025-11-28T14:52:56.861624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789370804104439:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:56.862316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789370804104515:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.863380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371564410681:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.862349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789370804104515:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.863511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789369621695484:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.863584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789369621695394:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.863652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371564410679:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.863651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368407093572:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.863715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487218:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.863735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789371564410598:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.863742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789369538487218:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.863793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368407093547:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.863832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789370269311531:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
2025-11-28T14:52:56.863975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789369621695466:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.864054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371564410677:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.863994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789369621695466:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.864426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789371564410600:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.864514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789370269311531:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.864658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789370269311512:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.864673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789370269311512:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.864699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789369621695466:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:56.865251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789369538487222:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.865291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789369538487222:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.865265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370804104513:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:56.865313Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789369538487218:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.865901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789370269311512:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:56.866073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789370804104504:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.866292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789369538487222:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.02025-11-28T14:52:56.866237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789370804104515:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:56.866292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789369621695457:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.866194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.866292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789369538487210:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.866332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.866558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:56.866539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371564410685:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53268
2025-11-28T14:52:56.866408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370804104504:2107] [node 4] ICP22 created new session: [2:7577789370804104541:2107]
2025-11-28T14:52:56.866666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789369538487210:2109] [node 5] ICP22 created new session: [1:7577789369538487249:2109]
2025-11-28T14:52:56.866362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789369621695457:2109] [node 6] ICP22 created new session: [5:7577789369621695490:2109]
2025-11-28T14:52:56.866862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789369538487249:2109] [node 5] ICS09 handshake done sender: [1:7577789369538487218:2113] self: [1:7577789369538487217:0] peer: [5:7577789369621695487:0] socket: 262 qp: -1
2025-11-28T14:52:56.866627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370804104541:2107] [node 4] ICS09 handshake done sender: [2:7577789370804104513:2114] self: [2:7577789370804104512:0] peer: [4:7577789371564410680:0] socket: 242 qp: -1
2025-11-28T14:52:56.866884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789370269311502:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.866874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371564410686:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53268
2025-11-28T14:52:56.866699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370804104541:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:56.866897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789369538487249:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:56.866875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093576:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
::ffff:127.1.0.02025-11-28T14:52:56.866391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789369621695490:2109] [node 6] ICS09 handshake done sender: [5:7577789369621695466:2113] self: [5:7577789369621695465:0] peer: [6:7577789368407093573:0] socket: 230 qp: -1
2025-11-28T14:52:56.867257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.866553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789369621695490:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:56.866876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093574:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
2025-11-28T14:52:56.867328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:56.867308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.866874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.867426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789369621695489:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54062
2025-11-28T14:52:56.867346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789370269311502:2107] [node 4] ICP22 created new session: [3:7577789370269311539:2107]
2025-11-28T14:52:56.868124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093575:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
2025-11-28T14:52:56.868240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789369538487212:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.867430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789370269311539:2107] [node 4] ICS09 handshake done sender: [3:7577789370269311512:2113] self: [3:7577789370269311511:0] peer: [4:7577789371564410684:0] socket: 274 qp: -1
2025-11-28T14:52:56.868690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789370804104505:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.868623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.867506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789370269311539:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:56.868720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789369538487212:2107] [node 4] ICP22 created new session: [1:7577789369538487251:2107]
2025-11-28T14:52:56.868748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.867968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.870108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789370804104505:2110] [node 6] ICP22 created new session: [2:7577789370804104543:2110]
2025-11-28T14:52:56.870041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789369538487251:2107] [node 4] ICS09 handshake done sender: [1:7577789369538487222:2115] self: [1:7577789369538487220:0] peer: [4:7577789371564410683:0] socket: 229 qp: -1
2025-11-28T14:52:56.870075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789368407093547:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11987) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789368407093572:2118] held: no
2025-11-28T14:52:56.870069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789369538487251:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:56.870162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789370804104543:2110] [node 6] ICS09 handshake done sender: [2:7577789370804104515:2115] self: [2:7577789370804104514:0] peer: [6:7577789368407093571:0] socket: 260 qp: -1
2025-11-28T14:52:56.871644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368407093572:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.871658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371564410678:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:56.870168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.870198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789370804104543:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:56.870486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:56.872151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368407093575:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.872255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371564410677:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.872243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789369621695394:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789369621695484:2120] held: no
2025-11-28T14:52:56.872322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368407093570:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:56.872652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368407093547:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.872540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789371564410600:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789371564410677:2116] held: no
2025-11-28T14:52:56.872690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:56.872788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368407093547:2106] [node 5] ICP22 created new session: [6:7577789368407093577:2106]
2025-11-28T14:52:56.872889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368407093577:2106] [node 5] ICS09 handshake done sender: [6:7577789368407093572:2118] self: [6:7577789368407093573:0] peer: [5:7577789369621695465:0] socket: 258 qp: -1
2025-11-28T14:52:56.872925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368407093577:2106] [node 5] ICS10 traffic start
2025-11-28T14:52:56.872913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789371564410599:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789371564410678:2117] held: no
2025-11-28T14:52:56.872958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:56.873048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789369621695484:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:56.873130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789371564410600:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.873296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.873398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368407093487:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.873370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371564410600:2050] [node 3] ICP22 created new session: [4:7577789371564410688:2050]
2025-11-28T14:52:56.873343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789369621695394:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.873380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789371564410599:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.873420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:56.873410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.873506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:56.873490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371564410688:2050] [node 3] ICS09 handshake done sender: [4:7577789371564410677:2116] self: [4:7577789371564410684:0] peer: [3:7577789370269311511:0] socket: 226 qp: -1
2025-11-28T14:52:56.873506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789369621695394:2048] [node 1] ICP22 created new session: [5:7577789369621695492:2048]
2025-11-28T14:52:56.873554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368407093487:2049] [node 2] ICP22 created new session: [6:7577789368407093579:2049]
2025-11-28T14:52:56.873514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371564410599:2049] [node 2] ICP22 created new session: [4:7577789371564410689:2049]
2025-11-28T14:52:56.873605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368407093579:2049] [node 2] ICS09 handshake done sender: [6:7577789368407093570:2117] self: [6:7577789368407093571:0] peer: [2:7577789370804104514:0] socket: 277 qp: -1
2025-11-28T14:52:56.873516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371564410688:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:56.873639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368407093579:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:56.873542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371564410689:2049] [node 2] ICS09 handshake done sender: [4:7577789371564410678:2117] self: [4:7577789371564410680:0] peer: [2:7577789370804104512:0] socket: 244 qp: -1
2025-11-28T14:52:56.873665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:56.873614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789369621695492:2048] [node 1] ICS09 handshake done sender: [5:7577789369621695484:2120] self: [5:7577789369621695487:0] peer: [1:7577789369538487217:0] socket: 225 qp: -1
2025-11-28T14:52:56.873603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:56.873609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371564410689:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:56.873650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789369621695492:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.873628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:56.873687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.874591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371564410687:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53268
2025-11-28T14:52:56.875647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789368407093487:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:56.877294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789371564410598:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789371564410679:2118] held: no
2025-11-28T14:52:56.877305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368407093488:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.877839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789370269311517:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.877876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789370269311517:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.878059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371564410679:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:56.878649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789371564410598:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.878751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789370269311517:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:56.878749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:56.878833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371564410598:2048] [node 1] ICP22 created new session: [4:7577789371564410695:2048]
2025-11-28T14:52:56.878865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371564410695:2048] [node 1] ICS09 handshake done sender: [4:7577789371564410679:2118] self: [4:7577789371564410683:0] peer: [1:7577789369538487220:0] socket: 278 qp: -1
2025-11-28T14:52:56.878959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789370269311504:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.878892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371564410695:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:56.879230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.878911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.879338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789370269311504:2109] [node 6] ICP22 created new session: [3:7577789370269311542:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.879540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789370269311542:2109] [node 6] ICS09 handshake done sender: [3:7577789370269311517:2114] self: [3:7577789370269311513:0] peer: [6:7577789368407093582:0] socket: 281 qp: -1
2025-11-28T14:52:56.879595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789371564410694:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53268
2025-11-28T14:52:56.879587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789370269311542:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:56.879664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.879795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789369621695494:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54062
2025-11-28T14:52:56.879922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093583:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.881247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093584:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
2025-11-28T14:52:56.881350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789368407093488:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789368407093575:2120] held: no
2025-11-28T14:52:56.881851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368407093575:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:56.881961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368407093488:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.881938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371564410694:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.882010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:56.882002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789369621695494:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.882053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368407093488:2050] [node 3] ICP22 created new session: [6:7577789368407093585:2050]
2025-11-28T14:52:56.882109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368407093585:2050] [node 3] ICS09 handshake done sender: [6:7577789368407093575:2120] self: [6:7577789368407093582:0] peer: [3:7577789370269311513:0] socket: 284 qp: -1
2025-11-28T14:52:56.882152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368407093585:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:56.882191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:56.882194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789369621695458:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.882538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371564410671:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.882564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789371564410671:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.882864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789368407093584:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:56.883079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789368407093550:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:56.883233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371564410671:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:56.883297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789371564410676:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.883316Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789371564410676:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:56.883740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789369621695496:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54062
2025-11-28T14:52:56.883772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789371564410662:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.883854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:56.883889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371564410662:2108] [node 5] ICP22 created new session: [4:7577789371564410699:2108]
2025-11-28T14:52:56.883923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789369621695494:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:56.883940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371564410699:2108] [node 5] ICS09 handshake done sender: [4:7577789371564410671:2114] self: [4:7577789371564410669:0] peer: [5:7577789369621695495:0] socket: 234 qp: -1
2025-11-28T14:52:56.883971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371564410699:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:56.883997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:56.884200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789369621695458:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.884305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.884346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789369621695458:2108] [node 4] ICP22 created new session: [5:7577789369621695497:2108]
2025-11-28T14:52:56.884404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789369621695497:2108] [node 4] ICS09 handshake done sender: [5:7577789369621695494:2122] self: [5:7577789369621695495:0] peer: [4:7577789371564410669:0] socket: 245 qp: -1
2025-11-28T14:52:56.884432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789369621695497:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:56.885150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.885393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789371564410676:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:56.885854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789368407093588:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16205
2025-11-28T14:52:56.886038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789371564410663:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:56.886091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:56.886142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789371564410663:2110] [node 6] ICP22 created new session: [4:7577789371564410702:2110]
2025-11-28T14:52:56.886194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789371564410702:2110] [node 6] ICS09 handshake done sender: [4:7577789371564410676:2115] self: [4:7577789371564410675:0] peer: [6:7577789368407093587:0] socket: 241 qp: -1
2025-11-28T14:52:56.886207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789368407093584:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:56.886270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789371564410702:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:56.886314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:56.886379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789368407093550:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:56.886415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:56.886451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789368407093550:2107] [node 4] ICP22 created new session: [6:7577789368407093589:2107]
2025-11-28T14:52:56.886503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789368407093589:2107] [node 4] ICS09 handshake done sender: [6:7577789368407093584:2123] self: [6:7577789368407093587:0] peer: [4:7577789371564410675:0] socket: 248 qp: -1
2025-11-28T14:52:56.886526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789368407093589:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:56.886547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:56.904491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:56.904681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789370804104537:2048] [node 1] ICS07 socket disconnect 238 reason# EndOfStream
2025-11-28T14:52:56.904708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789370804104537:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.904799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.904842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789370804104537:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:56.905551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:56.905768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789370269311532:2048] [node 1] ICS07 socket disconnect 227 reason# EndOfStream
2025-11-28T14:52:56.905806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789370269311532:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.905897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.905963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789370269311532:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:56.906681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789370269311439:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.906745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789370269311532:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.906792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789370269311439:2048] [node 1] ICP30 unregister session Session# [3:7577789370269311532:2048] VirtualId# [3:7577789370269311523:0]
2025-11-28T14:52:56.906865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789370804104439:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:56.906905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.906917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789370804104537:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:56.906951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789370269311439:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.906966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.907925Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789370804104439:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.908526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789370804104537:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.908584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789370804104439:2048] [node 1] ICP30 unregister session Session# [2:7577789370804104537:2048] VirtualId# [2:7577789370804104526:0]
2025-11-28T14:52:56.908692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.908718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789370804104439:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.908727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.908821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:56.909103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789371564410695:2048] [node 1] ICS01 socket: 278 reason# EndOfStream
2025-11-28T14:52:56.909164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789371564410598:2048] [node 1] ICP30 unregister session Session# [4:7577789371564410695:2048] VirtualId# [4:7577789371564410683:0]
2025-11-28T14:52:56.909194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789371564410695:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.909260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.909305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.909559Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:56.909767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789369621695492:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-11-28T14:52:56.909822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789369621695394:2048] [node 1] ICP30 unregister session Session# [5:7577789369621695492:2048] VirtualId# [5:7577789369621695487:0]
2025-11-28T14:52:56.909857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789369621695492:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.909920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.909968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.910143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:56.910374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789368407093567:2048] [node 1] ICS07 socket disconnect 237 reason# EndOfStream
2025-11-28T14:52:56.910417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368407093567:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.910473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:56.910516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789368407093567:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:56.911200Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789368407093486:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.911320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368407093567:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.911370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368407093486:2048] [node 1] ICP30 unregister session Session# [6:7577789368407093567:2048] VirtualId# [6:7577789368407093565:0]
2025-11-28T14:52:56.911480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:56.911517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789368407093486:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.911533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.916610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789370269311439:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:56.918657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789370804104439:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:56.921734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789368407093486:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:56.967547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:56.967764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789370269311534:2049] [node 2] ICS07 socket disconnect 236 reason# EndOfStream
2025-11-28T14:52:56.967791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789370269311534:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.967860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:56.967903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789370269311534:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:56.969696Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:56.969866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789371564410689:2049] [node 2] ICS01 socket: 244 reason# EndOfStream
2025-11-28T14:52:56.969853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789370269311440:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:56.969926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789371564410599:2049] [node 2] ICP30 unregister session Session# [4:7577789371564410689:2049] VirtualId# [4:7577789371564410680:0]
2025-11-28T14:52:56.969917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789370269311534:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:56.969966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789371564410689:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.970041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:56.970116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:56.970500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789370269311440:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.970537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789370269311534:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.970585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789370269311440:2049] [node 2] ICP30 unregister session Session# [3:7577789370269311534:2049] VirtualId# [3:7577789370269311524:0]
2025-11-28T14:52:56.970659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:56.970685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789370269311440:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.970694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.970864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:56.971066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789369621695482:2049] [node 2] ICS07 socket disconnect 253 reason# EndOfStream
2025-11-28T14:52:56.971098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789369621695482:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.971168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:56.971250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789369621695482:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:56.971628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:56.971850Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789369621695395:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.971878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789368407093579:2049] [node 2] ICS07 socket disconnect 277 reason# EndOfStream
2025-11-28T14:52:56.971947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789369621695482:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.971958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368407093579:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:56.971990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789369621695395:2049] [node 2] ICP30 unregister session Session# [5:7577789369621695482:2049] VirtualId# [5:7577789369621695476:0]
2025-11-28T14:52:56.972030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:56.972077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789368407093579:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:56.972196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:56.972235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789369621695395:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.972254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.972976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789368407093487:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:56.973026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368407093579:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:56.973100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368407093487:2049] [node 2] ICP30 unregister session Session# [6:7577789368407093579:2049] VirtualId# [6:7577789368407093571:0]
2025-11-28T14:52:56.973193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:56.973229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789368407093487:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.973246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64510) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:56.982198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789369621695395:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:56.983200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789368407093487:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:57.016385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:57.016634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789371564410688:2050] [node 3] ICS01 socket: 226 reason# EndOfStream
2025-11-28T14:52:57.016685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789371564410600:2050] [node 3] ICP30 unregister session Session# [4:7577789371564410688:2050] VirtualId# [4:7577789371564410684:0]
2025-11-28T14:52:57.016737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789371564410688:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.016832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:57.016880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:57.017294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:57.017497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789369621695480:2050] [node 3] ICS07 socket disconnect 254 reason# EndOfStream
2025-11-28T14:52:57.017532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789369621695480:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.017601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:57.017657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789369621695480:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:57.018132Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:57.018287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368407093585:2050] [node 3] ICS01 socket: 284 reason# EndOfStream
2025-11-28T14:52:57.018326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368407093488:2050] [node 3] ICP30 unregister session Session# [6:7577789368407093585:2050] VirtualId# [6:7577789368407093582:0]
2025-11-28T14:52:57.018346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368407093585:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.018330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789369621695396:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14427) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:57.018402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:57.018421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789369621695480:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:57.018449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:57.018479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789369621695396:2050] [node 3] ICP30 unregister session Session# [5:7577789369621695480:2050] VirtualId# [5:7577789369621695477:0]
2025-11-28T14:52:57.018586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:57.018618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789369621695396:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.018633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.028277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789369621695396:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:57.073179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:57.073457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789369621695497:2108] [node 4] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-28T14:52:57.073495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789369621695497:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.073585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:57.073633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789369621695497:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:57.074244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:57.074240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789369621695458:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5328) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:57.074279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789369621695497:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:57.074353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789369621695458:2108] [node 4] ICP30 unregister session Session# [5:7577789369621695497:2108] VirtualId# [5:7577789369621695495:0]
2025-11-28T14:52:57.074440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:57.074460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789369621695458:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5328) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.074469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5328) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.074485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368407093589:2107] [node 4] ICS01 socket: 248 reason# EndOfStream
2025-11-28T14:52:57.074537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368407093550:2107] [node 4] ICP30 unregister session Session# [6:7577789368407093589:2107] VirtualId# [6:7577789368407093587:0]
2025-11-28T14:52:57.074569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368407093589:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.074638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:57.074706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:57.084612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789369621695458:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:57.132408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:57.132575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789368407093577:2106] [node 5] ICS01 socket: 258 reason# EndOfStream
2025-11-28T14:52:57.132609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789368407093547:2106] [node 5] ICP30 unregister session Session# [6:7577789368407093577:2106] VirtualId# [6:7577789368407093573:0]
2025-11-28T14:52:57.132636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789368407093577:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.132701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:57.132735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:57.219633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789375330731856:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:57.219633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789375330731857:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:57.236479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789374804578095:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.236722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789374804578096:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:57.253511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789373990488773:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.253741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789373990488774:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:57.270032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789373245960962:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.270063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789373245960963:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:57.270083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789373245960964:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:57.286833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789375023027185:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.286890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789375023027186:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:57.287016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789375023027187:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:57.303594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372611782690:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.303644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372611782692:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:57.303761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372611782691:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:57.374259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789373990488838:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:57.374487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789375330731938:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:57.374520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789373245961026:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:57.374607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789375330731937:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:57.374570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789373245961027:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:57.375426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789373990488839:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:57.375725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789375023027255:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:57.375914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789373990488840:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:57.375952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789374804578185:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:57.375987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789374804578186:2135] [node 5] ICP01 ready to work
2025-11-28T14:52:57.377193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789375330731939:2118] [node 6] ICP01 ready to work
2025-11-28T14:52:57.380019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789374804578187:2125] [node 6] ICP01 ready to work
2025-11-28T14:52:57.392822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372611782753:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:57.393824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372611782754:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:57.394248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789374804578096:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26221508ns) {Configure:104(26216012ns) {ProcessConfigured:135(188606ns) {StartInitialHandshake:157(186756ns) {DropHandshakes:470(2149ns) {DropIncomingHandshake:442(393ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(2304ns) PrepareNewSessionHandshake:477(775ns) {SwitchToState:179(309ns)}}}}}
2025-11-28T14:52:57.394628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789373245960962:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19808517ns) {Configure:104(19804170ns) {ProcessConfigured:135(38391ns) {StartInitialHandshake:157(34748ns) {DropHandshakes:470(1057ns) {DropIncomingHandshake:442(374ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(637ns) PrepareNewSessionHandshake:477(664ns) {SwitchToState:179(307ns)}}}}}
2025-11-28T14:52:57.394840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789375330731856:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19995408ns) {Configure:104(19991512ns) {ProcessConfigured:135(50643ns) {StartInitialHandshake:157(48814ns) {DropHandshakes:470(1402ns) {DropIncomingHandshake:442(359ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(646ns) PrepareNewSessionHandshake:477(880ns) {SwitchToState:179(346ns)}}}}}
2025-11-28T14:52:57.395255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789375330731857:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18836664ns) {Configure:104(18830535ns) {ProcessConfigured:135(162073ns) {StartInitialHandshake:157(160056ns) {DropHandshakes:470(1833ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(549ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:477(873ns) {SwitchToState:179(346ns)}}}}}
2025-11-28T14:52:57.395612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789373990488773:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19195179ns) {Configure:104(19188025ns) {ProcessConfigured:135(206252ns) {StartInitialHandshake:157(204614ns) {DropHandshakes:470(1159ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(494ns) PrepareNewSessionHandshake:477(709ns) {SwitchToState:179(313ns)}}}}}
2025-11-28T14:52:57.395610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789373245960963:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19368644ns) {Configure:104(19365187ns) {ProcessConfigured:135(176409ns) {StartInitialHandshake:157(174589ns) {DropHandshakes:470(961ns) {DropIncomingHandshake:442(309ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(884ns) {SwitchToState:179(361ns)}}}}}
2025-11-28T14:52:57.396739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789375023027187:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21446975ns) {Configure:104(21444656ns) {ProcessConfigured:135(471013ns) {StartInitialHandshake:157(469631ns) {DropHandshakes:470(6646ns) {DropIncomingHandshake:442(218ns) DropOutgoingHandshake:457(467ns)} GenerateSessionVirtualId:988(544ns) PrepareNewSessionHandshake:477(656ns) {SwitchToState:179(301ns)}}}}}
2025-11-28T14:52:57.396963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789373990488774:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16494222ns) {Configure:104(16490179ns) {ProcessConfigured:135(417896ns) {StartInitialHandshake:157(416026ns) {DropHandshakes:470(5666ns) {DropIncomingHandshake:442(491ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(1206ns) {SwitchToState:179(645ns)}}}}}
2025-11-28T14:52:57.397021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789375023027185:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21683301ns) {Configure:104(21679617ns) {ProcessConfigured:135(437860ns) {StartInitialHandshake:157(435516ns) {DropHandshakes:470(1138ns) {DropIncomingHandshake:442(305ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(524ns) PrepareNewSessionHandshake:477(1641ns) {SwitchToState:179(522ns)}}}}}
2025-11-28T14:52:57.397103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789375023027262:2110] [node 6] ICP01 ready to work
2025-11-28T14:52:57.397171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789375023027186:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16677241ns) {Configure:104(16675283ns) {ProcessConfigured:135(91265ns) {StartInitialHandshake:157(90353ns) {DropHandshakes:470(1155ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(137ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(513ns) {SwitchToState:179(119ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.400736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789373245961040:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55620
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.401709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789373990488849:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26993
2025-11-28T14:52:57.401700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789375330731954:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41305
2025-11-28T14:52:57.402923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789373990488853:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26993
2025-11-28T14:52:57.402422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789375330731953:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41305
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.406497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789373990488849:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.407086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789374804578199:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10851
2025-11-28T14:52:57.407073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789373245961042:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55620
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.407812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789375330731955:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41305
2025-11-28T14:52:57.406791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789373990488774:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.407983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789373245961042:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.408027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789374804578200:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10851
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.409840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789374804578189:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.408201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731954:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.409873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789374804578189:2137] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.409967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789373245960962:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.408604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731953:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.410142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789374804578199:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.411228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789373990488853:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.411213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789375023027268:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2158
2025-11-28T14:52:57.411277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789374804578203:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10851
2025-11-28T14:52:57.411241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731945:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.412389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789373245961040:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.411274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789375330731945:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.411579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789373990488773:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.411215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789375023027263:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2158
2025-11-28T14:52:57.413289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789374804578200:2142] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.413675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789374804578203:2144] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.413484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789373245960963:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.413715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731955:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.412891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789375023027263:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.413183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789375023027255:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.414145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789374804578189:2137] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.414106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731943:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.414805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789373245961033:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.413280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789375023027268:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.415497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789373990488858:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26993
2025-11-28T14:52:57.414129Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789375330731943:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.414807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789374804578204:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10851
2025-11-28T14:52:57.414832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789373245961033:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.414229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789375023027187:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.415669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789373990488846:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.414319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789375330731956:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41305
2025-11-28T14:52:57.415068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789374804578095:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.415003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789375330731945:2130] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:57.415705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789373990488846:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.416419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789373990488859:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26993
2025-11-28T14:52:57.415494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789374804578205:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10851
2025-11-28T14:52:57.415671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789375330731957:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41305
2025-11-28T14:52:57.419610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789373245961033:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:57.420117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789373990488858:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.415665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789374804578193:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.420793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789373245961045:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55620
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.419249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789375330731943:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:57.421376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789373990488859:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.415682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789374804578193:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.420072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789375330731937:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.421722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789373245961027:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.421989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789375023027274:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2158
2025-11-28T14:52:57.415712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789374804578096:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.421991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789373990488860:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26993
2025-11-28T14:52:57.420236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731941:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.421910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:57.418768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.420257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789375330731941:2128] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.420795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:57.421961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789373245961027:2108] [node 5] ICP22 created new session: [4:7577789373245961048:2108]
2025-11-28T14:52:57.423554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789375023027276:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2158
2025-11-28T14:52:57.422167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789373990488774:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25386) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789373990488849:2116] held: no
2025-11-28T14:52:57.419384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789374804578096:2049] [node 3] ICP22 created new session: [2:7577789374804578210:2049]
2025-11-28T14:52:57.421097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789375330731857:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.424277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789375023027263:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:57.423164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789373245961048:2108] [node 5] ICS09 handshake done sender: [4:7577789373245961033:2113] self: [4:7577789373245961032:0] peer: [5:7577789375023027272:0] socket: 231 qp: -1
2025-11-28T14:52:57.419958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789374804578210:2049] [node 3] ICS09 handshake done sender: [2:7577789374804578189:2137] self: [2:7577789374804578188:0] peer: [3:7577789373990488856:0] socket: 228 qp: -1
2025-11-28T14:52:57.425486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789373990488846:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:57.423323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789373245961042:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.421196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789375330731937:2107] [node 4] ICP22 created new session: [1:7577789375330731963:2107]
2025-11-28T14:52:57.424297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789375023027275:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2158
2025-11-28T14:52:57.420145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789374804578210:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:57.421346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:57.425388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789375023027255:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.423340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789373245961048:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:57.420243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789374804578193:2139] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:57.426993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789373990488861:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26993
2025-11-28T14:52:57.421745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789375330731857:2049] [node 3] ICP22 created new session: [1:7577789375330731964:2049]
2025-11-28T14:52:57.425749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:57.423498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:57.420332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:57.422012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789375330731964:2049] [node 3] ICS09 handshake done sender: [1:7577789375330731943:2129] self: [1:7577789375330731942:0] peer: [3:7577789373990488857:0] socket: 238 qp: -1
2025-11-28T14:52:57.427356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789373990488849:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:57.426023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789375023027255:2108] [node 4] ICP22 created new session: [5:7577789375023027277:2108]
2025-11-28T14:52:57.420847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789374804578204:2145] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.425379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789373245960962:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.427240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789375023027277:2108] [node 4] ICS09 handshake done sender: [5:7577789375023027263:2120] self: [5:7577789375023027272:0] peer: [4:7577789373245961032:0] socket: 245 qp: -1
2025-11-28T14:52:57.428593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789373990488839:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.422134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789375330731963:2107] [node 4] ICS09 handshake done sender: [1:7577789375330731945:2130] self: [1:7577789375330731944:0] peer: [4:7577789373245961043:0] socket: 242 qp: -1
2025-11-28T14:52:57.425409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:57.422176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789374804578205:2146] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.428769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789373990488774:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.427393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789375023027277:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.422701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789374804578185:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.425684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789373245960962:2048] [node 1] ICP22 created new session: [4:7577789373245961050:2048]
2025-11-28T14:52:57.428771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:57.422180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789375330731964:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:57.427402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789375023027268:2123] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:57.428820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789373990488839:2108] [node 5] ICP22 created new session: [3:7577789373990488864:2108]
2025-11-28T14:52:57.422838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:57.425778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789373245961050:2048] [node 1] ICS09 handshake done sender: [4:7577789373245961042:2117] self: [4:7577789373245961043:0] peer: [1:7577789375330731944:0] socket: 246 qp: -1
2025-11-28T14:52:57.425876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789373245961050:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.422685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789375330731963:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:57.427933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:57.426906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:57.423181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789374804578185:2107] [node 4] ICP22 created new session: [2:7577789374804578214:2107]
2025-11-28T14:52:57.428972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:57.422689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: 2025-11-28T14:52:57.423473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789374804578214:2107] [node 4] ICS09 handshake done sender: [2:7577789374804578193:2139] self: [2:7577789374804578190:0] peer: [4:7577789373245961044:0] socket: 232 qp: -1
::ffff:127.1.0.0
2025-11-28T14:52:57.432315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782769:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
2025-11-28T14:52:57.428650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789375023027187:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.422725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:57.424270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731957:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.429317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:57.431125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789373245961047:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55620
2025-11-28T14:52:57.429318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789373990488774:2049] [node 2] ICP22 created new session: [3:7577789373990488865:2049]
2025-11-28T14:52:57.423719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789374804578214:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:57.429814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789375023027187:2050] [node 3] ICP22 created new session: [5:7577789375023027280:2050]
2025-11-28T14:52:57.425245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731956:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.432407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782768:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
2025-11-28T14:52:57.429543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789373990488864:2108] [node 5] ICS09 handshake done sender: [3:7577789373990488846:2114] self: [3:7577789373990488845:0] peer: [5:7577789375023027273:0] socket: 249 qp: -1
2025-11-28T14:52:57.430075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789375023027280:2050] [node 3] ICS09 handshake done sender: [5:7577789375023027268:2123] self: [5:7577789375023027273:0] peer: [3:7577789373990488845:0] socket: 255 qp: -1
2025-11-28T14:52:57.431201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789375330731941:2128] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:52:57.429880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789373990488864:2108] [node 5] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:52:57.423837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:57.433272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789373245960963:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25386) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789373245961040:2115] held: no
2025-11-28T14:52:57.433227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789375330731856:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.433393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782770:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
2025-11-28T14:52:57.430044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789373990488865:2049] [node 2] ICS09 handshake done sender: [3:7577789373990488849:2116] self: [3:7577789373990488856:0] peer: [2:7577789374804578188:0] socket: 235 qp: -1
2025-11-28T14:52:57.430107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789375023027280:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:57.433291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:57.430079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789373990488865:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:57.433428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789375330731856:2048] [node 2] ICP22 created new session: [1:7577789375330731972:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.433472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789374804578219:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10851
2025-11-28T14:52:57.430155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789375023027275:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.430109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:57.430253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:57.430523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:57.433487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789375330731972:2048] [node 2] ICS09 handshake done sender: [1:7577789375330731941:2128] self: [1:7577789375330731940:0] peer: [2:7577789374804578209:0] socket: 236 qp: -1
2025-11-28T14:52:57.435352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789372611782768:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.433269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789373990488853:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:57.434502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789375023027186:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.433562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789375330731972:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:57.435846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372611782754:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.433309Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789373990488773:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22945) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789373990488853:2117] held: no
2025-11-28T14:52:57.436083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789373245960962:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:57.434794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:57.435076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789375023027282:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2158
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.437047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789373990488839:2108] [node 5] ICP18 handshake (actor [3:7577789373990488859:2120]) is too old
2025-11-28T14:52:57.437395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789373990488773:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.437437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:57.437405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782771:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
2025-11-28T14:52:57.437374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789375023027267:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.438603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789373245961040:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:57.437623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789373990488773:2048] [node 1] ICP22 created new session: [3:7577789373990488870:2048]
2025-11-28T14:52:57.438812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782772:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
2025-11-28T14:52:57.438862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789373990488870:2048] [node 1] ICS09 handshake done sender: [3:7577789373990488853:2117] self: [3:7577789373990488857:0] peer: [1:7577789375330731942:0] socket: 247 qp: -1
2025-11-28T14:52:57.437402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789375023027267:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.438902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789373990488870:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:57.439002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:57.439024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789374804578096:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17379470ns) {HandleHandshakeStatus:329(17377256ns) {DropIncomingHandshake:442(707857ns) DropOutgoingHandshake:457(550ns) SwitchToState:179(254ns) ProcessPendingSessionEvents:495(10763032ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9855658ns max# 1154401ns}}}
2025-11-28T14:52:57.439273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789375023027267:2122] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:57.439721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789374804578200:2142] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:57.440013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789374804578195:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.440033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789374804578195:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.440999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789374804578095:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22945) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789374804578200:2142] held: no
2025-11-28T14:52:57.441552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789375023027262:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.441590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: 2025-11-28T14:52:57.441630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789375023027262:2110] [node 6] ICP22 created new session: [5:7577789375023027285:2110]
::ffff:127.1.0.0
2025-11-28T14:52:57.441627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789372611782769:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.441704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789375023027285:2110] [node 6] ICS09 handshake done sender: [5:7577789375023027267:2122] self: [5:7577789375023027265:0] peer: [6:7577789372611782773:0] socket: 229 qp: -1
2025-11-28T14:52:57.441680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789373245960963:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.441738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789375023027285:2110] [node 6] ICS10 traffic start
2025-11-28T14:52:57.441735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:57.441768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:57.441754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782774:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
2025-11-28T14:52:57.441759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372611782691:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.441794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789373245960963:2049] [node 2] ICP22 created new session: [4:7577789373245961055:2049]
2025-11-28T14:52:57.441905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789373245961055:2049] [node 2] ICS09 handshake done sender: [4:7577789373245961040:2115] self: [4:7577789373245961044:0] peer: [2:7577789374804578190:0] socket: 234 qp: -1
2025-11-28T14:52:57.441960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789373245961055:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:57.442025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:57.442437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789372611782768:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:57.442097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789374804578095:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.442140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:57.442381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789374804578202:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.443284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372611782754:2109] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.442397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789374804578202:2143] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.443335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:57.443568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372611782754:2109] [node 5] ICP22 created new session: [6:7577789372611782776:2109]
2025-11-28T14:52:57.442419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789374804578095:2048] [node 1] ICP22 created new session: [2:7577789374804578221:2048]
2025-11-28T14:52:57.443063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789374804578221:2048] [node 1] ICS09 handshake done sender: [2:7577789374804578200:2142] self: [2:7577789374804578209:0] peer: [1:7577789375330731940:0] socket: 244 qp: -1
2025-11-28T14:52:57.443822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789372611782776:2109] [node 5] ICS09 handshake done sender: [6:7577789372611782768:2115] self: [6:7577789372611782773:0] peer: [5:7577789375023027265:0] socket: 239 qp: -1
2025-11-28T14:52:57.443276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789374804578221:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:57.444101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789372611782776:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:57.443527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789374804578195:2140] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:57.444134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:57.443583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:57.444132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789374804578186:2135] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.444179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:57.444396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789374804578186:2135] [node 5] ICP22 created new session: [2:7577789374804578224:2135]
2025-11-28T14:52:57.445097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789374804578224:2135] [node 5] ICS09 handshake done sender: [2:7577789374804578195:2140] self: [2:7577789374804578194:0] peer: [5:7577789375023027283:0] socket: 279 qp: -1
2025-11-28T14:52:57.445090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789373990488872:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26993
2025-11-28T14:52:57.445125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789374804578224:2135] [node 5] ICS10 traffic start
2025-11-28T14:52:57.445204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:57.445183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789372611782772:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.445221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789372611782770:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.445700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372611782690:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.445667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789374804578202:2143] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:57.445673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789373245961054:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55620
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.445730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372611782692:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.447347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782779:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.447430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789375023027186:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25386) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789375023027275:2125] held: no
2025-11-28T14:52:57.447590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731947:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.447438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789375023027282:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.447630Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789375330731947:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.447747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789372611782771:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.447630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789375023027287:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2158
2025-11-28T14:52:57.447653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789373990488848:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.447971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789375023027185:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.447898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789374804578187:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.448526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789373245961057:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55620
2025-11-28T14:52:57.447862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789372611782769:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:57.447693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789373990488848:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.448532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789373245961054:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.449465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:57.448798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789373245960964:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.449031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789373990488872:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.449815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789374804578187:2125] [node 6] ICP22 created new session: [2:7577789374804578226:2125]
2025-11-28T14:52:57.449520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789375330731951:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.449852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372611782691:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.449711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789373990488848:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:57.449552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789375330731951:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.450400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:57.450285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789373990488855:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.450271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789374804578226:2125] [node 6] ICS09 handshake done sender: [2:7577789374804578202:2143] self: [2:7577789374804578201:0] peer: [6:7577789372611782775:0] socket: 241 qp: -1
2025-11-28T14:52:57.450495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789374804578226:2125] [node 6] ICS10 traffic start
2025-11-28T14:52:57.451413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789373245961057:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.450319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789373990488840:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.450519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789375330731947:2131] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:57.452071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789375023027275:2125] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:57.451610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789375330731951:2132] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:57.450531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:57.452038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789375330731939:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.451540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372611782691:2049] [node 2] ICP22 created new session: [6:7577789372611782782:2049]
2025-11-28T14:52:57.452193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:57.452471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789375023027186:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.451813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789372611782782:2049] [node 2] ICS09 handshake done sender: [6:7577789372611782769:2116] self: [6:7577789372611782775:0] peer: [2:7577789374804578201:0] socket: 248 qp: -1
2025-11-28T14:52:57.452612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:57.452060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789372611782782:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:57.452226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789375330731939:2118] [node 6] ICP22 created new session: [1:7577789375330731976:2118]
2025-11-28T14:52:57.450324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789373990488855:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.452079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372611782753:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.453780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789375330731976:2118] [node 6] ICS09 handshake done sender: [1:7577789375330731947:2131] self: [1:7577789375330731946:0] peer: [6:7577789372611782781:0] socket: 233 qp: -1
2025-11-28T14:52:57.452117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:57.453475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789375023027186:2049] [node 2] ICP22 created new session: [5:7577789375023027290:2049]
2025-11-28T14:52:57.451566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:57.453990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789375023027290:2049] [node 2] ICS09 handshake done sender: [5:7577789375023027275:2125] self: [5:7577789375023027283:0] peer: [2:7577789374804578194:0] socket: 281 qp: -1
2025-11-28T14:52:57.451632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789373990488840:2109] [node 6] ICP22 created new session: [3:7577789373990488875:2109]
2025-11-28T14:52:57.454017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789375023027290:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:57.453809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789375330731938:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.451684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789373990488875:2109] [node 6] ICS09 handshake done sender: [3:7577789373990488848:2115] self: [3:7577789373990488847:0] peer: [6:7577789372611782780:0] socket: 240 qp: -1
2025-11-28T14:52:57.454032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789375023027289:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2158
2025-11-28T14:52:57.454205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:57.452052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789373990488875:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:57.453990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789375330731976:2118] [node 6] ICS10 traffic start
2025-11-28T14:52:57.452202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:57.454018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.454036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:57.453804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789373990488855:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:57.454103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789373990488838:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.454449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:57.454212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789375330731938:2108] [node 5] ICP22 created new session: [1:7577789375330731978:2108]
2025-11-28T14:52:57.454688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789373990488838:2107] [node 4] ICP22 created new session: [3:7577789373990488878:2107]
2025-11-28T14:52:57.454479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789375330731978:2108] [node 5] ICS09 handshake done sender: [1:7577789375330731951:2132] self: [1:7577789375330731948:0] peer: [5:7577789375023027288:0] socket: 285 qp: -1
2025-11-28T14:52:57.454925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789373245961060:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55620
2025-11-28T14:52:57.454512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789375330731978:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:57.455084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789373245961035:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.457270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789372611782691:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:57.454574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:57.454900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789373990488878:2107] [node 4] ICS09 handshake done sender: [3:7577789373990488855:2118] self: [3:7577789373990488854:0] peer: [4:7577789373245961058:0] socket: 289 qp: -1
2025-11-28T14:52:57.455100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789373245961035:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.454933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789373990488878:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:57.454969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:57.457437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789373245960964:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789373245961054:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.458965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789373245961035:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.459221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782783:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
2025-11-28T14:52:57.459974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782784:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
2025-11-28T14:52:57.460029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789373245961026:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.460087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:57.460085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789372611782692:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789372611782772:2119] held: no
2025-11-28T14:52:57.460108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789373245961054:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:57.460088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789372611782690:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22945) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789372611782770:2117] held: no
2025-11-28T14:52:57.460150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789375023027282:2127] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:57.460122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789373245961026:2109] [node 6] ICP22 created new session: [4:7577789373245961062:2109]
2025-11-28T14:52:57.460158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789373245961062:2109] [node 6] ICS09 handshake done sender: [4:7577789373245961035:2114] self: [4:7577789373245961034:0] peer: [6:7577789372611782785:0] socket: 263 qp: -1
2025-11-28T14:52:57.460192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789373245961062:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:57.460252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789375023027185:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22945) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789375023027282:2127] held: no
2025-11-28T14:52:57.460215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:57.460732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789373245960964:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.460779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:57.461038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789373245960964:2050] [node 3] ICP22 created new session: [4:7577789373245961064:2050]
2025-11-28T14:52:57.461167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789373245961064:2050] [node 3] ICS09 handshake done sender: [4:7577789373245961054:2120] self: [4:7577789373245961058:0] peer: [3:7577789373990488854:0] socket: 291 qp: -1
2025-11-28T14:52:57.461239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789373245961064:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:57.461268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:57.461582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789375023027185:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.461646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:57.461684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789375023027185:2048] [node 1] ICP22 created new session: [5:7577789375023027292:2048]
2025-11-28T14:52:57.461720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789375023027292:2048] [node 1] ICS09 handshake done sender: [5:7577789375023027282:2127] self: [5:7577789375023027288:0] peer: [1:7577789375330731948:0] socket: 251 qp: -1
2025-11-28T14:52:57.461748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789375023027292:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:57.461798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:57.462296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789372611782772:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:57.462413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789372611782770:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:57.462999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789372611782753:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17625) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789372611782771:2118] held: no
2025-11-28T14:52:57.463068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372611782690:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.463068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372611782692:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.463121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:57.463163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:57.463166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372611782690:2048] [node 1] ICP22 created new session: [6:7577789372611782788:2048]
2025-11-28T14:52:57.463196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789372611782788:2048] [node 1] ICS09 handshake done sender: [6:7577789372611782770:2117] self: [6:7577789372611782781:0] peer: [1:7577789375330731946:0] socket: 284 qp: -1
2025-11-28T14:52:57.463205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372611782692:2050] [node 3] ICP22 created new session: [6:7577789372611782789:2050]
2025-11-28T14:52:57.463223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789372611782788:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:57.463245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:57.463267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789372611782789:2050] [node 3] ICS09 handshake done sender: [6:7577789372611782772:2119] self: [6:7577789372611782780:0] peer: [3:7577789373990488847:0] socket: 287 qp: -1
2025-11-28T14:52:57.463299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789372611782789:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.463369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:57.463549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372611782787:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4465
2025-11-28T14:52:57.464150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789372611782771:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:57.464420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372611782753:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.464469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:57.464499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372611782753:2108] [node 4] ICP22 created new session: [6:7577789372611782792:2108]
2025-11-28T14:52:57.464523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789372611782792:2108] [node 4] ICS09 handshake done sender: [6:7577789372611782771:2118] self: [6:7577789372611782785:0] peer: [4:7577789373245961034:0] socket: 286 qp: -1
2025-11-28T14:52:57.464563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789372611782792:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:57.464602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:57.496165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:57.496336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789374804578221:2048] [node 1] ICS01 socket: 244 reason# EndOfStream
2025-11-28T14:52:57.496382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789374804578095:2048] [node 1] ICP30 unregister session Session# [2:7577789374804578221:2048] VirtualId# [2:7577789374804578209:0]
2025-11-28T14:52:57.496422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789374804578221:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.496534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:57.496593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:57.497323Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:57.497479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789373990488870:2048] [node 1] ICS01 socket: 247 reason# EndOfStream
2025-11-28T14:52:57.497514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789373990488773:2048] [node 1] ICP30 unregister session Session# [3:7577789373990488870:2048] VirtualId# [3:7577789373990488857:0]
2025-11-28T14:52:57.497533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789373990488870:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.497607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:57.497644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:57.499702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:57.499882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789373245961050:2048] [node 1] ICS07 socket disconnect 246 reason# EndOfStream
2025-11-28T14:52:57.499909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789373245961050:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.499981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:57.500022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789373245961050:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:57.500425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:57.500639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789373245960962:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22945) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:57.500677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789375023027292:2048] [node 1] ICS01 socket: 251 reason# EndOfStream
2025-11-28T14:52:57.500680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789373245961050:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:57.500709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789375023027185:2048] [node 1] ICP30 unregister session Session# [5:7577789375023027292:2048] VirtualId# [5:7577789375023027288:0]
2025-11-28T14:52:57.500736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789375023027292:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.500802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:57.500707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789373245960962:2048] [node 1] ICP30 unregister session Session# [4:7577789373245961050:2048] VirtualId# [4:7577789373245961043:0]
2025-11-28T14:52:57.500867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:57.500767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:57.500789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789373245960962:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22945) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.500797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22945) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.501681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:57.501821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789372611782788:2048] [node 1] ICS01 socket: 284 reason# EndOfStream
2025-11-28T14:52:57.501849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372611782690:2048] [node 1] ICP30 unregister session Session# [6:7577789372611782788:2048] VirtualId# [6:7577789372611782781:0]
2025-11-28T14:52:57.501865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789372611782788:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.501915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:57.501944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:57.510568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789373245960962:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:57.544969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:57.545120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789373990488865:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-11-28T14:52:57.545155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789373990488774:2049] [node 2] ICP30 unregister session Session# [3:7577789373990488865:2049] VirtualId# [3:7577789373990488856:0]
2025-11-28T14:52:57.545179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789373990488865:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.545286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:57.545341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:57.546906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:57.547054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789373245961055:2049] [node 2] ICS01 socket: 234 reason# EndOfStream
2025-11-28T14:52:57.547086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789373245960963:2049] [node 2] ICP30 unregister session Session# [4:7577789373245961055:2049] VirtualId# [4:7577789373245961044:0]
2025-11-28T14:52:57.547112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789373245961055:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.547177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:57.547215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:57.547834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:57.547991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789372611782782:2049] [node 2] ICS07 socket disconnect 248 reason# EndOfStream
2025-11-28T14:52:57.548028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789372611782782:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.548076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:57.548114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789372611782782:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:57.548802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789372611782691:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25386) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:57.548906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789372611782782:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:57.548945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372611782691:2049] [node 2] ICP30 unregister session Session# [6:7577789372611782782:2049] VirtualId# [6:7577789372611782775:0]
2025-11-28T14:52:57.548985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:57.549028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789372611782691:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.548996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:57.549049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.549210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789375023027290:2049] [node 2] ICS01 socket: 281 reason# EndOfStream
2025-11-28T14:52:57.549265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789375023027186:2049] [node 2] ICP30 unregister session Session# [5:7577789375023027290:2049] VirtualId# [5:7577789375023027283:0]
2025-11-28T14:52:57.549298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789375023027290:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.549378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:57.549435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:57.558646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789372611782691:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:57.588214Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:57.588421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789373245961064:2050] [node 3] ICS01 socket: 291 reason# EndOfStream
2025-11-28T14:52:57.588467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789373245960964:2050] [node 3] ICP30 unregister session Session# [4:7577789373245961064:2050] VirtualId# [4:7577789373245961058:0]
2025-11-28T14:52:57.588499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789373245961064:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.588572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:57.588606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:57.589238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:57.589405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789375023027280:2050] [node 3] ICS07 socket disconnect 255 reason# EndOfStream
2025-11-28T14:52:57.589438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789375023027280:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.589518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:57.589556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789375023027280:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:57.590238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:57.590248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789375023027187:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29033) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:57.590334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789375023027280:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:57.590366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789375023027187:2050] [node 3] ICP30 unregister session Session# [5:7577789375023027280:2050] VirtualId# [5:7577789375023027273:0]
2025-11-28T14:52:57.590419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789372611782789:2050] [node 3] ICS01 socket: 287 reason# EndOfStream
2025-11-28T14:52:57.590448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:57.590452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372611782692:2050] [node 3] ICP30 unregister session Session# [6:7577789372611782789:2050] VirtualId# [6:7577789372611782780:0]
2025-11-28T14:52:57.590472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789372611782789:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.590474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789375023027187:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29033) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.590486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29033) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.590526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:57.590556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:57.600666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789375023027187:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:57.651753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:57.652006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789375023027277:2108] [node 4] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-28T14:52:57.652054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789375023027277:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.652167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:57.652216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789375023027277:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:57.652483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:57.652650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789372611782792:2108] [node 4] ICS01 socket: 286 reason# EndOfStream
2025-11-28T14:52:57.652686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372611782753:2108] [node 4] ICP30 unregister session Session# [6:7577789372611782792:2108] VirtualId# [6:7577789372611782785:0]
2025-11-28T14:52:57.652709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789372611782792:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.652767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:57.652818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:57.652904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789375023027255:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17625) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:57.652954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789375023027277:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:57.653027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789375023027255:2108] [node 4] ICP30 unregister session Session# [5:7577789375023027277:2108] VirtualId# [5:7577789375023027272:0]
2025-11-28T14:52:57.653167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:57.653195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789375023027255:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.653205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.697034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:57.697201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789372611782776:2109] [node 5] ICS07 socket disconnect 239 reason# EndOfStream
2025-11-28T14:52:57.697229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789372611782776:2109] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:57.697320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:57.697365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789372611782776:2109] [node 5] ICS15 start handshake
2025-11-28T14:52:57.697989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789372611782754:2109] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28168) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:57.698074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789372611782776:2109] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:57.698105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372611782754:2109] [node 5] ICP30 unregister session Session# [6:7577789372611782776:2109] VirtualId# [6:7577789372611782773:0]
2025-11-28T14:52:57.698203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:57.698229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789372611782754:2109] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.698240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:57.708230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789372611782754:2109] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:57.797620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789374213609938:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:57.797784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789374213609939:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:57.814006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789372852645239:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:57.814006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789372852645238:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.831101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789375576259157:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.831154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789375576259158:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:57.846870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789375038560833:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.846887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789375038560835:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:57.846900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789375038560834:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:57.862139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789376355244242:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.862179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789376355244244:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:57.862239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789376355244243:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:57.878482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372267584987:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:57.878478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372267584988:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:57.878504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372267584989:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:57.948748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789374213610015:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:57.948886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789375576259220:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:57.953470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789375576259221:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:57.957924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789374213610020:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:57.957746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789372852645301:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:57.958468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789374213610021:2115] [node 6] ICP01 ready to work
2025-11-28T14:52:57.957229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789375576259222:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:57.958599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372267585064:2120] [node 5] ICP01 ready to work
2025-11-28T14:52:57.958616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789375038560914:2109] [node 5] ICP01 ready to work
2025-11-28T14:52:57.958566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789376355244305:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:57.958443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789372852645303:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:57.959397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789376355244306:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:57.958672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789375038560915:2115] [node 6] ICP01 ready to work
2025-11-28T14:52:57.959542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789372267585065:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:57.958768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789372852645302:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:57.964533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789372267584989:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21220717ns) {Configure:104(21212347ns) {ProcessConfigured:135(183010ns) {StartInitialHandshake:157(181521ns) {DropHandshakes:470(1108ns) {DropIncomingHandshake:442(333ns) DropOutgoingHandshake:457(151ns)} GenerateSessionVirtualId:988(491ns) PrepareNewSessionHandshake:477(778ns) {SwitchToState:179(289ns)}}}}}
2025-11-28T14:52:57.967365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789374213609939:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(30309936ns) {Configure:104(22975344ns) {ProcessConfigured:135(1238378ns) {StartInitialHandshake:157(1236760ns) {DropHandshakes:470(1239ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:477(1761ns) {SwitchToState:179(268ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.968725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372267585074:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:57.971889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789376355244317:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:52:57.972273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789372267585074:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:52:57.972806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789374213610030:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7267
2025-11-28T14:52:57.974067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789372852645318:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42273
2025-11-28T14:52:57.974506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789375576259231:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57097
No RDMA context found for GID: 2025-11-28T14:52:57.974326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789374213610031:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7267
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.974516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789375038560924:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21359
2025-11-28T14:52:57.975357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372267584988:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.974218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789372852645317:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42273
2025-11-28T14:52:57.974832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789375576259234:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57097
2025-11-28T14:52:57.975794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372267585078:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
2025-11-28T14:52:57.975255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789374213610030:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.975814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789375038560928:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21359
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.976148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789372852645307:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.976206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789375038560929:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21359
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.976334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789375038560924:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.976528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789375576259236:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57097
2025-11-28T14:52:57.976185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789372852645307:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.976241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789374213610037:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7267
2025-11-28T14:52:57.978625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372267585081:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.978628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789375038560834:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.976536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789375576259231:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.976532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789374213610031:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.979447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789376355244317:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.979280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789375038560928:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.979697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789375038560833:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.980221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789376355244319:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.980234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789372852645311:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.980282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789374213610027:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.980291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789376355244320:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-28T14:52:57.981698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789372267585078:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.980268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789372852645311:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.980314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789374213610027:2129] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.980294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789372852645307:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:57.984125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789375576259234:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.983404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789376355244319:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.984120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372267585064:2120] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.983898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789374213610037:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.985121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789372267585081:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.983827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789376355244321:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-28T14:52:57.980424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789372852645317:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.985390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789375038560929:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.985852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372267584989:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.986632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372267585083:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
2025-11-28T14:52:57.985412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789374213610027:2129] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:57.985526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789375038560835:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.984732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789372852645318:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.984708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789376355244242:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.986913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789375576259236:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.985578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789374213610029:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.984777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789372852645303:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.985780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789376355244311:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.986944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789375576259230:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:52:57.985598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789374213610029:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.987826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789375038560922:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:52:57.985807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789376355244311:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.984793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789372852645311:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:57.987866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789375038560922:2130] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.986963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789375576259230:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.985066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:57.989131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789374213610029:2131] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.989400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789375038560934:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21359
2025-11-28T14:52:57.987826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789375576259225:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.985118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789372852645303:2109] [node 6] ICP22 created new session: [2:7577789372852645322:2109]
2025-11-28T14:52:57.986656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789376355244321:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.990558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789372267585064:2120] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789372267585078:2129] held: no
2025-11-28T14:52:57.989217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789374213610015:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.986812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789376355244306:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.990668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789375038560922:2130] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:57.987770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789376355244311:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:57.987842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789375576259225:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.990221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789374213610024:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.985384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789372852645322:2109] [node 6] ICS09 handshake done sender: [2:7577789372852645307:2111] self: [2:7577789372852645306:0] peer: [6:7577789372267585082:0] socket: 234 qp: -1
2025-11-28T14:52:57.993836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789375038560936:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21359
2025-11-28T14:52:57.994134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789372267585074:2128] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:57.989589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789375576259230:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:57.985397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789372852645238:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.988034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789376355244305:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.994820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789375038560914:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.990244Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789374213610024:2128] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.990932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789375576259241:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57097
2025-11-28T14:52:57.990931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:57.985611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789372852645322:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:57.996259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372267584988:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.989389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.995287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.985698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789372852645301:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.994238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789375576259225:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:57.996917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:57.989901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789376355244305:2109] [node 6] ICP22 created new session: [5:7577789376355244328:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.993513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789374213610015:2107] [node 4] ICP22 created new session: [1:7577789374213610043:2107]
2025-11-28T14:52:57.994882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789375576259221:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.995337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789375038560914:2109] [node 5] ICP22 created new session: [4:7577789375038560939:2109]
2025-11-28T14:52:57.985829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:57.992013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789376355244328:2109] [node 6] ICS09 handshake done sender: [5:7577789376355244311:2111] self: [5:7577789376355244307:0] peer: [6:7577789372267585084:0] socket: 246 qp: -1
2025-11-28T14:52:57.993888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789374213610043:2107] [node 4] ICS09 handshake done sender: [1:7577789374213610027:2129] self: [1:7577789374213610023:0] peer: [4:7577789375038560933:0] socket: 256 qp: -1
2025-11-28T14:52:57.997599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372267584988:2049] [node 2] ICP22 created new session: [6:7577789372267585088:2049]
2025-11-28T14:52:57.986574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:57.995312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:57.993492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789376355244328:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:57.998283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789372267585088:2049] [node 2] ICS09 handshake done sender: [6:7577789372267585074:2128] self: [6:7577789372267585082:0] peer: [2:7577789372852645306:0] socket: 238 qp: -1
2025-11-28T14:52:57.995815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789375038560924:2132] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:57.993904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789374213610020:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.986918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789372852645301:2107] [node 4] ICP22 created new session: [2:7577789372852645325:2107]
2025-11-28T14:52:57.993498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789376355244320:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.996355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789375038560939:2109] [node 5] ICS09 handshake done sender: [4:7577789375038560922:2130] self: [4:7577789375038560918:0] peer: [5:7577789376355244327:0] socket: 267 qp: -1
2025-11-28T14:52:57.995361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789375576259221:2107] [node 4] ICP22 created new session: [3:7577789375576259245:2107]
2025-11-28T14:52:57.998324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789372267585088:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.993947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:57.987825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789372852645325:2107] [node 4] ICS09 handshake done sender: [2:7577789372852645311:2113] self: [2:7577789372852645308:0] peer: [4:7577789375038560932:0] socket: 236 qp: -1
2025-11-28T14:52:57.995769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789375576259241:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.993865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:57.998657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372267585086:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
2025-11-28T14:52:57.987854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789372852645325:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:57.996611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789375038560939:2109] [node 5] ICS10 traffic start
2025-11-28T14:52:57.994143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789374213610043:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:57.997883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789376355244244:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:57.996329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789375576259222:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.998870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:57.987881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:57.994534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:57.998052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789376355244329:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-28T14:52:57.997165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.005067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789372852645329:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42273
2025-11-28T14:52:58.002740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789372267585078:2129] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:57.996410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:57.994543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789374213610020:2108] [node 5] ICP22 created new session: [1:7577789374213610044:2108]
2025-11-28T14:52:57.998812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789376355244330:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-28T14:52:57.999608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789376355244331:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-28T14:52:58.002943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372267585087:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
2025-11-28T14:52:57.994891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789374213610044:2108] [node 5] ICS09 handshake done sender: [1:7577789374213610029:2131] self: [1:7577789374213610026:0] peer: [5:7577789376355244325:0] socket: 259 qp: -1
2025-11-28T14:52:57.996606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789375576259222:2109] [node 6] ICP22 created new session: [3:7577789375576259246:2109]
2025-11-28T14:52:57.999774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789376355244319:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.005506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372267585064:2120] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.002994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789376355244242:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.997800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789375038560833:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25372) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789375038560928:2133] held: no
2025-11-28T14:52:57.995830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789374213610044:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:57.996956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789375576259245:2107] [node 4] ICS09 handshake done sender: [3:7577789375576259230:2113] self: [3:7577789375576259223:0] peer: [4:7577789375038560935:0] socket: 265 qp: -1
2025-11-28T14:52:58.007703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789372852645318:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.006076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789372267584989:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2527) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789372267585081:2131] held: no
2025-11-28T14:52:57.998232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789375038560834:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:57.996033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.003062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789376355244321:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:57.997605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789375576259245:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:57.996692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789374213610024:2128] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:58.003293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789376355244330:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:57.998636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:58.006693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:58.002053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789374213609938:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:57.997872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789375576259246:2109] [node 6] ICS09 handshake done sender: [3:7577789375576259225:2110] self: [3:7577789375576259224:0] peer: [6:7577789372267585085:0] socket: 258 qp: -1
2025-11-28T14:52:58.003598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:57.998674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789375038560834:2049] [node 2] ICP22 created new session: [4:7577789375038560941:2049]
2025-11-28T14:52:58.006992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372267585064:2120] [node 5] ICP22 created new session: [6:7577789376562552389:2120]
2025-11-28T14:52:58.002718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:57.997900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.003624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789376355244242:2048] [node 1] ICP22 created new session: [5:7577789380650211631:2048]
2025-11-28T14:52:58.007708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789376562552389:2120] [node 5] ICS09 handshake done sender: [6:7577789372267585078:2129] self: [6:7577789372267585084:0] peer: [5:7577789376355244307:0] socket: 247 qp: -1
2025-11-28T14:52:58.003069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789374213609938:2048] [node 2] ICP22 created new session: [1:7577789378508577347:2048]
2025-11-28T14:52:57.999080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789375038560938:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21359
2025-11-28T14:52:57.998058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789375576259246:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:58.007746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789376562552389:2120] [node 5] ICS10 traffic start
2025-11-28T14:52:58.005075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789378508577347:2048] [node 2] ICS09 handshake done sender: [1:7577789374213610024:2128] self: [1:7577789374213610022:0] peer: [2:7577789372852645323:0] socket: 254 qp: -1
2025-11-28T14:52:58.005051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789380650211631:2048] [node 1] ICS09 handshake done sender: [5:7577789376355244319:2115] self: [5:7577789376355244325:0] peer: [1:7577789374213610026:0] socket: 261 qp: -1
2025-11-28T14:52:57.999374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789375038560941:2049] [node 2] ICS09 handshake done sender: [4:7577789375038560924:2132] self: [4:7577789375038560932:0] peer: [2:7577789372852645308:0] socket: 239 qp: -1
2025-11-28T14:52:58.008206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789372267585081:2131] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:58.005556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789380650211631:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.009565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789372852645310:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.008392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.006026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:58.009220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372267584989:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:57.999635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789375038560941:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:58.005297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789378508577347:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:58.006434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789376355244306:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.009596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789372852645310:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:57.998539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:58.005322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:58.006721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:58.000474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:58.007202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789376355244306:2108] [node 4] ICP22 created new session: [5:7577789380650211633:2108]
2025-11-28T14:52:58.009381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:58.006710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789375038560835:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2527) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789375038560929:2134] held: no
2025-11-28T14:52:58.002921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789375576259228:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.007538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789380650211633:2108] [node 4] ICS09 handshake done sender: [5:7577789376355244321:2117] self: [5:7577789376355244327:0] peer: [4:7577789375038560918:0] socket: 272 qp: -1
2025-11-28T14:52:58.010819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789372852645238:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.007902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789380650211633:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:58.009590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372267584989:2050] [node 3] ICP22 created new session: [6:7577789376562552391:2050]
2025-11-28T14:52:58.007098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789375038560928:2133] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.010863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:58.008499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789375038560929:2134] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:58.008197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:58.010941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789376355244243:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.010596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789372267585090:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
2025-11-28T14:52:58.010999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789372852645238:2048] [node 1] ICP22 created new session: [2:7577789377147612627:2048]
2025-11-28T14:52:58.010453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789375038560835:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.002947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789375576259228:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.010663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789376562552391:2050] [node 3] ICS09 handshake done sender: [6:7577789372267585081:2131] self: [6:7577789372267585085:0] peer: [3:7577789375576259224:0] socket: 268 qp: -1
2025-11-28T14:52:58.010630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789375038560833:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.011342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789377147612627:2048] [node 1] ICS09 handshake done sender: [2:7577789372852645318:2116] self: [2:7577789372852645323:0] peer: [1:7577789374213610022:0] socket: 255 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.010653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:58.006972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789375576259158:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.010853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789376562552391:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:58.011379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789377147612627:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.011502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:58.010866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789375038560835:2050] [node 3] ICP22 created new session: [4:7577789379333528241:2050]
2025-11-28T14:52:58.008428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789375576259228:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:58.010890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:58.010916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789379333528241:2050] [node 3] ICS09 handshake done sender: [4:7577789375038560929:2134] self: [4:7577789375038560935:0] peer: [3:7577789375576259223:0] socket: 271 qp: -1
2025-11-28T14:52:58.010904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.010942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789375038560833:2048] [node 1] ICP22 created new session: [4:7577789379333528242:2048]
2025-11-28T14:52:58.009269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789375576259251:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57097
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.011080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789379333528241:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:58.011114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.011901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789374213610050:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7267
2025-11-28T14:52:58.013359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789375576259220:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.011278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789379333528242:2048] [node 1] ICS09 handshake done sender: [4:7577789375038560928:2133] self: [4:7577789375038560933:0] peer: [1:7577789374213610023:0] socket: 257 qp: -1
2025-11-28T14:52:58.011411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789379333528242:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.013776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:58.011521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:58.015120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789372852645303:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23899807ns) {HandleHandshakeStatus:329(23898084ns) {DropIncomingHandshake:442(475ns) DropOutgoingHandshake:457(417ns) SwitchToState:179(807ns) ProcessPendingSessionEvents:495(20271742ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18192904ns max# 2037846ns}}}
2025-11-28T14:52:58.014983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789380650211635:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-28T14:52:58.014098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789375576259220:2108] [node 5] ICP22 created new session: [3:7577789379871226550:2108]
2025-11-28T14:52:58.015819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789372852645301:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22617697ns) {HandleHandshakeStatus:329(22615966ns) {DropIncomingHandshake:442(178ns) DropOutgoingHandshake:457(298ns) SwitchToState:179(179ns) ProcessPendingSessionEvents:495(18007878ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16740089ns max# 1454079ns}}}
2025-11-28T14:52:58.016169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789372267585090:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.015941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789376355244305:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20992011ns) {HandleHandshakeStatus:329(20989960ns) {DropIncomingHandshake:442(8700ns) DropOutgoingHandshake:457(1571ns) SwitchToState:179(184ns) ProcessPendingSessionEvents:495(12791828ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12711370ns max# 1580916ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.015380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789379871226550:2108] [node 5] ICS09 handshake done sender: [3:7577789375576259228:2111] self: [3:7577789375576259226:0] peer: [5:7577789376355244334:0] socket: 266 qp: -1
2025-11-28T14:52:58.016283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789372852645316:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.016337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372267584987:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:58.015806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789379871226550:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:58.016319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789372852645316:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.016308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789378508577349:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7267
2025-11-28T14:52:58.016343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789376562552393:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
2025-11-28T14:52:58.016608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789377147612626:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42273
2025-11-28T14:52:58.016147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.017302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789376355244320:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:58.017278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789375576259251:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.017296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789379333528245:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21359
2025-11-28T14:52:58.017295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789372852645238:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:58.017603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789375038560914:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(16264940ns) {HandleHandshakeStatus:329(16101241ns) {DropIncomingHandshake:442(225ns) DropOutgoingHandshake:457(309ns) SwitchToState:179(136ns) ProcessPendingSessionEvents:495(12339519ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11983419ns max# 1145157ns}}}
2025-11-28T14:52:58.017619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789374213610020:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17043354ns) {HandleHandshakeStatus:329(17041022ns) {DropIncomingHandshake:442(522ns) DropOutgoingHandshake:457(378ns) SwitchToState:179(270ns) ProcessPendingSessionEvents:495(13713246ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12067021ns max# 1449470ns}}}
2025-11-28T14:52:58.018444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789379333528245:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.018831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789376355244244:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2527) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789376355244320:2116] held: no
2025-11-28T14:52:58.018366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789374213610036:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.018403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789374213610036:2134] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.018867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789374213610015:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22893117ns) {HandleHandshakeStatus:329(22891195ns) {DropIncomingHandshake:442(13480ns) DropOutgoingHandshake:457(271ns) SwitchToState:179(690ns) ProcessPendingSessionEvents:495(14539355ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13578229ns max# 1141820ns}}}
2025-11-28T14:52:58.020338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789378508577349:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.020465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789377147612626:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.020724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789375576259222:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16090380ns) {HandleHandshakeStatus:329(16088515ns) {DropIncomingHandshake:442(10426ns) DropOutgoingHandshake:457(235ns) SwitchToState:179(162ns) ProcessPendingSessionEvents:495(12107378ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11672559ns max# 1431778ns}}}
2025-11-28T14:52:58.020453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789376562552393:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.020975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789375576259157:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.020787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789372267585065:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.021155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789375576259221:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16625067ns) {HandleHandshakeStatus:329(16614179ns) {DropIncomingHandshake:442(13152ns) DropOutgoingHandshake:457(230ns) SwitchToState:179(100ns) ProcessPendingSessionEvents:495(12089060ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11157637ns max# 1148989ns}}}
2025-11-28T14:52:58.021038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789375038560931:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.021073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789375038560931:2135] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.021545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789376355244244:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.021603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:58.021654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789376355244244:2050] [node 3] ICP22 created new session: [5:7577789380650211638:2050]
2025-11-28T14:52:58.021625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789372852645310:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:58.021625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789379871226552:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57097
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.022263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789380650211638:2050] [node 3] ICS09 handshake done sender: [5:7577789376355244320:2116] self: [5:7577789376355244334:0] peer: [3:7577789375576259226:0] socket: 269 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.022443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789375038560931:2135] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:58.022291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789374213610028:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.022321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789376562552396:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
2025-11-28T14:52:58.022315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789380650211638:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:58.022315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789374213610028:2130] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.022438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.022373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789374213610020:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:58.022985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789375576259241:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:58.023385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789372852645239:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.023595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789380650211637:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50229
2025-11-28T14:52:58.023639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:58.023634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789375576259158:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.024080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789372852645316:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:58.022467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789374213610036:2134] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:58.024099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789372852645239:2049] [node 3] ICP22 created new session: [2:7577789377147612631:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.024106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789374213610021:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.024373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789377147612631:2049] [node 3] ICS09 handshake done sender: [2:7577789372852645310:2112] self: [2:7577789372852645309:0] peer: [3:7577789379871226548:0] socket: 282 qp: -1
2025-11-28T14:52:58.024873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789372267585090:2135] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.024571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:58.024143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:58.024408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789377147612631:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:58.025957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789376562552397:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36982
2025-11-28T14:52:58.024919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.024960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789375576259158:2049] [node 2] ICP22 created new session: [3:7577789379871226555:2049]
2025-11-28T14:52:58.026490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789376355244330:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:58.026218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789372852645302:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.024152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789374213610028:2130] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:58.026075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789379871226555:2049] [node 2] ICS09 handshake done sender: [3:7577789375576259241:2118] self: [3:7577789379871226548:0] peer: [2:7577789372852645309:0] socket: 283 qp: -1
2025-11-28T14:52:58.026698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:58.024376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789374213610021:2115] [node 6] ICP22 created new session: [1:7577789378508577353:2115]
2025-11-28T14:52:58.026738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372267584987:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.026533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789379871226555:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:58.026758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789372852645302:2108] [node 5] ICP22 created new session: [2:7577789377147612633:2108]
2025-11-28T14:52:58.027013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:58.027360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789375038560915:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.027174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372267584987:2048] [node 1] ICP22 created new session: [6:7577789376562552398:2048]
2025-11-28T14:52:58.026980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789377147612633:2108] [node 5] ICS09 handshake done sender: [2:7577789372852645316:2114] self: [2:7577789372852645315:0] peer: [5:7577789380650211636:0] socket: 288 qp: -1
2025-11-28T14:52:58.026612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:58.027705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:58.027670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789376562552398:2048] [node 1] ICS09 handshake done sender: [6:7577789372267585090:2135] self: [6:7577789376562552394:0] peer: [1:7577789374213610035:0] socket: 297 qp: -1
2025-11-28T14:52:58.024873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789378508577353:2115] [node 6] ICS09 handshake done sender: [1:7577789374213610036:2134] self: [1:7577789374213610035:0] peer: [6:7577789376562552394:0] socket: 295 qp: -1
2025-11-28T14:52:58.027751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789375038560915:2115] [node 6] ICP22 created new session: [4:7577789379333528248:2115]
2025-11-28T14:52:58.027167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789377147612633:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.028133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789379871226554:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57097
2025-11-28T14:52:58.025965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789378508577353:2115] [node 6] ICS10 traffic start
2025-11-28T14:52:58.027805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789376562552398:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.026010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:58.027875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789379333528248:2115] [node 6] ICS09 handshake done sender: [4:7577789375038560931:2135] self: [4:7577789375038560930:0] peer: [6:7577789376562552395:0] socket: 302 qp: -1
2025-11-28T14:52:58.027832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:58.027195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.028169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789379333528248:2115] [node 6] ICS10 traffic start
2025-11-28T14:52:58.026885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789374213609939:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.028785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789376355244243:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.027025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:58.028201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:58.029548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789376562552393:2136] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:58.030756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789375576259157:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25372) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789375576259251:2119] held: no
2025-11-28T14:52:58.027193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789374213609939:2049] [node 3] ICP22 created new session: [1:7577789378508577355:2049]
2025-11-28T14:52:58.031493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:58.027728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789378508577355:2049] [node 3] ICS09 handshake done sender: [1:7577789374213610028:2130] self: [1:7577789374213610025:0] peer: [3:7577789379871226553:0] socket: 240 qp: -1
2025-11-28T14:52:58.032065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789372267585065:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.027863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789378508577355:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:58.031749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789375576259251:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.028114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.032121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:58.031568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789376355244243:2049] [node 2] ICP22 created new session: [5:7577789380650211640:2049]
2025-11-28T14:52:58.032247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789372267585065:2106] [node 4] ICP22 created new session: [6:7577789376562552400:2106]
2025-11-28T14:52:58.032034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789380650211640:2049] [node 2] ICS09 handshake done sender: [5:7577789376355244330:2120] self: [5:7577789380650211636:0] peer: [2:7577789372852645315:0] socket: 245 qp: -1
2025-11-28T14:52:58.032105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789380650211640:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:58.035301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789376562552400:2106] [node 4] ICS09 handshake done sender: [6:7577789376562552393:2136] self: [6:7577789376562552395:0] peer: [4:7577789375038560930:0] socket: 303 qp: -1
2025-11-28T14:52:58.032226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:58.035365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789376562552400:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:58.035622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789372852645239:2049] [node 3] ICP18 handshake (actor [2:7577789377147612626:2118]) is too old
2025-11-28T14:52:58.035410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:58.036194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789375576259157:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.036243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:58.036341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789375576259157:2048] [node 1] ICP22 created new session: [3:7577789379871226557:2048]
2025-11-28T14:52:58.036435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789379871226557:2048] [node 1] ICS09 handshake done sender: [3:7577789375576259251:2119] self: [3:7577789379871226553:0] peer: [1:7577789374213610025:0] socket: 298 qp: -1
2025-11-28T14:52:58.036501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789379871226557:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.036630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:58.036671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789372267585065:2106] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:58.036678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789376355244243:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:58.096359Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:58.096526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789377147612627:2048] [node 1] ICS07 socket disconnect 255 reason# EndOfStream
2025-11-28T14:52:58.096559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789377147612627:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.096651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.096692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789377147612627:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:58.097402Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:58.097559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789379871226557:2048] [node 1] ICS01 socket: 298 reason# EndOfStream
2025-11-28T14:52:58.097591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789375576259157:2048] [node 1] ICP30 unregister session Session# [3:7577789379871226557:2048] VirtualId# [3:7577789379871226553:0]
2025-11-28T14:52:58.097610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789379871226557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.097674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.097720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.098351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789372852645238:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25372) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:58.098382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789377147612627:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:58.098992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789372852645238:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25372) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.099083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789377147612627:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.099352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789372852645238:2048] [node 1] ICP30 unregister session Session# [2:7577789377147612627:2048] VirtualId# [2:7577789372852645323:0]
2025-11-28T14:52:58.099425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.099463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789372852645238:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25372) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.099487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25372) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.099598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:58.099818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789379333528242:2048] [node 1] ICS01 socket: 257 reason# EndOfStream
2025-11-28T14:52:58.099867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789375038560833:2048] [node 1] ICP30 unregister session Session# [4:7577789379333528242:2048] VirtualId# [4:7577789375038560933:0]
2025-11-28T14:52:58.099905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789379333528242:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.099991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.100043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.100462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:58.100657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789380650211631:2048] [node 1] ICS01 socket: 261 reason# EndOfStream
2025-11-28T14:52:58.100709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789376355244242:2048] [node 1] ICP30 unregister session Session# [5:7577789380650211631:2048] VirtualId# [5:7577789376355244325:0]
2025-11-28T14:52:58.100742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789380650211631:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.100833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.100909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.101395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:58.101539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789376562552398:2048] [node 1] ICS07 socket disconnect 297 reason# EndOfStream
2025-11-28T14:52:58.101572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789376562552398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.101653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.101699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789376562552398:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:58.102451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789372267584987:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25372) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.102532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789376562552398:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.102564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372267584987:2048] [node 1] ICP30 unregister session Session# [6:7577789376562552398:2048] VirtualId# [6:7577789376562552394:0]
2025-11-28T14:52:58.102610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.102644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789372267584987:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25372) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.102656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25372) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.112664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789372267584987:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:58.144733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:58.144897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789379871226555:2049] [node 2] ICS07 socket disconnect 283 reason# EndOfStream
2025-11-28T14:52:58.144925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789379871226555:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.145006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:58.145042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789379871226555:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:58.147012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:58.147117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789375576259158:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8613) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:58.147143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789379871226555:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:58.147258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789375038560941:2049] [node 2] ICS01 socket: 239 reason# EndOfStream
2025-11-28T14:52:58.147303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789375038560834:2049] [node 2] ICP30 unregister session Session# [4:7577789375038560941:2049] VirtualId# [4:7577789375038560932:0]
2025-11-28T14:52:58.147334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789375038560941:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.147414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:58.147667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:58.148221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789375576259158:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8613) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.148258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789379871226555:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.148282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789375576259158:2049] [node 2] ICP30 unregister session Session# [3:7577789379871226555:2049] VirtualId# [3:7577789379871226548:0]
2025-11-28T14:52:58.148321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:58.148472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:58.148501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789375576259158:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8613) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.148510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8613) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.148531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789380650211640:2049] [node 2] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-28T14:52:58.148580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789380650211640:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.148648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:58.148692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789380650211640:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:58.149392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:58.149534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789376355244243:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8613) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.149578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789372267585088:2049] [node 2] ICS01 socket: 238 reason# EndOfStream
2025-11-28T14:52:58.149587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789380650211640:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.149664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789376355244243:2049] [node 2] ICP30 unregister session Session# [5:7577789380650211640:2049] VirtualId# [5:7577789380650211636:0]
2025-11-28T14:52:58.149662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372267584988:2049] [node 2] ICP30 unregister session Session# [6:7577789372267585088:2049] VirtualId# [6:7577789372267585082:0]
2025-11-28T14:52:58.149702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789372267585088:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.149786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:58.149796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:58.149848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:58.149865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789376355244243:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8613) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.149906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8613) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.158775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789375576259158:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:58.159783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789376355244243:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:58.210818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:58.211054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789379333528241:2050] [node 3] ICS01 socket: 271 reason# EndOfStream
2025-11-28T14:52:58.211104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789375038560835:2050] [node 3] ICP30 unregister session Session# [4:7577789379333528241:2050] VirtualId# [4:7577789375038560935:0]
2025-11-28T14:52:58.211146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789379333528241:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.211226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:58.211288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:58.211956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:58.212127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789380650211638:2050] [node 3] ICS01 socket: 269 reason# EndOfStream
2025-11-28T14:52:58.212169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789376355244244:2050] [node 3] ICP30 unregister session Session# [5:7577789380650211638:2050] VirtualId# [5:7577789376355244334:0]
2025-11-28T14:52:58.212198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789380650211638:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.212267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:58.212318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:58.213095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:58.213315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789376562552391:2050] [node 3] ICS01 socket: 268 reason# EndOfStream
2025-11-28T14:52:58.213387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372267584989:2050] [node 3] ICP30 unregister session Session# [6:7577789376562552391:2050] VirtualId# [6:7577789372267585085:0]
2025-11-28T14:52:58.213422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789376562552391:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.213539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:58.213604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:58.254597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:58.254797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789380650211633:2108] [node 4] ICS01 socket: 272 reason# EndOfStream
2025-11-28T14:52:58.254842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789376355244306:2108] [node 4] ICP30 unregister session Session# [5:7577789380650211633:2108] VirtualId# [5:7577789376355244327:0]
2025-11-28T14:52:58.254900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789380650211633:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.255001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:58.255053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:58.255409Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:58.255644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789376562552400:2106] [node 4] ICS07 socket disconnect 303 reason# EndOfStream
2025-11-28T14:52:58.255682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789376562552400:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.255775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:58.255839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789376562552400:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:58.256582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789372267585065:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28499) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.256679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789376562552400:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.256734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372267585065:2106] [node 4] ICP30 unregister session Session# [6:7577789376562552400:2106] VirtualId# [6:7577789376562552395:0]
2025-11-28T14:52:58.256840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:58.256888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789372267585065:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28499) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.256910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28499) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.266279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789372267585065:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:58.318983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:58.319217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789376562552389:2120] [node 5] ICS01 socket: 247 reason# EndOfStream
2025-11-28T14:52:58.319260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789372267585064:2120] [node 5] ICP30 unregister session Session# [6:7577789376562552389:2120] VirtualId# [6:7577789372267585084:0]
2025-11-28T14:52:58.319289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789376562552389:2120] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.319401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:58.319445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:58.430824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789376595441385:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:58.431071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789376595441386:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:58.458530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789376517507868:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:58.458532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789376517507869:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:58.481075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789378979995372:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:58.481143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789378979995371:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:58.506163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789377086374204:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:58.506202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789377086374206:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:58.506236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789377086374205:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:58.528874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789378012515596:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:58.528876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789378012515597:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:58.529172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789378012515598:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:58.556515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789379409233161:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:58.556518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789379409233162:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:58.556719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789379409233163:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:58.636120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789379409233222:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:58.646662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789376517507932:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:58.644216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789376595441449:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:58.646996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789376517507933:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:58.646982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789377086374268:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:58.646743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789376595441451:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:58.647025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789376517507934:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:58.646990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789377086374269:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:58.646364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789378012515670:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:58.646591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789378979995461:2117] [node 5] ICP01 ready to work
2025-11-28T14:52:58.646760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789376595441450:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:58.646759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789378979995462:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:58.647059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789379409233224:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:58.647335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789378012515671:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:58.647479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789378979995463:2127] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.659952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789377086374281:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3946
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.659934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789378012515684:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-28T14:52:58.662369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233236:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-28T14:52:58.663096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789377086374285:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3946
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.666200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789377086374284:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3946
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.667292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789376517507948:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15098
2025-11-28T14:52:58.666453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789378979995477:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8695
2025-11-28T14:52:58.666440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233242:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.663811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789376595441462:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32852
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.670080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789376517507948:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.668147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789377086374286:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3946
2025-11-28T14:52:58.668186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789378012515686:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-28T14:52:58.669528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233240:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
2025-11-28T14:52:58.670365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789378012515684:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.670737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789378012515598:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.667300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789376595441466:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32852
2025-11-28T14:52:58.669588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789377086374284:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.670137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.670336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789376517507949:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15098
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.670629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789379409233236:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.670137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789376595441467:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32852
2025-11-28T14:52:58.671053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789379409233240:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.670392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789377086374286:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.670623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789376595441462:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.671478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789376517507950:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15098
2025-11-28T14:52:58.671548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789379409233163:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.670723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789377086374281:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.672041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789378979995469:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.671509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789376517507949:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.671767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233243:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
2025-11-28T14:52:58.672078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789378979995469:2137] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.671087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789376595441466:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.675323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789378012515677:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.670850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789377086374204:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.675290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789376517507950:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.672702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789378979995471:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.677257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789376517507868:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.671149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789376595441467:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.671775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789379409233224:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.675361Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789378012515677:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.672722Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789378979995471:2139] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.671035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789377086374285:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.678076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789376517507940:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.672044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789376595441458:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.678098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789376517507940:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.672727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789378979995470:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.678132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789378012515686:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.671128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789377086374206:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.671815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789379409233241:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.672075Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789376595441458:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.672765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789378979995470:2138] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.672083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789377086374276:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.671919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789379409233222:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.675850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789376595441458:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:58.678442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789378012515677:2121] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.679519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789376517507942:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.672106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789377086374276:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.676908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789378979995471:2139] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:58.675907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789379409233242:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.678732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789378012515597:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.679546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789376517507942:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.679560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789376595441459:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.675931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789377086374276:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:58.676873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789379409233243:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.676945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789378979995469:2137] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:58.679578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789376595441459:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.679770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789376517507939:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.685397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789378012515671:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.676954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789379409233162:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.678324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789377086374268:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.676969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789378979995477:2142] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.679861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789376595441451:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.685429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:58.679801Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789376517507939:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.678504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:58.677343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789379409233161:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.678419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789378979995463:2127] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.686461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789378012515689:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-28T14:52:58.681445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789376517507942:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:58.681002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789376595441460:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.678458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789378979995372:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.678738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789377086374268:2109] [node 6] ICP22 created new session: [4:7577789377086374293:2109]
2025-11-28T14:52:58.679771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789379409233224:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27151) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789379409233240:2116] held: no
2025-11-28T14:52:58.682489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789376517507940:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:58.681036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789376595441460:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.686532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789378012515671:2109] [node 6] ICP22 created new session: [5:7577789378012515692:2109]
2025-11-28T14:52:58.681060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:58.679806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789377086374293:2109] [node 6] ICS09 handshake done sender: [4:7577789377086374276:2112] self: [4:7577789377086374271:0] peer: [6:7577789379409233244:0] socket: 248 qp: -1
2025-11-28T14:52:58.688952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233250:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
2025-11-28T14:52:58.686653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789376517507939:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:58.678785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789378979995470:2138] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:58.687287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789378012515692:2109] [node 6] ICS09 handshake done sender: [5:7577789378012515677:2121] self: [5:7577789378012515674:0] peer: [6:7577789379409233246:0] socket: 249 qp: -1
2025-11-28T14:52:58.679846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789377086374293:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:52:58.687511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789378012515692:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.02025-11-28T14:52:58.688340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789376517507869:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.679002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:58.690245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233249:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
2025-11-28T14:52:58.682428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789376595441451:2107] [node 4] ICP22 created new session: [1:7577789376595441472:2107]
2025-11-28T14:52:58.679884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.683397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789376595441459:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:58.679056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789378979995461:2117] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.687910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:58.691995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789379409233236:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.683997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789377086374292:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3946
2025-11-28T14:52:58.683554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789376595441472:2107] [node 4] ICS09 handshake done sender: [1:7577789376595441458:2112] self: [1:7577789376595441453:0] peer: [4:7577789377086374288:0] socket: 270 qp: -1
2025-11-28T14:52:58.689024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.679086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:58.683979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789376595441472:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:58.692207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233251:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
2025-11-28T14:52:58.689101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789376517507869:2049] [node 3] ICP22 created new session: [2:7577789376517507959:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.689388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789378012515684:2126] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:58.679143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789378979995461:2117] [node 5] ICP22 created new session: [3:7577789378979995483:2117]
2025-11-28T14:52:58.687483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789377086374204:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789377086374286:2119] held: no
2025-11-28T14:52:58.684009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:58.692478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789379409233163:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.689100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789376517507933:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.690590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789378012515598:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.684983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789376595441385:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.689356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789377086374294:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3946
2025-11-28T14:52:58.692654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789379409233240:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:58.689399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:58.686478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:58.690351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789377086374281:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.692046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789378012515691:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-28T14:52:58.693167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:58.686999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789376595441385:2048] [node 2] ICP22 created new session: [1:7577789376595441476:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.690059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789376517507933:2109] [node 6] ICP22 created new session: [2:7577789376517507960:2109]
2025-11-28T14:52:58.679749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789378979995463:2127] [node 6] ICP22 created new session: [3:7577789378979995482:2127]
2025-11-28T14:52:58.692059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.691911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789377086374206:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.681269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789378979995483:2117] [node 5] ICS09 handshake done sender: [3:7577789378979995469:2137] self: [3:7577789378979995468:0] peer: [5:7577789378012515687:0] socket: 242 qp: -1
2025-11-28T14:52:58.690230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789376517507932:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.694343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233253:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
2025-11-28T14:52:58.692694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:58.687309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789376595441476:2048] [node 2] ICS09 handshake done sender: [1:7577789376595441459:2113] self: [1:7577789376595441455:0] peer: [2:7577789376517507953:0] socket: 275 qp: -1
2025-11-28T14:52:58.694941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789379409233163:2050] [node 3] ICP22 created new session: [6:7577789379409233255:2050]
2025-11-28T14:52:58.692632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789378012515598:2050] [node 3] ICP22 created new session: [5:7577789378012515696:2050]
2025-11-28T14:52:58.687882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789376595441476:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:58.682410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789378979995483:2117] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.692758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789377086374206:2050] [node 3] ICP22 created new session: [4:7577789377086374298:2050]
2025-11-28T14:52:58.692826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789378012515696:2050] [node 3] ICS09 handshake done sender: [5:7577789378012515684:2126] self: [5:7577789378012515687:0] peer: [3:7577789378979995468:0] socket: 245 qp: -1
2025-11-28T14:52:58.690517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789376517507960:2109] [node 6] ICS09 handshake done sender: [2:7577789376517507940:2112] self: [2:7577789376517507937:0] peer: [6:7577789379409233247:0] socket: 256 qp: -1
2025-11-28T14:52:58.682443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.693777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789377086374298:2050] [node 3] ICS09 handshake done sender: [4:7577789377086374281:2116] self: [4:7577789377086374289:0] peer: [3:7577789378979995465:0] socket: 244 qp: -1
2025-11-28T14:52:58.695023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789377086374298:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:58.688187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789376595441460:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:58.693119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789378012515696:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:58.696591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789377086374286:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.683493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789378979995482:2127] [node 6] ICS09 handshake done sender: [3:7577789378979995471:2139] self: [3:7577789378979995466:0] peer: [6:7577789379409233245:0] socket: 240 qp: -1
2025-11-28T14:52:58.690699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789376517507959:2049] [node 3] ICS09 handshake done sender: [2:7577789376517507942:2114] self: [2:7577789376517507938:0] peer: [3:7577789378979995480:0] socket: 262 qp: -1
2025-11-28T14:52:58.695268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789379409233252:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36183
2025-11-28T14:52:58.697056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.688200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:58.693171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.695345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789379409233255:2050] [node 3] ICS09 handshake done sender: [6:7577789379409233236:2115] self: [6:7577789379409233245:0] peer: [3:7577789378979995466:0] socket: 243 qp: -1
2025-11-28T14:52:58.690726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789376517507959:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:58.695380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789379409233255:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:58.698462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789377086374204:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.683750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789378979995482:2127] [node 6] ICS10 traffic start
2025-11-28T14:52:58.692026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789376595441449:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.691504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789376517507960:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:58.696669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.697287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789378012515693:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-28T14:52:58.697768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789379409233224:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.691887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:58.700139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789377086374278:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.692197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:58.683988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789378979995462:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.697888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789379409233162:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789379409233242:2118] held: no
2025-11-28T14:52:58.691910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.700159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789377086374278:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.697802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789378012515691:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.684990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:58.692240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789376595441449:2109] [node 6] ICP22 created new session: [1:7577789376595441481:2109]
2025-11-28T14:52:58.692068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:58.697936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789379409233241:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:58.700335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789377086374300:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3946
2025-11-28T14:52:58.697867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789378012515697:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-28T14:52:58.685040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789378979995462:2107] [node 4] ICP22 created new session: [3:7577789378979995488:2107]
2025-11-28T14:52:58.698839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:58.701378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:58.701425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789377086374204:2048] [node 1] ICP22 created new session: [4:7577789377086374301:2048]
2025-11-28T14:52:58.692242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789376517507932:2108] [node 5] ICP22 created new session: [2:7577789376517507962:2108]
2025-11-28T14:52:58.698882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789379409233224:2108] [node 4] ICP22 created new session: [6:7577789379409233257:2108]
2025-11-28T14:52:58.693128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789376595441477:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32852
2025-11-28T14:52:58.685967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:58.701509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789377086374278:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:58.698187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789378012515670:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.700503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789379409233257:2108] [node 4] ICS09 handshake done sender: [6:7577789379409233240:2116] self: [6:7577789379409233244:0] peer: [4:7577789377086374271:0] socket: 250 qp: -1
2025-11-28T14:52:58.692694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789376517507962:2108] [node 5] ICS09 handshake done sender: [2:7577789376517507939:2111] self: [2:7577789376517507936:0] peer: [5:7577789378012515690:0] socket: 264 qp: -1
2025-11-28T14:52:58.693810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789376595441481:2109] [node 6] ICS09 handshake done sender: [1:7577789376595441460:2114] self: [1:7577789376595441454:0] peer: [6:7577789379409233248:0] socket: 278 qp: -1
2025-11-28T14:52:58.702544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789377086374269:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.687217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789378979995488:2107] [node 4] ICS09 handshake done sender: [3:7577789378979995470:2138] self: [3:7577789378979995465:0] peer: [4:7577789377086374289:0] socket: 241 qp: -1
2025-11-28T14:52:58.699619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789378012515698:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-28T14:52:58.694263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789376595441481:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:58.692832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789376517507956:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15098
2025-11-28T14:52:58.703176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789377086374301:2048] [node 1] ICS09 handshake done sender: [4:7577789377086374286:2119] self: [4:7577789377086374288:0] peer: [1:7577789376595441453:0] socket: 272 qp: -1
2025-11-28T14:52:58.701447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789379409233257:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:58.687451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789378979995488:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:58.693125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789376517507962:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:58.695291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:58.701479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:58.699849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789378012515697:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.703851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789377086374301:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.687482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:58.693883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.702929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789379409233161:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789379409233243:2119] held: no
2025-11-28T14:52:58.699720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789376595441477:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.693079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789378979995487:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8695
2025-11-28T14:52:58.700209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789378012515686:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:58.703884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:58.705205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789379409233222:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.695087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789376517507958:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15098
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.702934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789376595441461:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.698106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789378979995491:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8695
2025-11-28T14:52:58.700476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789378012515596:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.704152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:58.697968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789376517507950:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.702964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789376595441461:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.705850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:58.704537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789377086374269:2108] [node 5] ICP22 created new session: [4:7577789377086374304:2108]
2025-11-28T14:52:58.699705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789378979995477:2142] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:58.702092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789378012515597:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.706021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789379409233222:2107] [node 5] ICP22 created new session: [6:7577789379409233259:2107]
2025-11-28T14:52:58.703194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789376595441483:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32852
2025-11-28T14:52:58.698861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789376517507956:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.708587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789379409233259:2107] [node 5] ICS09 handshake done sender: [6:7577789379409233241:2117] self: [6:7577789379409233246:0] peer: [5:7577789378012515674:0] socket: 259 qp: -1
2025-11-28T14:52:58.705802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789376595441461:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:58.705540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789377086374304:2108] [node 5] ICS09 handshake done sender: [4:7577789377086374278:2114] self: [4:7577789377086374274:0] peer: [5:7577789378012515700:0] socket: 297 qp: -1
2025-11-28T14:52:58.704296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:58.699811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789378979995491:2144] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.701417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789378979995492:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8695
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.708614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789379409233243:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.708382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789376595441450:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.705565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789377086374304:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:58.708859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789379409233259:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:58.699799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789376517507964:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15098
2025-11-28T14:52:58.703930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789378979995372:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.709035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:58.709067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.706066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.705139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789378012515597:2049] [node 2] ICP22 created new session: [5:7577789378012515703:2049]
2025-11-28T14:52:58.709696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789379409233242:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:58.709704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789376595441450:2108] [node 5] ICP22 created new session: [1:7577789376595441486:2108]
2025-11-28T14:52:58.712115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789379409233161:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.707704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789377086374300:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.705624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789378012515703:2049] [node 2] ICS09 handshake done sender: [5:7577789378012515686:2127] self: [5:7577789378012515690:0] peer: [2:7577789376517507936:0] socket: 267 qp: -1
2025-11-28T14:52:58.711007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789376595441486:2108] [node 5] ICS09 handshake done sender: [1:7577789376595441461:2115] self: [1:7577789376595441456:0] peer: [5:7577789378012515701:0] socket: 312 qp: -1
2025-11-28T14:52:58.704233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:58.702550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789376517507868:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.712129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789379409233162:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.712846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:58.708888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789377086374268:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(22204477ns) {HandleHandshakeStatus:329(22201369ns) {DropIncomingHandshake:442(19361ns) DropOutgoingHandshake:457(252ns) SwitchToState:179(212ns) ProcessPendingSessionEvents:495(17258516ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16260895ns max# 1142066ns}}}
2025-11-28T14:52:58.711439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789376595441486:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:58.705908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789378012515703:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:58.703965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789376517507964:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.714561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789379409233161:2048] [node 1] ICP22 created new session: [6:7577789379409233261:2048]
2025-11-28T14:52:58.714579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:58.709079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789377086374205:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.711480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:58.714810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789379409233261:2048] [node 1] ICS09 handshake done sender: [6:7577789379409233243:2119] self: [6:7577789379409233248:0] peer: [1:7577789376595441454:0] socket: 280 qp: -1
2025-11-28T14:52:58.704272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789378979995372:2049] [node 2] ICP22 created new session: [3:7577789378979995493:2049]
2025-11-28T14:52:58.711602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789376595441451:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(24130743ns) {HandleHandshakeStatus:329(24128366ns) {DropIncomingHandshake:442(16449ns) DropOutgoingHandshake:457(1726ns) SwitchToState:179(806ns) ProcessPendingSessionEvents:495(16500845ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14541177ns max# 1542907ns}}}
2025-11-28T14:52:58.714860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789379409233261:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.707223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:58.712825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789377086374307:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3946
2025-11-28T14:52:58.712003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789376595441457:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.714893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789379409233162:2049] [node 2] ICP22 created new session: [6:7577789379409233262:2049]
2025-11-28T14:52:58.704301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:58.709521Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789378012515670:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27151) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789378012515691:2129] held: no
2025-11-28T14:52:58.714894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789377086374300:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:58.705476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789378979995493:2049] [node 2] ICS09 handshake done sender: [3:7577789378979995477:2142] self: [3:7577789378979995480:0] peer: [2:7577789376517507938:0] socket: 265 qp: -1
2025-11-28T14:52:58.712023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789376595441457:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.705184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789376517507868:2048] [node 1] ICP22 created new session: [2:7577789376517507968:2048]
2025-11-28T14:52:58.715791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789377086374205:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.705516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789378979995493:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:58.705807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789376517507968:2048] [node 1] ICS09 handshake done sender: [2:7577789376517507950:2118] self: [2:7577789376517507953:0] peer: [1:7577789376595441455:0] socket: 279 qp: -1
2025-11-28T14:52:58.715326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:58.705730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:58.713512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789378012515671:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18573728ns) {HandleHandshakeStatus:329(18571920ns) {DropIncomingHandshake:442(168ns) DropOutgoingHandshake:457(277ns) SwitchToState:179(121ns) ProcessPendingSessionEvents:495(13657792ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13569949ns max# 1382028ns}}}
2025-11-28T14:52:58.705838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789376517507968:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.711314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789378979995371:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:58.715162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789376595441457:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:58.712828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789378979995463:2127] [node 6] ICP35 event processing took too much time PendingConnection:279(25585130ns) {HandleHandshakeStatus:329(25583333ns) {DropIncomingHandshake:442(184ns) DropOutgoingHandshake:457(511ns) SwitchToState:179(184ns) ProcessPendingSessionEvents:495(17353643ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17243457ns max# 1216521ns}}}
2025-11-28T14:52:58.714938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789378012515702:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-28T14:52:58.715408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789379409233262:2049] [node 2] ICS09 handshake done sender: [6:7577789379409233242:2118] self: [6:7577789379409233247:0] peer: [2:7577789376517507937:0] socket: 266 qp: -1
2025-11-28T14:52:58.715824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:58.707417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:58.715951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789379409233262:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:58.713525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789378979995461:2117] [node 5] ICP35 event processing took too much time PendingConnection:279(25392423ns) {HandleHandshakeStatus:329(25298385ns) {DropIncomingHandshake:442(389ns) DropOutgoingHandshake:457(292ns) SwitchToState:179(161ns) ProcessPendingSessionEvents:495(20453448ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19360597ns max# 1807819ns}}}
2025-11-28T14:52:58.715619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789376595441385:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(21498210ns) {HandleHandshakeStatus:329(21496026ns) {DropIncomingHandshake:442(699ns) DropOutgoingHandshake:457(372ns) SwitchToState:179(459ns) ProcessPendingSessionEvents:495(15020968ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14917966ns max# 1793335ns}}}
2025-11-28T14:52:58.715451Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789378979995462:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22231228ns) {HandleHandshakeStatus:329(21943035ns) {DropIncomingHandshake:442(449ns) DropOutgoingHandshake:457(401ns) SwitchToState:179(955ns) ProcessPendingSessionEvents:495(14813999ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14279483ns max# 1274688ns}}}
2025-11-28T14:52:58.716493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789378012515691:2129] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:58.709688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789376517507941:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:58.716148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789377086374205:2049] [node 2] ICP22 created new session: [4:7577789377086374308:2049]
2025-11-28T14:52:58.716937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789378012515596:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789378012515697:2131] held: no
2025-11-28T14:52:58.718673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789378979995496:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8695
2025-11-28T14:52:58.716431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789377086374308:2049] [node 2] ICS09 handshake done sender: [4:7577789377086374300:2122] self: [4:7577789377086374306:0] peer: [2:7577789376517507935:0] socket: 309 qp: -1
2025-11-28T14:52:58.709707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789376517507941:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:58.717111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789376595441386:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.716024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:58.710771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789376517507941:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:58.716452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789377086374308:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:58.717715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:58.716471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:58.717766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789376595441386:2049] [node 3] ICP22 created new session: [1:7577789376595441489:2049]
2025-11-28T14:52:58.717678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789378012515705:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-28T14:52:58.715009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789376517507934:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:58.718440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789379409233163:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(19568279ns) {HandleHandshakeStatus:329(19566379ns) {DropIncomingHandshake:442(472ns) DropOutgoingHandshake:457(5028ns) SwitchToState:179(358ns) ProcessPendingSessionEvents:495(13461428ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13362328ns max# 1612924ns}}}
2025-11-28T14:52:58.718309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789376595441449:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(17529903ns) {HandleHandshakeStatus:329(17528424ns) {DropIncomingHandshake:442(9548ns) DropOutgoingHandshake:457(184ns) SwitchToState:179(489ns) ProcessPendingSessionEvents:495(13966562ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12440931ns max# 1175139ns}}}
2025-11-28T14:52:58.718281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789378012515670:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.715373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:58.718662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789378012515598:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17994975ns) {HandleHandshakeStatus:329(17993110ns) {DropIncomingHandshake:442(1396ns) DropOutgoingHandshake:457(6550ns) SwitchToState:179(327ns) ProcessPendingSessionEvents:495(10862803ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10474478ns max# 1328060ns}}}
2025-11-28T14:52:58.718440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789376595441489:2049] [node 3] ICS09 handshake done sender: [1:7577789376595441457:2111] self: [1:7577789376595441452:0] peer: [3:7577789378979995495:0] socket: 305 qp: -1
2025-11-28T14:52:58.718828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:58.715459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789376517507869:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17715007ns) {HandleHandshakeStatus:329(17713559ns) {DropIncomingHandshake:442(164ns) DropOutgoingHandshake:457(216ns) SwitchToState:179(435ns) ProcessPendingSessionEvents:495(12963050ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12488770ns max# 1847228ns}}}
2025-11-28T14:52:58.718701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789376595441489:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:58.718902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789378979995491:2144] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.718876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789378012515670:2108] [node 4] ICP22 created new session: [5:7577789378012515706:2108]
2025-11-28T14:52:58.718740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:58.715856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789376517507934:2107] [node 4] ICP22 created new session: [2:7577789376517507971:2107]
2025-11-28T14:52:58.718834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789376595441385:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:58.719502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789378012515706:2108] [node 4] ICS09 handshake done sender: [5:7577789378012515691:2129] self: [5:7577789378012515700:0] peer: [4:7577789377086374274:0] socket: 300 qp: -1
2025-11-28T14:52:58.719330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789378979995462:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:58.716425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789376517507971:2107] [node 4] ICS09 handshake done sender: [2:7577789376517507941:2113] self: [2:7577789376517507935:0] peer: [4:7577789377086374306:0] socket: 273 qp: -1
2025-11-28T14:52:58.720185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789377086374206:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(22569220ns) {HandleHandshakeStatus:329(22567989ns) {DropIncomingHandshake:442(267ns) DropOutgoingHandshake:457(2588ns) SwitchToState:179(143ns) ProcessPendingSessionEvents:495(15965693ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15860713ns max# 1325168ns}}}
2025-11-28T14:52:58.716478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789376517507971:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:58.719820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789378012515671:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:58.716735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:58.716995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789376517507933:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18166409ns) {HandleHandshakeStatus:329(18164153ns) {DropIncomingHandshake:442(12005ns) DropOutgoingHandshake:457(188ns) SwitchToState:179(91ns) ProcessPendingSessionEvents:495(13420051ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13313123ns max# 1344438ns}}}
2025-11-28T14:52:58.717762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789376517507932:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19502528ns) {HandleHandshakeStatus:329(19500333ns) {DropIncomingHandshake:442(1073ns) DropOutgoingHandshake:457(609ns) SwitchToState:179(304ns) ProcessPendingSessionEvents:495(12205227ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12110476ns max# 1338976ns}}}
2025-11-28T14:52:58.719953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789378012515706:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:58.720166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:58.720810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789376517507932:2108] [node 5] ICP18 handshake (actor [2:7577789376517507964:2121]) is too old
2025-11-28T14:52:58.720769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789378012515697:2131] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:58.720826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789376517507869:2049] [node 3] ICP18 handshake (actor [2:7577789376517507956:2119]) is too old
2025-11-28T14:52:58.720165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789378979995371:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.721440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789378012515596:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:58.720575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:58.720623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789378979995371:2048] [node 1] ICP22 created new session: [3:7577789378979995497:2048]
2025-11-28T14:52:58.721521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:58.721922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789378012515596:2048] [node 1] ICP22 created new session: [5:7577789378012515708:2048]
2025-11-28T14:52:58.720661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789378979995497:2048] [node 1] ICS09 handshake done sender: [3:7577789378979995491:2144] self: [3:7577789378979995495:0] peer: [1:7577789376595441452:0] socket: 307 qp: -1
2025-11-28T14:52:58.720688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789378979995497:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.722050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789378012515708:2048] [node 1] ICS09 handshake done sender: [5:7577789378012515697:2131] self: [5:7577789378012515701:0] peer: [1:7577789376595441456:0] socket: 277 qp: -1
2025-11-28T14:52:58.720786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:58.722076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789378012515708:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:58.722122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:58.722418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789378979995371:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:58.771712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:58.771891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789376517507968:2048] [node 1] ICS07 socket disconnect 279 reason# EndOfStream
2025-11-28T14:52:58.771915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789376517507968:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.771991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.772037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789376517507968:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:58.772873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:58.773121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789378979995497:2048] [node 1] ICS07 socket disconnect 307 reason# EndOfStream
2025-11-28T14:52:58.773168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789378979995497:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.773267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.773328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789378979995497:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:58.774253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789378979995371:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.774011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789376517507868:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Socket error# Connection reset by peer state# SendExRequestHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
2025-11-28T14:52:58.774749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789378979995497:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.774041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789376517507968:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:58.774841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789378979995371:2048] [node 1] ICP30 unregister session Session# [3:7577789378979995497:2048] VirtualId# [3:7577789378979995495:0]
2025-11-28T14:52:58.775142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.775211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789378979995371:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.775235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.775383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789376517507868:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.775578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789376517507968:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.775619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789376517507868:2048] [node 1] ICP30 unregister session Session# [2:7577789376517507968:2048] VirtualId# [2:7577789376517507953:0]
2025-11-28T14:52:58.775765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.775806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789376517507868:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.775824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21632) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.775827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:58.775972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789377086374301:2048] [node 1] ICS01 socket: 272 reason# EndOfStream
2025-11-28T14:52:58.776005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789377086374204:2048] [node 1] ICP30 unregister session Session# [4:7577789377086374301:2048] VirtualId# [4:7577789377086374288:0]
2025-11-28T14:52:58.776035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789377086374301:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.776096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.776148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.776616Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:58.776762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789378012515708:2048] [node 1] ICS01 socket: 277 reason# EndOfStream
2025-11-28T14:52:58.776789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789378012515596:2048] [node 1] ICP30 unregister session Session# [5:7577789378012515708:2048] VirtualId# [5:7577789378012515701:0]
2025-11-28T14:52:58.776806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789378012515708:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.776853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.776890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.777401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:58.777536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789379409233261:2048] [node 1] ICS01 socket: 280 reason# EndOfStream
2025-11-28T14:52:58.777567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789379409233161:2048] [node 1] ICP30 unregister session Session# [6:7577789379409233261:2048] VirtualId# [6:7577789379409233248:0]
2025-11-28T14:52:58.777585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789379409233261:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.777631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:58.777675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:58.785447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789378979995371:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:58.840470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:58.840705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789378979995493:2049] [node 2] ICS07 socket disconnect 265 reason# EndOfStream
2025-11-28T14:52:58.840742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789378979995493:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.840858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:58.840923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789378979995493:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:58.841741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789378979995372:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.841898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789378979995493:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.841947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789378979995372:2049] [node 2] ICP30 unregister session Session# [3:7577789378979995493:2049] VirtualId# [3:7577789378979995480:0]
2025-11-28T14:52:58.842007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:58.842059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789378979995372:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.842080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.842554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:58.842704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789377086374308:2049] [node 2] ICS07 socket disconnect 309 reason# EndOfStream
2025-11-28T14:52:58.842725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789377086374308:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.842797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:58.842830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789377086374308:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:58.843369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789377086374205:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.843337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:58.843446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789377086374308:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.843470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789377086374205:2049] [node 2] ICP30 unregister session Session# [4:7577789377086374308:2049] VirtualId# [4:7577789377086374306:0]
2025-11-28T14:52:58.843495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:58.843519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789377086374205:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.843542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789378012515703:2049] [node 2] ICS07 socket disconnect 267 reason# EndOfStream
2025-11-28T14:52:58.843529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.843574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789378012515703:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.843633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:58.843674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789378012515703:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:58.844288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789378012515597:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.844227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:58.844377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789379409233262:2049] [node 2] ICS01 socket: 266 reason# EndOfStream
2025-11-28T14:52:58.844376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789378012515703:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.844403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789379409233162:2049] [node 2] ICP30 unregister session Session# [6:7577789379409233262:2049] VirtualId# [6:7577789379409233247:0]
2025-11-28T14:52:58.844408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789378012515597:2049] [node 2] ICP30 unregister session Session# [5:7577789378012515703:2049] VirtualId# [5:7577789378012515690:0]
2025-11-28T14:52:58.844442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:58.844419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789379409233262:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.844466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789378012515597:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.844478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.844469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:58.844496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:58.853018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789377086374205:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:58.854065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789378012515597:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:58.902381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:58.902543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789377086374298:2050] [node 3] ICS07 socket disconnect 244 reason# EndOfStream
2025-11-28T14:52:58.902569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789377086374298:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.902655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:58.902709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789377086374298:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:58.903173Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:58.903362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789377086374206:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63265) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.903399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789378012515696:2050] [node 3] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-28T14:52:58.903426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789377086374298:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.903438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789378012515696:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.903503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789377086374206:2050] [node 3] ICP30 unregister session Session# [4:7577789377086374298:2050] VirtualId# [4:7577789377086374289:0]
2025-11-28T14:52:58.903540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:58.903609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:58.903586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789378012515696:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:58.903696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789377086374206:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63265) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.903714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63265) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.903953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:58.904111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789379409233255:2050] [node 3] ICS07 socket disconnect 243 reason# EndOfStream
2025-11-28T14:52:58.904156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789379409233255:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.904246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:58.904525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789378012515598:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63265) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.904598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789378012515696:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.904294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789379409233255:2050] [node 3] ICS15 start handshake
2025-11-28T14:52:58.904638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789378012515598:2050] [node 3] ICP30 unregister session Session# [5:7577789378012515696:2050] VirtualId# [5:7577789378012515687:0]
2025-11-28T14:52:58.904787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:58.904826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789378012515598:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63265) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.904850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63265) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.905325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789379409233163:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63265) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:58.905384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789379409233255:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:58.905408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789379409233163:2050] [node 3] ICP30 unregister session Session# [6:7577789379409233255:2050] VirtualId# [6:7577789379409233245:0]
2025-11-28T14:52:58.905538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:58.905573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789379409233163:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63265) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.905588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63265) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:58.913475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789377086374206:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:58.914492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789378012515598:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:58.915478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789379409233163:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:52:58.952754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:58.953045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789378012515706:2108] [node 4] ICS01 socket: 300 reason# EndOfStream
2025-11-28T14:52:58.953106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789378012515670:2108] [node 4] ICP30 unregister session Session# [5:7577789378012515706:2108] VirtualId# [5:7577789378012515700:0]
2025-11-28T14:52:58.953147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789378012515706:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.953296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:58.953356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:58.953781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:58.953927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789379409233257:2108] [node 4] ICS01 socket: 250 reason# EndOfStream
2025-11-28T14:52:58.953955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789379409233224:2108] [node 4] ICP30 unregister session Session# [6:7577789379409233257:2108] VirtualId# [6:7577789379409233244:0]
2025-11-28T14:52:58.953975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789379409233257:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:58.954036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:58.954064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:59.008204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:59.008465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789379409233259:2107] [node 5] ICS07 socket disconnect 259 reason# EndOfStream
2025-11-28T14:52:59.008498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789379409233259:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.008593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:59.008637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789379409233259:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:59.009304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789379409233222:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.009378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789379409233259:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.009424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789379409233222:2107] [node 5] ICP30 unregister session Session# [6:7577789379409233259:2107] VirtualId# [6:7577789379409233246:0]
2025-11-28T14:52:59.009512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:59.009540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789379409233222:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.009553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.018912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789379409233222:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:59.207471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381667957847:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:59.207510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381667957848:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:59.226113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789382639826696:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.226174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789382639826697:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:59.245729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383821661365:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.245761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383821661366:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:59.266630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789383850785502:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.266667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789383850785503:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:59.266693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789383850785504:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:59.289489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789384781288705:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.289524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789384781288706:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:59.289554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789384781288707:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:59.305460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789383692128209:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.305496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789383692128210:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:59.305518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789383692128211:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:59.388584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789382639826759:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:59.388808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381667957910:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:59.389087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789382639826760:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:59.389332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383821661429:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:59.389727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383821661430:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:59.389903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789383692128273:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:59.389937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789382639826761:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:59.389911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789384781288768:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:59.389998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381667957911:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:59.390089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789383850785574:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:59.390213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383821661431:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:59.390298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789383692128274:2107] [node 5] ICP01 ready to work
2025-11-28T14:52:59.390360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789384781288769:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:59.390447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381667957912:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:59.393364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789383850785577:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.399645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381667957923:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43322
2025-11-28T14:52:59.399711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381667957924:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43322
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.401752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383821661442:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31833
2025-11-28T14:52:59.401819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383821661443:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31833
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.401963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383821661444:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31833
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.401988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381667957928:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43322
2025-11-28T14:52:59.402286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383821661445:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31833
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.404514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789383850785584:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57166
2025-11-28T14:52:59.404602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789383850785585:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57166
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.405203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789383850785588:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57166
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.405778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789382639826775:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-28T14:52:59.405859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789382639826776:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.406257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789382639826777:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.406627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789382639826775:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.406675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789382639826778:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.406944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789382639826779:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-28T14:52:59.406839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789383850785584:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.406782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957924:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.407067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789382639826777:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.407084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957923:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.407319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957928:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.407324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789382639826776:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.407401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789382639826696:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.407695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381667957929:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43322
2025-11-28T14:52:59.407723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383821661443:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.407965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383821661442:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.408099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957914:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.408120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381667957914:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.408319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789382639826778:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.408668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383821661445:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.408761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789383821661366:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.408839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383821661449:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31833
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.408882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957929:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.408961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381667957930:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43322
2025-11-28T14:52:59.408994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383821661444:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.409006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789382639826765:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.409030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789382639826765:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.409366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789383821661365:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.409445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381667957914:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:59.409824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957916:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.409842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381667957916:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.410403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381667957847:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.410442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.410536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381667957847:2048] [node 2] ICP22 created new session: [1:7577789381667957936:2048]
2025-11-28T14:52:59.410567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381667957936:2048] [node 2] ICS09 handshake done sender: [1:7577789381667957914:2110] self: [1:7577789381667957913:0] peer: [2:7577789382639826782:0] socket: 258 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.410588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381667957936:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:59.410612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.410689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789382639826784:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57877
2025-11-28T14:52:59.410712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789383850785585:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.410940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383821661454:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31833
2025-11-28T14:52:59.411182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789382639826765:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.411332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789383850785588:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.411413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789383850785503:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.411458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789382639826779:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.411496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789382639826697:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.411558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:59.411559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383821661449:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.411589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789382639826697:2049] [node 3] ICP22 created new session: [2:7577789382639826786:2049]
2025-11-28T14:52:59.411605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383821661442:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:59.411620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789382639826786:2049] [node 3] ICS09 handshake done sender: [2:7577789382639826765:2111] self: [2:7577789382639826764:0] peer: [3:7577789383821661451:0] socket: 247 qp: -1
2025-11-28T14:52:59.411645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789382639826786:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:59.411626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789383850785502:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.411663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.411850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789383850785504:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.411887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789383821661366:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.411895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957930:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.411936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.411966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383821661366:2049] [node 2] ICP22 created new session: [3:7577789383821661455:2049]
2025-11-28T14:52:59.411990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383821661455:2049] [node 2] ICS09 handshake done sender: [3:7577789383821661442:2116] self: [3:7577789383821661451:0] peer: [2:7577789382639826764:0] socket: 256 qp: -1
2025-11-28T14:52:59.412013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383821661455:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:59.412029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.412107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957918:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.412123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381667957918:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.412558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789382639826776:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.412940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381667957916:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.413101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789382639826767:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.413121Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789382639826767:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.413532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381667957848:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.413536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789384781288783:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-28T14:52:59.413556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789383850785595:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57166
2025-11-28T14:52:59.413566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:59.413591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381667957848:2049] [node 3] ICP22 created new session: [1:7577789381667957942:2049]
2025-11-28T14:52:59.413625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381667957942:2049] [node 3] ICS09 handshake done sender: [1:7577789381667957916:2111] self: [1:7577789381667957915:0] peer: [3:7577789383821661453:0] socket: 259 qp: -1
2025-11-28T14:52:59.413644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381667957942:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:59.413660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.414949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789382639826696:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.414946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383821661437:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.414967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789383821661437:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.415024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.415056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789382639826696:2048] [node 1] ICP22 created new session: [2:7577789382639826788:2048]
2025-11-28T14:52:59.415096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789382639826788:2048] [node 1] ICS09 handshake done sender: [2:7577789382639826776:2116] self: [2:7577789382639826782:0] peer: [1:7577789381667957913:0] socket: 264 qp: -1
2025-11-28T14:52:59.415119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789382639826788:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.415143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:59.415241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381667957918:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:59.415813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789383850785585:2124] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.415920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789382639826696:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:59.416123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789383850785502:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.416196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.416243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789383850785502:2048] [node 1] ICP22 created new session: [4:7577789383850785596:2048]
2025-11-28T14:52:59.416298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789383850785596:2048] [node 1] ICS09 handshake done sender: [4:7577789383850785585:2124] self: [4:7577789383850785593:0] peer: [1:7577789381667957917:0] socket: 269 qp: -1
2025-11-28T14:52:59.416332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789383850785596:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.416457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:59.417417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789383821661365:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789383821661444:2118] held: no
2025-11-28T14:52:59.417594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381667957910:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.417660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.417696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381667957910:2107] [node 4] ICP22 created new session: [1:7577789381667957944:2107]
2025-11-28T14:52:59.417749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381667957944:2107] [node 4] ICS09 handshake done sender: [1:7577789381667957918:2112] self: [1:7577789381667957917:0] peer: [4:7577789383850785593:0] socket: 262 qp: -1
2025-11-28T14:52:59.417781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381667957944:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:59.417811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.418018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383821661458:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31833
2025-11-28T14:52:59.418715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383821661437:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.419306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789384781288784:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.419363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789383821661429:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.419432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.419444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789384781288785:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-28T14:52:59.419467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383821661429:2107] [node 4] ICP22 created new session: [3:7577789383821661461:2107]
2025-11-28T14:52:59.419517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383821661461:2107] [node 4] ICS09 handshake done sender: [3:7577789383821661437:2113] self: [3:7577789383821661436:0] peer: [4:7577789383850785594:0] socket: 276 qp: -1
2025-11-28T14:52:59.419545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383821661461:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:59.419566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:59.419735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789382639826767:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.420238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789382639826759:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.420184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789383850785598:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57166
2025-11-28T14:52:59.420324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.420397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789382639826759:2107] [node 4] ICP22 created new session: [2:7577789382639826794:2107]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:59.420447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789382639826794:2107] [node 4] ICS09 handshake done sender: [2:7577789382639826767:2112] self: [2:7577789382639826766:0] peer: [4:7577789383850785592:0] socket: 248 qp: -1
2025-11-28T14:52:59.420494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383821661444:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.420519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789382639826794:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:59.420579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:59.420574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128288:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.421556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789384781288786:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-28T14:52:59.421643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789383821661365:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.421743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.421801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383821661365:2048] [node 1] ICP22 created new session: [3:7577789383821661464:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.421848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383821661464:2048] [node 1] ICS09 handshake done sender: [3:7577789383821661444:2118] self: [3:7577789383821661453:0] peer: [1:7577789381667957915:0] socket: 270 qp: -1
2025-11-28T14:52:59.421893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128289:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
2025-11-28T14:52:59.421893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383821661464:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.421927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.422671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789383850785588:2126] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.422768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789384781288783:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.422731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128290:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
2025-11-28T14:52:59.422867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789384781288705:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.422968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789383850785599:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57166
2025-11-28T14:52:59.423336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789383850785504:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789383850785588:2126] held: no
2025-11-28T14:52:59.423558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957920:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.423598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381667957920:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.424052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789384781288785:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.424095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789383850785504:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.424355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:59.424349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789384781288706:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.424450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789383850785504:2050] [node 3] ICP22 created new session: [4:7577789383850785601:2050]
2025-11-28T14:52:59.424497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789383850785601:2050] [node 3] ICS09 handshake done sender: [4:7577789383850785588:2126] self: [4:7577789383850785594:0] peer: [3:7577789383821661436:0] socket: 279 qp: -1
2025-11-28T14:52:59.424528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789383850785601:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:59.424596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.424807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789384781288784:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.424871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789384781288789:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-28T14:52:59.424924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789384781288707:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.425355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789382639826769:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.425385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789382639826769:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.426149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381667957920:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:59.426340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789383692128288:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.426509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789382639826769:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:59.426541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789383692128210:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.426634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381667957911:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.426657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789384781288783:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.427193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789383692128290:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.427190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.427220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383821661439:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.427244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381667957911:2108] [node 5] ICP22 created new session: [1:7577789381667957948:2108]
2025-11-28T14:52:59.427635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789383692128209:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.427247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789383821661439:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.427705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381667957948:2108] [node 5] ICS09 handshake done sender: [1:7577789381667957920:2113] self: [1:7577789381667957919:0] peer: [5:7577789384781288787:0] socket: 292 qp: -1
2025-11-28T14:52:59.427661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789383850785503:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789383850785584:2123] held: no
2025-11-28T14:52:59.428074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381667957948:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:59.428077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789382639826771:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.428122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.428102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789382639826771:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.428717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789382639826760:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.428789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.428818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789382639826760:2108] [node 5] ICP22 created new session: [2:7577789382639826797:2108]
2025-11-28T14:52:59.428801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789383692128289:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.428853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789382639826797:2108] [node 5] ICS09 handshake done sender: [2:7577789382639826769:2113] self: [2:7577789382639826768:0] peer: [5:7577789384781288788:0] socket: 296 qp: -1
2025-11-28T14:52:59.428840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383821661439:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:59.428877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789382639826797:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:59.428905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.428867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789384781288705:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.428881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789383692128211:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.428914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.428950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789384781288705:2048] [node 1] ICP22 created new session: [5:7577789384781288792:2048]
2025-11-28T14:52:59.428988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789384781288792:2048] [node 1] ICS09 handshake done sender: [5:7577789384781288783:2115] self: [5:7577789384781288787:0] peer: [1:7577789381667957919:0] socket: 283 qp: -1
2025-11-28T14:52:59.429024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789384781288792:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.429058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:59.429512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383821661441:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.429532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789383821661441:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.430026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789383850785584:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:59.430068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789383821661430:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.430243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381667957922:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.430287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.431113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383821661430:2108] [node 5] ICP22 created new session: [3:7577789383821661467:2108]
2025-11-28T14:52:59.430267Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381667957922:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.431148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383821661467:2108] [node 5] ICS09 handshake done sender: [3:7577789383821661439:2114] self: [3:7577789383821661438:0] peer: [5:7577789384781288790:0] socket: 277 qp: -1
2025-11-28T14:52:59.431172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383821661467:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:59.431198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.431435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789382639826771:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:59.431499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381667957922:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:59.431824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789382639826761:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.431916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:59.431943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128294:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
2025-11-28T14:52:59.431994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789382639826761:2109] [node 6] ICP22 created new session: [2:7577789382639826800:2109]
2025-11-28T14:52:59.432066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789382639826800:2109] [node 6] ICS09 handshake done sender: [2:7577789382639826771:2114] self: [2:7577789382639826770:0] peer: [6:7577789383692128291:0] socket: 299 qp: -1
2025-11-28T14:52:59.432103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789382639826800:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:59.432137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.432412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128295:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
2025-11-28T14:52:59.432647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381667957912:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.432697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:59.432865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381667957912:2109] [node 6] ICP22 created new session: [1:7577789381667957951:2109]
2025-11-28T14:52:59.433039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381667957951:2109] [node 6] ICS09 handshake done sender: [1:7577789381667957922:2114] self: [1:7577789381667957921:0] peer: [6:7577789383692128292:0] socket: 282 qp: -1
2025-11-28T14:52:59.433068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381667957951:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:59.433053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789383692128288:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:59.433115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:59.433237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789383692128290:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.433390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789383692128210:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.433270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789383850785503:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.433429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.433455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789383692128210:2049] [node 2] ICP22 created new session: [6:7577789383692128296:2049]
2025-11-28T14:52:59.433455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.433514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789383692128296:2049] [node 2] ICS09 handshake done sender: [6:7577789383692128288:2116] self: [6:7577789383692128291:0] peer: [2:7577789382639826770:0] socket: 300 qp: -1
2025-11-28T14:52:59.433542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789383692128296:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:59.433506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789383850785503:2049] [node 2] ICP22 created new session: [4:7577789383850785604:2049]
2025-11-28T14:52:59.433564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.433569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383821661441:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:59.433552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789383850785604:2049] [node 2] ICS09 handshake done sender: [4:7577789383850785584:2123] self: [4:7577789383850785592:0] peer: [2:7577789382639826766:0] socket: 265 qp: -1
2025-11-28T14:52:59.433580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789383850785604:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:59.433617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.434201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789383821661431:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.434291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:59.434331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383821661431:2109] [node 6] ICP22 created new session: [3:7577789383821661470:2109]
2025-11-28T14:52:59.434374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383821661470:2109] [node 6] ICS09 handshake done sender: [3:7577789383821661441:2115] self: [3:7577789383821661440:0] peer: [6:7577789383692128293:0] socket: 302 qp: -1
2025-11-28T14:52:59.434398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383821661470:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:59.434446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:59.435773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789383692128209:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.435854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.435882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789383692128209:2048] [node 1] ICP22 created new session: [6:7577789383692128300:2048]
2025-11-28T14:52:59.435914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789383692128300:2048] [node 1] ICS09 handshake done sender: [6:7577789383692128290:2118] self: [6:7577789383692128292:0] peer: [1:7577789381667957921:0] socket: 284 qp: -1
2025-11-28T14:52:59.435933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789383692128300:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.435975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.437298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789384781288791:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-28T14:52:59.437897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789384781288706:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789384781288785:2117] held: no
2025-11-28T14:52:59.438550Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789384781288707:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789384781288784:2116] held: no
2025-11-28T14:52:59.439465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789384781288786:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.439798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789384781288785:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.440449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789384781288794:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-28T14:52:59.440992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789384781288706:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.441039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.441083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789384781288706:2049] [node 2] ICP22 created new session: [5:7577789384781288797:2049]
2025-11-28T14:52:59.441130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789384781288797:2049] [node 2] ICS09 handshake done sender: [5:7577789384781288785:2117] self: [5:7577789384781288788:0] peer: [2:7577789382639826768:0] socket: 297 qp: -1
2025-11-28T14:52:59.441173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789384781288797:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:59.441208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.441152Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789383692128209:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:59.441581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789383692128211:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789383692128289:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.442305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128298:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.442772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789384781288784:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.442773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789383850785606:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57166
2025-11-28T14:52:59.442816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128299:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
2025-11-28T14:52:59.443047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789384781288768:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.443234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789384781288707:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.443273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:59.443307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789384781288707:2050] [node 3] ICP22 created new session: [5:7577789384781288800:2050]
2025-11-28T14:52:59.443365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789384781288800:2050] [node 3] ICS09 handshake done sender: [5:7577789384781288784:2116] self: [5:7577789384781288790:0] peer: [3:7577789383821661438:0] socket: 295 qp: -1
2025-11-28T14:52:59.443407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789384781288800:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:59.443442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.443472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789383692128299:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.443858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789383692128289:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.443892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789383850785607:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57166
2025-11-28T14:52:59.444102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789383692128211:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.444138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:59.444179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789383692128211:2050] [node 3] ICP22 created new session: [6:7577789383692128305:2050]
2025-11-28T14:52:59.444220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789383692128305:2050] [node 3] ICS09 handshake done sender: [6:7577789383692128289:2117] self: [6:7577789383692128293:0] peer: [3:7577789383821661440:0] socket: 303 qp: -1
2025-11-28T14:52:59.444246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789383692128305:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:59.444279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.444613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789383850785583:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.444641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789383850785583:2122] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.445483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789383850785583:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:59.445490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128304:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
2025-11-28T14:52:59.445793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789383850785574:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.445813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789383692128273:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.445868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.445914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789383850785574:2108] [node 5] ICP22 created new session: [4:7577789383850785609:2108]
2025-11-28T14:52:59.445962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789383850785609:2108] [node 5] ICS09 handshake done sender: [4:7577789383850785583:2122] self: [4:7577789383850785582:0] peer: [5:7577789384781288799:0] socket: 287 qp: -1
2025-11-28T14:52:59.445999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789383850785609:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:59.446030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.447729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789383850785587:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.447759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789383850785587:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.449766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789383850785606:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.449944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789383850785587:2125] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:59.449900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789384781288802:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.450043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789384781288803:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-28T14:52:59.450288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789383850785577:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.450356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:59.450444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789383850785577:2109] [node 6] ICP22 created new session: [4:7577789383850785612:2109]
2025-11-28T14:52:59.450401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789383692128304:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.450540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789384781288786:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:59.450516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789383850785612:2109] [node 6] ICS09 handshake done sender: [4:7577789383850785587:2125] self: [4:7577789383850785586:0] peer: [6:7577789383692128307:0] socket: 249 qp: -1
2025-11-28T14:52:59.450550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789383850785612:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.450574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:59.450830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128308:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
2025-11-28T14:52:59.450873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789384781288802:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.450926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789384781288768:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.450968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.451016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789384781288768:2108] [node 4] ICP22 created new session: [5:7577789384781288804:2108]
2025-11-28T14:52:59.451128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789384781288804:2108] [node 4] ICS09 handshake done sender: [5:7577789384781288786:2118] self: [5:7577789384781288799:0] peer: [4:7577789383850785582:0] socket: 301 qp: -1
2025-11-28T14:52:59.451180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789384781288804:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:59.451207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:59.451226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789383692128274:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.451404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789383692128299:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:59.451595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789383692128273:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.451640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.451682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789383692128273:2106] [node 4] ICP22 created new session: [6:7577789383692128310:2106]
2025-11-28T14:52:59.451731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789383692128310:2106] [node 4] ICS09 handshake done sender: [6:7577789383692128299:2122] self: [6:7577789383692128307:0] peer: [4:7577789383850785586:0] socket: 266 qp: -1
2025-11-28T14:52:59.451759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789383692128310:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:59.451784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:59.451883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789383850785607:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.451954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789383850785574:2108] [node 5] ICP18 handshake (actor [4:7577789383850785606:2130]) is too old
2025-11-28T14:52:59.453371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789383850785577:2109] [node 6] ICP18 handshake (actor [4:7577789383850785607:2131]) is too old
2025-11-28T14:52:59.454423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789384781288779:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.454459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789384781288779:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.455859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789384781288779:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.456069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789383692128312:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20778
2025-11-28T14:52:59.456166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789384781288769:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.456213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:59.456253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789384781288769:2109] [node 6] ICP22 created new session: [5:7577789384781288808:2109]
2025-11-28T14:52:59.456296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789384781288808:2109] [node 6] ICS09 handshake done sender: [5:7577789384781288779:2114] self: [5:7577789384781288778:0] peer: [6:7577789383692128309:0] socket: 260 qp: -1
2025-11-28T14:52:59.456373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789384781288808:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:59.456423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:59.456474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789383692128304:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:59.456649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789383692128274:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.456691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.456732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789383692128274:2107] [node 5] ICP22 created new session: [6:7577789383692128313:2107]
2025-11-28T14:52:59.456776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789383692128313:2107] [node 5] ICS09 handshake done sender: [6:7577789383692128304:2123] self: [6:7577789383692128309:0] peer: [5:7577789384781288778:0] socket: 267 qp: -1
2025-11-28T14:52:59.456814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789383692128313:2107] [node 5] ICS10 traffic start
2025-11-28T14:52:59.456847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.480590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:59.480782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789382639826788:2048] [node 1] ICS07 socket disconnect 264 reason# EndOfStream
2025-11-28T14:52:59.480816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789382639826788:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.480908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.480951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789382639826788:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:59.482064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:59.482215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789383821661464:2048] [node 1] ICS01 socket: 270 reason# EndOfStream
2025-11-28T14:52:59.482255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789383821661365:2048] [node 1] ICP30 unregister session Session# [3:7577789383821661464:2048] VirtualId# [3:7577789383821661453:0]
2025-11-28T14:52:59.482287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789383821661464:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.482358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.482411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.484593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789382639826696:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:59.484627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789382639826788:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:59.485823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:59.486082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789383850785596:2048] [node 1] ICS07 socket disconnect 269 reason# EndOfStream
2025-11-28T14:52:59.486128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789383850785596:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.486144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789382639826696:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.486206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.486185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789382639826788:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.486259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789383850785596:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:59.486217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789382639826696:2048] [node 1] ICP30 unregister session Session# [2:7577789382639826788:2048] VirtualId# [2:7577789382639826782:0]
2025-11-28T14:52:59.486320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.486359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789382639826696:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.486368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.487101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:59.487185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789383850785502:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.487238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789383850785596:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.487271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789383850785502:2048] [node 1] ICP30 unregister session Session# [4:7577789383850785596:2048] VirtualId# [4:7577789383850785593:0]
2025-11-28T14:52:59.487317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.487271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789384781288792:2048] [node 1] ICS07 socket disconnect 283 reason# EndOfStream
2025-11-28T14:52:59.487298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789384781288792:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.487376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789383850785502:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.487359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.487412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.487390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789384781288792:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:59.487943Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789384781288705:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.487977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789384781288792:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.488000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789384781288705:2048] [node 1] ICP30 unregister session Session# [5:7577789384781288792:2048] VirtualId# [5:7577789384781288787:0]
2025-11-28T14:52:59.488072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.488095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789384781288705:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.488107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.488203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:59.488360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789383692128300:2048] [node 1] ICS07 socket disconnect 284 reason# EndOfStream
2025-11-28T14:52:59.488406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789383692128300:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.488466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.488509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789383692128300:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:59.489074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789383692128209:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.489127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789383692128300:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.489159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789383692128209:2048] [node 1] ICP30 unregister session Session# [6:7577789383692128300:2048] VirtualId# [6:7577789383692128292:0]
2025-11-28T14:52:59.489240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.489273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789383692128209:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.489285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.497150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789383850785502:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:59.498151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789384781288705:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:59.499146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789383692128209:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:59.507628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:59.507800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789383821661455:2049] [node 2] ICS07 socket disconnect 256 reason# EndOfStream
2025-11-28T14:52:59.507826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789383821661455:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.507923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:59.507971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789383821661455:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:59.510754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789383821661366:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:59.510791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789383821661455:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:59.510767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:59.511016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789383850785604:2049] [node 2] ICS01 socket: 265 reason# EndOfStream
2025-11-28T14:52:59.511065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789383850785503:2049] [node 2] ICP30 unregister session Session# [4:7577789383850785604:2049] VirtualId# [4:7577789383850785592:0]
2025-11-28T14:52:59.511102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789383850785604:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.511179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:59.511233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:59.511402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789383821661366:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.511487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789383821661455:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.511514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789383821661366:2049] [node 2] ICP30 unregister session Session# [3:7577789383821661455:2049] VirtualId# [3:7577789383821661451:0]
2025-11-28T14:52:59.511607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:59.511646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789383821661366:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.511656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.512071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:59.512204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789384781288797:2049] [node 2] ICS01 socket: 297 reason# EndOfStream
2025-11-28T14:52:59.512241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789384781288706:2049] [node 2] ICP30 unregister session Session# [5:7577789384781288797:2049] VirtualId# [5:7577789384781288788:0]
2025-11-28T14:52:59.512267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789384781288797:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.512326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:59.512376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:59.513124Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:59.513253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789383692128296:2049] [node 2] ICS07 socket disconnect 300 reason# EndOfStream
2025-11-28T14:52:59.513275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789383692128296:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.513354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:59.513399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789383692128296:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:59.513989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789383692128210:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.514026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789383692128296:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.514051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789383692128210:2049] [node 2] ICP30 unregister session Session# [6:7577789383692128296:2049] VirtualId# [6:7577789383692128291:0]
2025-11-28T14:52:59.514127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:59.514150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789383692128210:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.514160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5602) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.521721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789383821661366:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:59.523715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789383692128210:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:59.552445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:52:59.552690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789383850785601:2050] [node 3] ICS01 socket: 279 reason# EndOfStream
2025-11-28T14:52:59.552746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789383850785504:2050] [node 3] ICP30 unregister session Session# [4:7577789383850785601:2050] VirtualId# [4:7577789383850785594:0]
2025-11-28T14:52:59.552792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789383850785601:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.552901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:59.552951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:59.553306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:52:59.553464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789384781288800:2050] [node 3] ICS01 socket: 295 reason# EndOfStream
2025-11-28T14:52:59.553505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789384781288707:2050] [node 3] ICP30 unregister session Session# [5:7577789384781288800:2050] VirtualId# [5:7577789384781288790:0]
2025-11-28T14:52:59.553543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789384781288800:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.553622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:59.553666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:59.554047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:52:59.554211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789383692128305:2050] [node 3] ICS01 socket: 303 reason# EndOfStream
2025-11-28T14:52:59.554241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789383692128211:2050] [node 3] ICP30 unregister session Session# [6:7577789383692128305:2050] VirtualId# [6:7577789383692128293:0]
2025-11-28T14:52:59.554264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789383692128305:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.554318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:52:59.554351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:52:59.578187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:52:59.578372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789384781288804:2108] [node 4] ICS07 socket disconnect 301 reason# EndOfStream
2025-11-28T14:52:59.578400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789384781288804:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.578484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:59.578548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789384781288804:2108] [node 4] ICS15 start handshake
2025-11-28T14:52:59.578939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:52:59.579125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789383692128310:2106] [node 4] ICS07 socket disconnect 266 reason# EndOfStream
2025-11-28T14:52:59.579156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789383692128310:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.579233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:52:59.579273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789384781288768:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20191) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.579294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789383692128310:2106] [node 4] ICS15 start handshake
2025-11-28T14:52:59.579328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789384781288804:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.579362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789384781288768:2108] [node 4] ICP30 unregister session Session# [5:7577789384781288804:2108] VirtualId# [5:7577789384781288799:0]
2025-11-28T14:52:59.579469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:59.579514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789384781288768:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.579526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.579916Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789383692128273:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20191) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.579954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789383692128310:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.579981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789383692128273:2106] [node 4] ICP30 unregister session Session# [6:7577789383692128310:2106] VirtualId# [6:7577789383692128307:0]
2025-11-28T14:52:59.580056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:52:59.580079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789383692128273:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.580090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.589302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789384781288768:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:52:59.590314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789383692128273:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:52:59.613062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:52:59.613258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789383692128313:2107] [node 5] ICS07 socket disconnect 267 reason# EndOfStream
2025-11-28T14:52:59.613288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789383692128313:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.613365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:52:59.613412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789383692128313:2107] [node 5] ICS15 start handshake
2025-11-28T14:52:59.614073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789383692128274:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19504) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.614114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789383692128313:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.614146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789383692128274:2107] [node 5] ICP30 unregister session Session# [6:7577789383692128313:2107] VirtualId# [6:7577789383692128309:0]
2025-11-28T14:52:59.614233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:52:59.614258Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789383692128274:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.614267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.624091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789383692128274:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:52:59.665495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381682451256:2048] [node 2] ICP01 ready to work
2025-11-28T14:52:59.665536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381682451257:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:59.681755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789384122330990:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.681777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789384122330991:2049] [node 3] ICP01 ready to work
2025-11-28T14:52:59.699887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383582416449:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.699930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383582416450:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:59.716672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789382358912976:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.716713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789382358912978:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:59.716721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789382358912977:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:59.732713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789382316327589:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:59.732718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789382316327588:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.732755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789382316327590:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:59.748690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789382907546117:2048] [node 1] ICP01 ready to work
2025-11-28T14:52:59.748694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789382907546118:2049] [node 2] ICP01 ready to work
2025-11-28T14:52:59.748722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789382907546119:2050] [node 3] ICP01 ready to work
2025-11-28T14:52:59.829012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789384122331063:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:59.831506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789382907546177:2105] [node 5] ICP01 ready to work
2025-11-28T14:52:59.831900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381682451319:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:59.832019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789382907546178:2106] [node 4] ICP01 ready to work
2025-11-28T14:52:59.832181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383582416540:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:59.832464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789382358913040:2108] [node 5] ICP01 ready to work
2025-11-28T14:52:59.831535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789384122331062:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:59.832338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383582416539:2136] [node 6] ICP01 ready to work
2025-11-28T14:52:59.832504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789382358913039:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:59.832776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381682451320:2107] [node 4] ICP01 ready to work
2025-11-28T14:52:59.832783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789382316327651:2108] [node 4] ICP01 ready to work
2025-11-28T14:52:59.832812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789381682451321:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:59.832960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789382316327652:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:59.833064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789384122331064:2109] [node 6] ICP01 ready to work
2025-11-28T14:52:59.833120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789383582416541:2116] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.841074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383582416550:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9747
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.841759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789384122331078:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4945
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.844702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789382316327661:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.846504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789382316327661:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.846796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789384122331079:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4945
2025-11-28T14:52:59.847124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789382316327589:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.847092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789384122331078:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.846943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383582416550:2141] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.847102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789383582416450:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.847869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546190:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
2025-11-28T14:52:59.848591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789384122331066:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.848625Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789384122331066:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.848666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789382358913054:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9459
2025-11-28T14:52:59.848739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789382907546190:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.849437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789382907546119:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.849693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789384122331070:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.849844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383582416557:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9747
2025-11-28T14:52:59.849721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789384122331070:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.849722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789382316327668:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-28T14:52:59.849852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383582416547:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.849732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789384122331080:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4945
2025-11-28T14:52:59.849876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789383582416547:2139] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.851111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789384122331081:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4945
2025-11-28T14:52:59.852465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383582416547:2139] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:59.854318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789384122331066:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.854588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789384122331070:2121] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.856145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789383582416539:2136] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.856873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789384122331080:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.856971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381682451330:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15902
2025-11-28T14:52:59.856192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:52:59.857154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789382358913054:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.857001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789382316327668:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.856921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789384122331081:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.856923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789384122331063:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.857155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381682451331:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15902
2025-11-28T14:52:59.856715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383582416539:2136] [node 6] ICP22 created new session: [3:7577789383582416560:2136]
2025-11-28T14:52:59.856963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.856970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383582416560:2136] [node 6] ICS09 handshake done sender: [3:7577789383582416547:2139] self: [3:7577789383582416546:0] peer: [6:7577789382907546196:0] socket: 259 qp: -1
2025-11-28T14:52:59.857467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546197:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
2025-11-28T14:52:59.856983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789384122330991:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.857181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383582416560:2136] [node 6] ICS10 traffic start
2025-11-28T14:52:59.857097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789384122331063:2108] [node 5] ICP22 created new session: [2:7577789384122331085:2108]
2025-11-28T14:52:59.857502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789382316327669:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-28T14:52:59.857217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:59.857548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789382316327651:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.857236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383582416559:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9747
2025-11-28T14:52:59.857236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789384122331085:2108] [node 5] ICS09 handshake done sender: [2:7577789384122331070:2121] self: [2:7577789384122331069:0] peer: [5:7577789382316327667:0] socket: 254 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.857328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:59.857336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789384122331085:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:59.857835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381682451335:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15902
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.857449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.857840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381682451334:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15902
2025-11-28T14:52:59.857890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789382358913048:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.857922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789382358913048:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.857472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789384122330991:2049] [node 3] ICP22 created new session: [2:7577789384122331086:2049]
2025-11-28T14:52:59.858035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789382907546190:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.858879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789382316327661:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:59.857524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789384122331086:2049] [node 3] ICS09 handshake done sender: [2:7577789384122331066:2119] self: [2:7577789384122331065:0] peer: [3:7577789383582416556:0] socket: 252 qp: -1
2025-11-28T14:52:59.858057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789381682451336:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15902
2025-11-28T14:52:59.858918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789382907546119:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.858160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451331:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.859038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:59.857565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789384122331086:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:59.859550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789382907546119:2050] [node 3] ICP22 created new session: [6:7577789382907546198:2050]
2025-11-28T14:52:59.859587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789382358913048:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:59.859575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383582416550:2141] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:59.859757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789382907546198:2050] [node 3] ICS09 handshake done sender: [6:7577789382907546190:2114] self: [6:7577789382907546196:0] peer: [3:7577789383582416546:0] socket: 265 qp: -1
2025-11-28T14:52:59.857611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.859762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789382316327589:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.858852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451334:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.859820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.859811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789382907546198:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:59.859862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789382316327589:2049] [node 2] ICP22 created new session: [5:7577789382316327673:2049]
2025-11-28T14:52:59.859855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.860409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789382358913040:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.859239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451335:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.859544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451336:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.860578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789382316327673:2049] [node 2] ICS09 handshake done sender: [5:7577789382316327661:2114] self: [5:7577789382316327667:0] peer: [2:7577789384122331069:0] socket: 260 qp: -1
2025-11-28T14:52:59.860631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.860612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789382316327673:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:59.861814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789382358913040:2108] [node 5] ICP22 created new session: [4:7577789382358913058:2108]
2025-11-28T14:52:59.861776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.862115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789382358913058:2108] [node 5] ICS09 handshake done sender: [4:7577789382358913048:2113] self: [4:7577789382358913047:0] peer: [5:7577789382316327670:0] socket: 273 qp: -1
2025-11-28T14:52:59.862356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789382358913058:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:59.862389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.862312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383582416557:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.862887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789383582416450:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.863007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789382316327671:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.863061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.863121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383582416450:2049] [node 2] ICP22 created new session: [3:7577789383582416564:2049]
2025-11-28T14:52:59.863136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789382358913056:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9459
2025-11-28T14:52:59.863185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383582416564:2049] [node 2] ICS09 handshake done sender: [3:7577789383582416550:2141] self: [3:7577789383582416556:0] peer: [2:7577789384122331065:0] socket: 255 qp: -1
2025-11-28T14:52:59.863216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383582416564:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:59.863246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.864468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789384122331079:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.864744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451330:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.864830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383582416563:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9747
2025-11-28T14:52:59.865856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789382316327672:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-28T14:52:59.866539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789382316327651:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62244) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789382316327668:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.867133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789382358913061:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9459
2025-11-28T14:52:59.867162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789384122331063:2108] [node 5] ICP18 handshake (actor [2:7577789384122331079:2125]) is too old
2025-11-28T14:52:59.867150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546201:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.867385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789384122331089:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4945
2025-11-28T14:52:59.867366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789382316327668:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:59.867466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789383582416539:2136] [node 6] ICP18 handshake (actor [3:7577789383582416557:2143]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.867573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789382316327671:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.867650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789382316327651:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.867671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789382316327675:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-28T14:52:59.867702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.867734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789382316327651:2108] [node 4] ICP22 created new session: [5:7577789382316327678:2108]
2025-11-28T14:52:59.867806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789382316327678:2108] [node 4] ICS09 handshake done sender: [5:7577789382316327668:2116] self: [5:7577789382316327670:0] peer: [4:7577789382358913047:0] socket: 280 qp: -1
2025-11-28T14:52:59.867833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789382316327678:2108] [node 4] ICS10 traffic start
2025-11-28T14:52:59.867862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:59.867952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789384122331089:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.868002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789382316327588:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.868176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789382316327675:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.868193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546202:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
2025-11-28T14:52:59.868586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383582416563:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.868809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451327:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.868836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381682451327:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.869331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789384122330990:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.870053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789382907546201:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.870306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789382907546118:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.870424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381682451327:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:59.870486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789384122331077:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.870455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789382358913061:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.870430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451323:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.870516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789384122331077:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.870454Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381682451323:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.870891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789382358912977:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.871110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789382316327682:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.871424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789382358913056:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.871115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789384122331072:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.871099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383582416566:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9747
2025-11-28T14:52:59.871564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789383582416449:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.871458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789382907546202:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.871522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381682451319:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.871925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789382358912978:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.871141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789384122331072:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.871576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.871581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789384122331077:2123] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:59.871620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381682451319:2108] [node 5] ICP22 created new session: [1:7577789381682451346:2108]
2025-11-28T14:52:59.872437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789382907546177:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.872648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789382316327671:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.872392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383582416567:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9747
2025-11-28T14:52:59.872668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789384122331064:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.872404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381682451346:2108] [node 5] ICS09 handshake done sender: [1:7577789381682451327:2112] self: [1:7577789381682451326:0] peer: [5:7577789382316327680:0] socket: 295 qp: -1
2025-11-28T14:52:59.872719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.872628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383582416566:2146] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.872559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381682451346:2108] [node 5] ICS10 traffic start
2025-11-28T14:52:59.872726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.872947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789384122331064:2109] [node 6] ICP22 created new session: [2:7577789384122331097:2109]
2025-11-28T14:52:59.872925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383582416549:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.873961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789382316327666:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.873094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789384122331097:2109] [node 6] ICS09 handshake done sender: [2:7577789384122331077:2123] self: [2:7577789384122331076:0] peer: [6:7577789382907546203:0] socket: 303 qp: -1
2025-11-28T14:52:59.872949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789383582416549:2140] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.873936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451325:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.873216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789384122331097:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:59.873803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546204:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
2025-11-28T14:52:59.873988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789382316327666:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.873968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381682451325:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.873241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:59.874236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383582416567:2147] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.875748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789382316327588:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.875962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.876118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789382316327588:2048] [node 1] ICP22 created new session: [5:7577789382316327684:2048]
2025-11-28T14:52:59.876062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381682451323:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:59.876327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789382358913064:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9459
2025-11-28T14:52:59.876280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383582416549:2140] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.876520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789382316327684:2048] [node 1] ICS09 handshake done sender: [5:7577789382316327671:2118] self: [5:7577789382316327680:0] peer: [1:7577789381682451326:0] socket: 296 qp: -1
2025-11-28T14:52:59.876565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789382316327684:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.876520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789382907546118:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789382907546201:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.876734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789383582416540:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.876675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789382316327683:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-28T14:52:59.876913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.876680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:59.876995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789384122331072:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:59.877000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789382358913065:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9459
2025-11-28T14:52:59.877128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789382316327666:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:59.876968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383582416540:2107] [node 4] ICP22 created new session: [3:7577789383582416573:2107]
2025-11-28T14:52:59.877005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383582416573:2107] [node 4] ICS09 handshake done sender: [3:7577789383582416549:2140] self: [3:7577789383582416548:0] peer: [4:7577789382358913063:0] socket: 298 qp: -1
2025-11-28T14:52:59.877582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789384122331062:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.877126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383582416573:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:59.877628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.877158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:59.877753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789384122331062:2107] [node 4] ICP22 created new session: [2:7577789384122331100:2107]
2025-11-28T14:52:59.877881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789382358913061:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:52:59.877725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381682451256:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.877985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789384122331100:2107] [node 4] ICS09 handshake done sender: [2:7577789384122331072:2122] self: [2:7577789384122331071:0] peer: [4:7577789382358913062:0] socket: 258 qp: -1
2025-11-28T14:52:59.878018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789384122331100:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:59.878021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.878059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:59.878093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789382316327652:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.878109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381682451256:2048] [node 2] ICP22 created new session: [1:7577789381682451351:2048]
2025-11-28T14:52:59.878534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789382907546201:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.878528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381682451351:2048] [node 2] ICS09 handshake done sender: [1:7577789381682451323:2110] self: [1:7577789381682451322:0] peer: [2:7577789384122331094:0] socket: 293 qp: -1
2025-11-28T14:52:59.878515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:59.878550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789382316327652:2109] [node 6] ICP22 created new session: [5:7577789382316327687:2109]
2025-11-28T14:52:59.878716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381682451351:2048] [node 2] ICS10 traffic start
2025-11-28T14:52:59.878754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.878873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789382316327687:2109] [node 6] ICS09 handshake done sender: [5:7577789382316327666:2115] self: [5:7577789382316327665:0] peer: [6:7577789382907546205:0] socket: 267 qp: -1
2025-11-28T14:52:59.878781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546207:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
2025-11-28T14:52:59.879035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789382316327687:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:59.878994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789382907546118:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.879808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:59.879030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.879893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789383582416575:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9747
2025-11-28T14:52:59.879831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789382907546118:2049] [node 2] ICP22 created new session: [6:7577789382907546208:2049]
2025-11-28T14:52:59.880156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789382358913056:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.879881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789382907546208:2049] [node 2] ICS09 handshake done sender: [6:7577789382907546201:2117] self: [6:7577789382907546203:0] peer: [2:7577789384122331076:0] socket: 305 qp: -1
2025-11-28T14:52:59.880193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789382358912978:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789382358913056:2116] held: no
2025-11-28T14:52:59.879913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789382907546208:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:59.880149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.880269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789382316327588:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:52:59.880672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789382358912977:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.880666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789382907546202:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:52:59.880725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:52:59.880745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789382358912978:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.880772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.880782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789382358912977:2049] [node 2] ICP22 created new session: [4:7577789382358913068:2049]
2025-11-28T14:52:59.880837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789382358912978:2050] [node 3] ICP22 created new session: [4:7577789382358913069:2050]
2025-11-28T14:52:59.881039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789382358913069:2050] [node 3] ICS09 handshake done sender: [4:7577789382358913056:2116] self: [4:7577789382358913063:0] peer: [3:7577789383582416548:0] socket: 299 qp: -1
2025-11-28T14:52:59.881343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381682451325:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.881152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789382358913068:2049] [node 2] ICS09 handshake done sender: [4:7577789382358913061:2117] self: [4:7577789382358913062:0] peer: [2:7577789384122331071:0] socket: 304 qp: -1
2025-11-28T14:52:59.881595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789384122331099:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4945
2025-11-28T14:52:59.881179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789382358913069:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:59.881190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789382358913068:2049] [node 2] ICS10 traffic start
2025-11-28T14:52:59.881853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789382316327683:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.882009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383582416563:2145] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.881248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.881297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:52:59.882948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789382907546177:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.882892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789383582416449:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.882993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.883188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381682451257:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.883183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.883240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:59.883034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789382907546177:2105] [node 5] ICP22 created new session: [6:7577789382907546210:2105]
2025-11-28T14:52:59.883210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383582416449:2048] [node 1] ICP22 created new session: [3:7577789383582416576:2048]
2025-11-28T14:52:59.883200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789382907546210:2105] [node 5] ICS09 handshake done sender: [6:7577789382907546202:2118] self: [6:7577789382907546205:0] peer: [5:7577789382316327665:0] socket: 307 qp: -1
2025-11-28T14:52:59.883378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381682451257:2049] [node 3] ICP22 created new session: [1:7577789381682451354:2049]
2025-11-28T14:52:59.883420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383582416576:2048] [node 1] ICS09 handshake done sender: [3:7577789383582416563:2145] self: [3:7577789383582416569:0] peer: [1:7577789381682451324:0] socket: 297 qp: -1
2025-11-28T14:52:59.883478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381682451354:2049] [node 3] ICS09 handshake done sender: [1:7577789381682451325:2111] self: [1:7577789381682451324:0] peer: [3:7577789383582416569:0] socket: 294 qp: -1
2025-11-28T14:52:59.883443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383582416576:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.883279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789382907546210:2105] [node 5] ICS10 traffic start
2025-11-28T14:52:59.883514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381682451354:2049] [node 3] ICS10 traffic start
2025-11-28T14:52:59.884152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.883362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.883465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:59.883559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789384122330990:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7742) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789384122331089:2128] held: no
2025-11-28T14:52:59.884250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789384122331089:2128] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.884849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789382316327590:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.885414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789384122330990:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.885290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789383582416555:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.885323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789383582416555:2142] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.886119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.886222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789384122330990:2048] [node 1] ICP22 created new session: [2:7577789384122331102:2048]
2025-11-28T14:52:59.886280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789384122331102:2048] [node 1] ICS09 handshake done sender: [2:7577789384122331089:2128] self: [2:7577789384122331094:0] peer: [1:7577789381682451322:0] socket: 261 qp: -1
2025-11-28T14:52:59.887184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789384122331102:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.887223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:59.888525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789383582416555:2142] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.889208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789382907546177:2105] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.889288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789382316327690:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-28T14:52:59.889423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546212:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
2025-11-28T14:52:59.889424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789382358913066:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9459
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.889541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789383582416541:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.889594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789382358913067:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9459
2025-11-28T14:52:59.889597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:52:59.889729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789383582416541:2116] [node 5] ICP22 created new session: [3:7577789383582416579:2116]
2025-11-28T14:52:59.889767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789382316327683:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:52:59.889766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789383582416579:2116] [node 5] ICS09 handshake done sender: [3:7577789383582416555:2142] self: [3:7577789383582416554:0] peer: [5:7577789382316327689:0] socket: 311 qp: -1
2025-11-28T14:52:59.889792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789383582416579:2116] [node 5] ICS10 traffic start
2025-11-28T14:52:59.889921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789382358913067:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.889916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789382316327590:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.889955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:52:59.889822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:52:59.890028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789382316327590:2050] [node 3] ICP22 created new session: [5:7577789382316327691:2050]
2025-11-28T14:52:59.890065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789382316327691:2050] [node 3] ICS09 handshake done sender: [5:7577789382316327683:2122] self: [5:7577789382316327689:0] peer: [3:7577789383582416554:0] socket: 313 qp: -1
2025-11-28T14:52:59.890084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789382316327691:2050] [node 3] ICS10 traffic start
2025-11-28T14:52:59.890224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:52:59.890708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789382358913066:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.890826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789382358912976:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.890951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546213:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
2025-11-28T14:52:59.891141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451329:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.891182Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381682451329:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.892762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381682451329:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:59.892823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789382907546212:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.892832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789382907546213:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.893010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381682451320:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.892912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789382907546178:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.892939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789382907546117:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:52:59.893135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.893024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789382358913075:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9459
2025-11-28T14:52:59.893237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381682451320:2107] [node 4] ICP22 created new session: [1:7577789381682451357:2107]
2025-11-28T14:52:59.893094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789382358913050:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.893350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381682451357:2107] [node 4] ICS09 handshake done sender: [1:7577789381682451329:2113] self: [1:7577789381682451328:0] peer: [4:7577789382358913074:0] socket: 318 qp: -1
2025-11-28T14:52:59.893114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789382358913050:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.893389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381682451357:2107] [node 4] ICS10 traffic start
2025-11-28T14:52:59.893432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:59.893898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789382358913066:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.894094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789381682451333:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:52:59.894122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789381682451333:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:52:59.894513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789382358912976:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7742) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789382358913066:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.895617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789381682451333:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:52:59.895618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546216:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
2025-11-28T14:52:59.895694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789382358912976:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.895703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789382358913050:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:52:59.895725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.895755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789382358912976:2048] [node 1] ICP22 created new session: [4:7577789382358913077:2048]
2025-11-28T14:52:59.895834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789382358913077:2048] [node 1] ICS09 handshake done sender: [4:7577789382358913066:2120] self: [4:7577789382358913074:0] peer: [1:7577789381682451328:0] socket: 320 qp: -1
2025-11-28T14:52:59.895899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789382358913077:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.895890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789382907546217:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25130
2025-11-28T14:52:59.895946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:59.896155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789381682451321:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.896288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:59.896297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789382358913039:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:52:59.896346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789381682451321:2109] [node 6] ICP22 created new session: [1:7577789381682451360:2109]
2025-11-28T14:52:59.896374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:52:59.896385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789382907546213:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:52:59.896405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789382907546212:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:52:59.896416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789382358913039:2109] [node 6] ICP22 created new session: [4:7577789382358913079:2109]
2025-11-28T14:52:59.896454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789382358913079:2109] [node 6] ICS09 handshake done sender: [4:7577789382358913050:2114] self: [4:7577789382358913049:0] peer: [6:7577789382907546214:0] socket: 266 qp: -1
2025-11-28T14:52:59.896480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789382358913079:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:59.896428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789381682451360:2109] [node 6] ICS09 handshake done sender: [1:7577789381682451333:2116] self: [1:7577789381682451332:0] peer: [6:7577789382907546215:0] socket: 253 qp: -1
2025-11-28T14:52:59.896583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:59.896671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789382907546178:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.896470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789381682451360:2109] [node 6] ICS10 traffic start
2025-11-28T14:52:59.896783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789382907546117:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:52:59.896665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:52:59.896823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:52:59.896830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:52:59.896909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789382907546117:2048] [node 1] ICP22 created new session: [6:7577789382907546219:2048]
2025-11-28T14:52:59.896935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789382907546219:2048] [node 1] ICS09 handshake done sender: [6:7577789382907546212:2121] self: [6:7577789382907546215:0] peer: [1:7577789381682451332:0] socket: 257 qp: -1
2025-11-28T14:52:59.896947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789382907546178:2106] [node 4] ICP22 created new session: [6:7577789382907546218:2106]
2025-11-28T14:52:59.896955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789382907546219:2048] [node 1] ICS10 traffic start
2025-11-28T14:52:59.896975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:52:59.897003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789382907546218:2106] [node 4] ICS09 handshake done sender: [6:7577789382907546213:2122] self: [6:7577789382907546214:0] peer: [4:7577789382358913049:0] socket: 269 qp: -1
2025-11-28T14:52:59.897038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789382907546218:2106] [node 4] ICS10 traffic start
2025-11-28T14:52:59.897080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:52:59.931608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:52:59.931764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789384122331102:2048] [node 1] ICS01 socket: 261 reason# EndOfStream
2025-11-28T14:52:59.931794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789384122330990:2048] [node 1] ICP30 unregister session Session# [2:7577789384122331102:2048] VirtualId# [2:7577789384122331094:0]
2025-11-28T14:52:59.931820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789384122331102:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.931896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.931941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.932626Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:52:59.932761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789383582416576:2048] [node 1] ICS01 socket: 297 reason# EndOfStream
2025-11-28T14:52:59.932801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789383582416449:2048] [node 1] ICP30 unregister session Session# [3:7577789383582416576:2048] VirtualId# [3:7577789383582416569:0]
2025-11-28T14:52:59.932824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789383582416576:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.932898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.932951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.934423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:52:59.934605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789382358913077:2048] [node 1] ICS01 socket: 320 reason# EndOfStream
2025-11-28T14:52:59.934641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789382358912976:2048] [node 1] ICP30 unregister session Session# [4:7577789382358913077:2048] VirtualId# [4:7577789382358913074:0]
2025-11-28T14:52:59.934675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789382358913077:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.934767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.934810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.935208Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:52:59.935343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789382316327684:2048] [node 1] ICS07 socket disconnect 296 reason# EndOfStream
2025-11-28T14:52:59.935367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789382316327684:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.935437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.935483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789382316327684:2048] [node 1] ICS15 start handshake
2025-11-28T14:52:59.935941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:52:59.936067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789382316327588:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7742) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.936106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789382316327684:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.936084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789382907546219:2048] [node 1] ICS01 socket: 257 reason# EndOfStream
2025-11-28T14:52:59.936168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789382316327588:2048] [node 1] ICP30 unregister session Session# [5:7577789382316327684:2048] VirtualId# [5:7577789382316327680:0]
2025-11-28T14:52:59.936112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789382907546117:2048] [node 1] ICP30 unregister session Session# [6:7577789382907546219:2048] VirtualId# [6:7577789382907546215:0]
2025-11-28T14:52:59.936196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789382907546219:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.936258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:52:59.936304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.936301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:52:59.936356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789382316327588:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.936391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.946647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789382316327588:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:52:59.971645Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:52:59.971809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789383582416564:2049] [node 2] ICS07 socket disconnect 255 reason# EndOfStream
2025-11-28T14:52:59.971848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789383582416564:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.971920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:59.971959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789383582416564:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:59.974344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789383582416450:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:52:59.974393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789383582416564:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:59.974455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:52:59.974605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789382358913068:2049] [node 2] ICS07 socket disconnect 304 reason# EndOfStream
2025-11-28T14:52:59.974629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789382358913068:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.974685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:59.974719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789382358913068:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:59.975257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789383582416450:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.975311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789383582416564:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.975347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789383582416450:2049] [node 2] ICP30 unregister session Session# [3:7577789383582416564:2049] VirtualId# [3:7577789383582416556:0]
2025-11-28T14:52:59.975438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:59.975472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789383582416450:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.975489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.975693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:52:59.975629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789382358912977:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.975661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789382358913068:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.975829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789382358912977:2049] [node 2] ICP30 unregister session Session# [4:7577789382358913068:2049] VirtualId# [4:7577789382358913062:0]
2025-11-28T14:52:59.975960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789382316327673:2049] [node 2] ICS07 socket disconnect 260 reason# EndOfStream
2025-11-28T14:52:59.975987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:59.976007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789382316327673:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.976011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789382358912977:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.976020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.976097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:59.976166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789382316327673:2049] [node 2] ICS15 start handshake
2025-11-28T14:52:59.976598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:52:59.976769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789382907546208:2049] [node 2] ICS01 socket: 305 reason# EndOfStream
2025-11-28T14:52:59.976812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789382907546118:2049] [node 2] ICP30 unregister session Session# [6:7577789382907546208:2049] VirtualId# [6:7577789382907546203:0]
2025-11-28T14:52:59.976948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789382316327589:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:52:59.976853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789382907546208:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:52:59.977028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:52:59.977037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789382316327673:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:52:59.977088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789382316327589:2049] [node 2] ICP30 unregister session Session# [5:7577789382316327673:2049] VirtualId# [5:7577789382316327667:0]
2025-11-28T14:52:59.977092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:59.977197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:52:59.977232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789382316327589:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.977250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20755) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:52:59.985545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789382358912977:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:52:59.986581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789382316327589:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:00.023432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:00.023615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789382358913069:2050] [node 3] ICS01 socket: 299 reason# EndOfStream
2025-11-28T14:53:00.023647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789382358912978:2050] [node 3] ICP30 unregister session Session# [4:7577789382358913069:2050] VirtualId# [4:7577789382358913063:0]
2025-11-28T14:53:00.023673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789382358913069:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.023756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:00.023824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:00.024350Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:00.024585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789382316327691:2050] [node 3] ICS07 socket disconnect 313 reason# EndOfStream
2025-11-28T14:53:00.024628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789382316327691:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.024727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:00.024779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789382316327691:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:00.025039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:00.025213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789382907546198:2050] [node 3] ICS07 socket disconnect 265 reason# EndOfStream
2025-11-28T14:53:00.025247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789382907546198:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.025321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:00.025375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789382907546198:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:00.025586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789382316327590:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4902) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.025685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789382316327691:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.025732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789382316327590:2050] [node 3] ICP30 unregister session Session# [5:7577789382316327691:2050] VirtualId# [5:7577789382316327689:0]
2025-11-28T14:53:00.025848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:00.025888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789382316327590:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4902) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.025933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4902) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.026021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789382907546119:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4902) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.026069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789382907546198:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.026182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789382907546119:2050] [node 3] ICP30 unregister session Session# [6:7577789382907546198:2050] VirtualId# [6:7577789382907546196:0]
2025-11-28T14:53:00.026302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:00.026336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789382907546119:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4902) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.026351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4902) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.035768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789382316327590:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:00.035765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789382907546119:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:00.066644Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:00.066801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789382316327678:2108] [node 4] ICS01 socket: 280 reason# EndOfStream
2025-11-28T14:53:00.066839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789382316327651:2108] [node 4] ICP30 unregister session Session# [5:7577789382316327678:2108] VirtualId# [5:7577789382316327670:0]
2025-11-28T14:53:00.066864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789382316327678:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.066949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:00.067001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:00.067429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:00.067597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789382907546218:2106] [node 4] ICS07 socket disconnect 269 reason# EndOfStream
2025-11-28T14:53:00.067631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789382907546218:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.067726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:00.067784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789382907546218:2106] [node 4] ICS15 start handshake
2025-11-28T14:53:00.068415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789382907546178:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62244) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.068509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789382907546218:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.068553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789382907546178:2106] [node 4] ICP30 unregister session Session# [6:7577789382907546218:2106] VirtualId# [6:7577789382907546214:0]
2025-11-28T14:53:00.068638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:00.068665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789382907546178:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62244) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.068675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62244) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.078749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789382907546178:2106] [node 4] ICP33 wake up from error state
2025-11-28T14:53:00.131115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:00.131293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789382907546210:2105] [node 5] ICS07 socket disconnect 307 reason# EndOfStream
2025-11-28T14:53:00.131334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789382907546210:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.131456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:00.131520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789382907546210:2105] [node 5] ICS15 start handshake
2025-11-28T14:53:00.132169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789382907546177:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30141) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.132254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789382907546210:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.132309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789382907546177:2105] [node 5] ICP30 unregister session Session# [6:7577789382907546210:2105] VirtualId# [6:7577789382907546205:0]
2025-11-28T14:53:00.132430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:53:00.132467Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789382907546177:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30141) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.132482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30141) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:00.187972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789385968988291:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:00.188048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789385968988292:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:00.205073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789388791543272:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.205102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789388791543273:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:00.222236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789385559684004:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.222270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789385559684005:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:00.239336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789388810259324:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:00.239336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789388810259323:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.239359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789388810259325:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:00.256134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789388857653812:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.256148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789388857653813:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:00.256176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789388857653814:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:00.273142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789389193872788:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:00.273142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789389193872787:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.273161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789389193872789:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:00.357829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789388857653875:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:00.357467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789385968988355:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:00.358060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789389193872850:2106] [node 5] ICP01 ready to work
2025-11-28T14:53:00.357944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789385968988356:2110] [node 6] ICP01 ready to work
2025-11-28T14:53:00.358166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789385559684068:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:00.358293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789388810259387:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:00.358030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789385968988357:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:00.358625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789388791543336:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:00.358301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789385559684067:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:00.358868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789388791543335:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:00.358392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789385559684069:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:00.358949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789388857653876:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:00.359054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789388810259388:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:00.359247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789388791543337:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:00.359058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789389193872851:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.371993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789388791543344:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48660
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.372275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789385968988364:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25471
2025-11-28T14:53:00.372635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789385559684076:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46128
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.373132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789385559684081:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46128
2025-11-28T14:53:00.373205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988364:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.373689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789385559684076:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.373924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789385968988369:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25471
2025-11-28T14:53:00.373722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789388791543344:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.373745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789385559684085:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46128
2025-11-28T14:53:00.375515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789385968988370:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25471
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.375976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789388810259402:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58461
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.376388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789388791543345:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48660
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.376740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789385968988374:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25471
2025-11-28T14:53:00.376583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789388810259403:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58461
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.378215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789388791543353:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48660
2025-11-28T14:53:00.378225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789388791543355:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48660
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.378720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988369:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.378891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789388857653890:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-28T14:53:00.378895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789385559684087:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46128
2025-11-28T14:53:00.378402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789388791543354:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48660
2025-11-28T14:53:00.378935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789385968988375:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25471
2025-11-28T14:53:00.378724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789388791543345:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.379254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789385559684081:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.379261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789385559684086:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46128
2025-11-28T14:53:00.379450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789388791543353:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.379959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789388791543354:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.379527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789385559684085:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.379846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789388810259402:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.380210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988370:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.380782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789388810259325:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.381061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789385559684075:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.381089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789385559684075:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.381171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988374:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.381170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789388791543355:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.381977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789388810259403:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.382329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789385559684086:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.382863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789388791543272:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.382844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789385559684004:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.382963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789388810259323:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.383002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789385559684075:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:00.383237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988361:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.383265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789385968988361:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.383325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789385559684087:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.384434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789388791543341:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.384465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789388791543341:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.384104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789388857653890:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.383331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988362:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.384096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789385559684005:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.383364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789385968988362:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.385780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789388810259406:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58461
2025-11-28T14:53:00.384523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789388857653813:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.386035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789388791543343:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.383507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988363:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.385556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789385559684067:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.383522Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789385968988363:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.386060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789388791543343:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.384597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988375:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.386060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:00.386152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789388791543341:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.385744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789385968988363:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:00.386801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789385559684067:2107] [node 4] ICP22 created new session: [3:7577789385559684094:2107]
2025-11-28T14:53:00.387045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789388791543273:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.386969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789385559684094:2107] [node 4] ICS09 handshake done sender: [3:7577789385559684075:2112] self: [3:7577789385559684073:0] peer: [4:7577789388810259404:0] socket: 281 qp: -1
2025-11-28T14:53:00.386007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789385968988361:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:00.387210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:00.387077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789385968988362:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:00.386997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789385559684094:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:00.387332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789388810259325:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12468) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789388810259402:2116] held: no
2025-11-28T14:53:00.387634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789388791543362:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48660
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.387520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789388810259402:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:00.387672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789388791543273:2049] [node 3] ICP22 created new session: [2:7577789388791543364:2049]
2025-11-28T14:53:00.387622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:00.390166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789388810259407:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58461
2025-11-28T14:53:00.387525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789385968988355:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.387706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789388791543343:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.387871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789388791543364:2049] [node 3] ICS09 handshake done sender: [2:7577789388791543341:2111] self: [2:7577789388791543340:0] peer: [3:7577789385559684093:0] socket: 292 qp: -1
2025-11-28T14:53:00.388434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:00.390856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789388810259403:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.388094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789388791543364:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:00.388467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789385968988291:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.390925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789388857653892:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-28T14:53:00.389059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789385968988355:2107] [node 4] ICP22 created new session: [1:7577789385968988384:2107]
2025-11-28T14:53:00.391248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872865:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
2025-11-28T14:53:00.388389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:00.392032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789385559684097:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46128
2025-11-28T14:53:00.390366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789388791543336:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.389686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:00.392950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789388810259325:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.390572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:00.389688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789385968988384:2107] [node 4] ICS09 handshake done sender: [1:7577789385968988363:2113] self: [1:7577789385968988359:0] peer: [4:7577789388810259405:0] socket: 284 qp: -1
2025-11-28T14:53:00.390821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789388791543336:2108] [node 5] ICP22 created new session: [2:7577789388791543367:2108]
2025-11-28T14:53:00.393604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789385559684098:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46128
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.392986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:00.389717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789385968988384:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:00.393859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789388857653890:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:00.390876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789388791543355:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:00.393167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789388810259325:2050] [node 3] ICP22 created new session: [4:7577789388810259409:2050]
2025-11-28T14:53:00.394190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872866:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
2025-11-28T14:53:00.390923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789388791543367:2108] [node 5] ICS09 handshake done sender: [2:7577789388791543343:2112] self: [2:7577789388791543339:0] peer: [5:7577789388857653891:0] socket: 293 qp: -1
No RDMA context found for GID: 2025-11-28T14:53:00.394019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789388857653813:2049] [node 2] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:53:00.393221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789388810259409:2050] [node 3] ICS09 handshake done sender: [4:7577789388810259402:2116] self: [4:7577789388810259404:0] peer: [3:7577789385559684073:0] socket: 283 qp: -1
2025-11-28T14:53:00.391096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789388791543367:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:00.389752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:00.394150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:00.389974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789385968988291:2048] [node 2] ICP22 created new session: [1:7577789385968988385:2048]
2025-11-28T14:53:00.394477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789385559684005:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5310) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789385559684087:2119] held: no
2025-11-28T14:53:00.393586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789388810259409:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:00.391121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:00.394183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789388857653813:2049] [node 2] ICP22 created new session: [5:7577789388857653893:2049]
2025-11-28T14:53:00.390792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789385968988385:2048] [node 2] ICS09 handshake done sender: [1:7577789385968988361:2111] self: [1:7577789385968988360:0] peer: [2:7577789388791543360:0] socket: 279 qp: -1
2025-11-28T14:53:00.394401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789388857653893:2049] [node 2] ICS09 handshake done sender: [5:7577789388857653890:2115] self: [5:7577789388857653891:0] peer: [2:7577789388791543339:0] socket: 296 qp: -1
2025-11-28T14:53:00.393617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:00.394712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789389193872865:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.394736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872867:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
2025-11-28T14:53:00.394433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789388857653893:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:00.393820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789388810259323:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.390818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789385968988385:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:00.390887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:00.393863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:00.394456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:00.394055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789388810259323:2048] [node 1] ICP22 created new session: [4:7577789388810259411:2048]
2025-11-28T14:53:00.392068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789385968988292:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.395169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789385559684086:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:00.395019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789389193872789:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.394409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789388810259411:2048] [node 1] ICS09 handshake done sender: [4:7577789388810259403:2117] self: [4:7577789388810259405:0] peer: [1:7577789385968988359:0] socket: 287 qp: -1
2025-11-28T14:53:00.392280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:00.395268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789388791543272:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.395313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:00.394442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789388810259411:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:00.394550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:00.392317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789385968988292:2049] [node 3] ICP22 created new session: [1:7577789385968988388:2049]
2025-11-28T14:53:00.395348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789388791543272:2048] [node 1] ICP22 created new session: [2:7577789388791543369:2048]
2025-11-28T14:53:00.393332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789385968988388:2049] [node 3] ICS09 handshake done sender: [1:7577789385968988362:2112] self: [1:7577789385968988358:0] peer: [3:7577789385559684092:0] socket: 285 qp: -1
2025-11-28T14:53:00.395449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789388791543369:2048] [node 1] ICS09 handshake done sender: [2:7577789388791543355:2119] self: [2:7577789388791543360:0] peer: [1:7577789385968988360:0] socket: 286 qp: -1
2025-11-28T14:53:00.395478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789388791543369:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:00.393608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789385968988388:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:00.395498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:00.396878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789385559684004:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.393842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:00.396969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789385559684079:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.396991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789385559684079:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.397006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:00.397033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789385559684004:2048] [node 1] ICP22 created new session: [3:7577789385559684100:2048]
2025-11-28T14:53:00.397820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789388857653895:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-28T14:53:00.397695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789385559684100:2048] [node 1] ICS09 handshake done sender: [3:7577789385559684086:2118] self: [3:7577789385559684092:0] peer: [1:7577789385968988358:0] socket: 290 qp: -1
2025-11-28T14:53:00.397761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789385559684100:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:00.397783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:00.398071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789388810259323:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:00.398123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789388857653895:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.398281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789389193872867:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.398602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789389193872851:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.398924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789388857653897:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-28T14:53:00.398684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789385559684087:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:00.400573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789388810259399:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.399828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789388857653876:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.400602Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789388810259399:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.401203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789385559684005:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.401470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789385559684079:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:00.401576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:00.401553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789388810259401:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.401628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789385559684005:2049] [node 2] ICP22 created new session: [3:7577789385559684103:2049]
2025-11-28T14:53:00.401575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789388810259401:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.401580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872869:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
2025-11-28T14:53:00.401837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789385559684103:2049] [node 2] ICS09 handshake done sender: [3:7577789385559684087:2119] self: [3:7577789385559684093:0] peer: [2:7577789388791543340:0] socket: 294 qp: -1
2025-11-28T14:53:00.402917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789385559684103:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:00.403119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789389193872869:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.403104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:00.403362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872871:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.403392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789389193872850:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.403502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789388857653899:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-28T14:53:00.403524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872872:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
2025-11-28T14:53:00.403606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789388857653887:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.403625Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789388857653887:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.403808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789389193872865:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:00.405450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789389193872789:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.405427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789385559684068:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.405593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789388810259399:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:00.405604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:00.405765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:00.405763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789389193872789:2050] [node 3] ICP22 created new session: [6:7577789389193872875:2050]
2025-11-28T14:53:00.405816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789385559684068:2109] [node 6] ICP22 created new session: [3:7577789385559684105:2109]
2025-11-28T14:53:00.405670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789388810259401:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:00.405867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789385559684105:2109] [node 6] ICS09 handshake done sender: [3:7577789385559684079:2114] self: [3:7577789385559684077:0] peer: [6:7577789389193872868:0] socket: 308 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:53:00.405812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789389193872875:2050] [node 3] ICS09 handshake done sender: [6:7577789389193872865:2115] self: [6:7577789389193872868:0] peer: [3:7577789385559684077:0] socket: 311 qp: -1
2025-11-28T14:53:00.405854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789388857653887:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:00.406068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789385559684105:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:00.405836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789389193872875:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:00.406118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789388810259387:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.406558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789388857653875:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.405855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:00.406148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:00.406378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:00.406598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789388857653900:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-28T14:53:00.406621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789388810259387:2109] [node 6] ICP22 created new session: [4:7577789388810259417:2109]
2025-11-28T14:53:00.407544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:00.408335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789388810259417:2109] [node 6] ICS09 handshake done sender: [4:7577789388810259399:2114] self: [4:7577789388810259398:0] peer: [6:7577789389193872870:0] socket: 263 qp: -1
2025-11-28T14:53:00.408660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789388810259417:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:00.408403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789388857653875:2109] [node 6] ICP22 created new session: [5:7577789388857653902:2109]
2025-11-28T14:53:00.408735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789389193872866:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.408691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789388810259416:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58461
2025-11-28T14:53:00.409030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789388857653902:2109] [node 6] ICS09 handshake done sender: [5:7577789388857653887:2113] self: [5:7577789388857653883:0] peer: [6:7577789389193872873:0] socket: 276 qp: -1
2025-11-28T14:53:00.408898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:00.409065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789388810259388:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.409056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789388857653899:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.409116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:00.409163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789388857653902:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:00.409198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:00.409243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789388810259388:2108] [node 5] ICP22 created new session: [4:7577789388810259419:2108]
2025-11-28T14:53:00.409399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789388810259419:2108] [node 5] ICS09 handshake done sender: [4:7577789388810259401:2115] self: [4:7577789388810259400:0] peer: [5:7577789388857653898:0] socket: 258 qp: -1
2025-11-28T14:53:00.409353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789388857653895:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:00.409437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789388810259419:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:00.409409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872874:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
2025-11-28T14:53:00.409485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789388810259416:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.409583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789388810259324:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.409786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789388857653876:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.409596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.409833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:00.410106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789388857653876:2108] [node 4] ICP22 created new session: [5:7577789388857653905:2108]
2025-11-28T14:53:00.410177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789388791543349:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.410190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872877:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
2025-11-28T14:53:00.410223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789388857653905:2108] [node 4] ICS09 handshake done sender: [5:7577789388857653895:2117] self: [5:7577789388857653898:0] peer: [4:7577789388810259400:0] socket: 269 qp: -1
2025-11-28T14:53:00.410221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789388791543349:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.410214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789389193872788:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.410243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789388857653812:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.410348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789389193872867:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:00.410245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789388857653905:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:00.411065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789388810259422:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58461
2025-11-28T14:53:00.410311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:00.410379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789388857653897:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.411157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789389193872872:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.411325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789389193872851:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.411349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789388857653814:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.411402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:00.411362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789388791543349:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:00.411821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789389193872869:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:00.411363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988368:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.411828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789389193872851:2108] [node 4] ICP22 created new session: [6:7577789389193872880:2108]
2025-11-28T14:53:00.411950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789388791543347:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.411402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789385968988368:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.411953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789389193872880:2108] [node 4] ICS09 handshake done sender: [6:7577789389193872867:2117] self: [6:7577789389193872870:0] peer: [4:7577789388810259398:0] socket: 313 qp: -1
2025-11-28T14:53:00.412028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789385559684080:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.412060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789385559684080:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.411971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789388791543347:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.411972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789389193872880:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:00.412117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789385968988368:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:00.411995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789388791543335:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.412908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:00.412640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:00.413210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789385968988357:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.413170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789388791543335:2107] [node 4] ICP22 created new session: [2:7577789388791543373:2107]
2025-11-28T14:53:00.413364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789388791543373:2107] [node 4] ICS09 handshake done sender: [2:7577789388791543349:2116] self: [2:7577789388791543348:0] peer: [4:7577789388810259420:0] socket: 297 qp: -1
2025-11-28T14:53:00.413247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:00.413521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789388791543373:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:00.413515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789385968988357:2108] [node 5] ICP22 created new session: [1:7577789385968988393:2108]
2025-11-28T14:53:00.413574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:00.414036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789389193872850:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.414354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789388791543347:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:00.413555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789385968988393:2108] [node 5] ICS09 handshake done sender: [1:7577789385968988368:2116] self: [1:7577789385968988365:0] peer: [5:7577789388857653906:0] socket: 316 qp: -1
No RDMA context found for GID: 2025-11-28T14:53:00.414082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
::ffff:127.1.0.0
2025-11-28T14:53:00.414285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789389193872850:2106] [node 5] ICP22 created new session: [6:7577789389193872882:2106]
2025-11-28T14:53:00.414004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789385968988393:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:00.414037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:00.414316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789389193872882:2106] [node 5] ICS09 handshake done sender: [6:7577789389193872869:2118] self: [6:7577789389193872873:0] peer: [5:7577789388857653883:0] socket: 315 qp: -1
2025-11-28T14:53:00.414418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789385559684080:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:00.414463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789388810259423:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58461
2025-11-28T14:53:00.414339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789389193872882:2106] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.414366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:00.415924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789388791543337:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.415999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789385559684069:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.416076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:00.416766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789388810259424:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58461
2025-11-28T14:53:00.416962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:00.416126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789388791543337:2109] [node 6] ICP22 created new session: [2:7577789388791543376:2109]
2025-11-28T14:53:00.416733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789388791543376:2109] [node 6] ICS09 handshake done sender: [2:7577789388791543347:2115] self: [2:7577789388791543346:0] peer: [6:7577789389193872879:0] socket: 259 qp: -1
2025-11-28T14:53:00.416915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789388791543376:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.416933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789388857653812:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789388857653899:2119] held: no
2025-11-28T14:53:00.416947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:00.417131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789389193872787:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.417141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789385559684069:2108] [node 5] ICP22 created new session: [3:7577789385559684108:2108]
2025-11-28T14:53:00.417457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789388857653909:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-28T14:53:00.417345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789385559684108:2108] [node 5] ICS09 handshake done sender: [3:7577789385559684080:2115] self: [3:7577789385559684078:0] peer: [5:7577789388857653908:0] socket: 267 qp: -1
2025-11-28T14:53:00.418019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789388810259324:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5310) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789388810259416:2120] held: no
2025-11-28T14:53:00.417393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789385559684108:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:00.417615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:00.418802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789385968988367:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.418776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789388857653814:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12468) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789388857653897:2118] held: no
2025-11-28T14:53:00.418830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789385968988367:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.420951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789388857653910:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-28T14:53:00.420944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789388810259416:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:00.421198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789388810259324:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.421298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789385968988367:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:00.421427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:00.421478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789388810259324:2049] [node 2] ICP22 created new session: [4:7577789388810259425:2049]
2025-11-28T14:53:00.421525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789388810259425:2049] [node 2] ICS09 handshake done sender: [4:7577789388810259416:2120] self: [4:7577789388810259420:0] peer: [2:7577789388791543348:0] socket: 319 qp: -1
2025-11-28T14:53:00.421745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789385968988356:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.421552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789388810259425:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:00.421781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789388857653899:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:00.421945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:00.421584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:00.422068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789385968988356:2110] [node 6] ICP22 created new session: [1:7577789385968988396:2110]
2025-11-28T14:53:00.421525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789389193872788:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5310) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789389193872866:2116] held: no
2025-11-28T14:53:00.422236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789385968988396:2110] [node 6] ICS09 handshake done sender: [1:7577789385968988367:2115] self: [1:7577789385968988366:0] peer: [6:7577789389193872885:0] socket: 291 qp: -1
2025-11-28T14:53:00.422289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789388857653812:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.422315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789385968988396:2110] [node 6] ICS10 traffic start
2025-11-28T14:53:00.422443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:00.422449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789388857653897:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:00.422648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:00.422805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789388857653812:2048] [node 1] ICP22 created new session: [5:7577789388857653911:2048]
2025-11-28T14:53:00.422869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789388857653814:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.422869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789388857653911:2048] [node 1] ICS09 handshake done sender: [5:7577789388857653899:2119] self: [5:7577789388857653906:0] peer: [1:7577789385968988365:0] socket: 318 qp: -1
2025-11-28T14:53:00.422902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:00.422912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789388857653911:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:00.422931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789388857653814:2050] [node 3] ICP22 created new session: [5:7577789388857653912:2050]
2025-11-28T14:53:00.422957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789388857653912:2050] [node 3] ICS09 handshake done sender: [5:7577789388857653897:2118] self: [5:7577789388857653908:0] peer: [3:7577789385559684078:0] socket: 314 qp: -1
2025-11-28T14:53:00.422958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:00.422978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789388857653912:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:00.422998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.423336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872884:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
2025-11-28T14:53:00.424455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789389193872787:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789389193872872:2120] held: no
2025-11-28T14:53:00.424783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789389193872866:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.425151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789389193872788:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.425183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:00.425185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789389193872886:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33577
2025-11-28T14:53:00.425207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789389193872788:2049] [node 2] ICP22 created new session: [6:7577789389193872887:2049]
2025-11-28T14:53:00.425261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789389193872887:2049] [node 2] ICS09 handshake done sender: [6:7577789389193872866:2116] self: [6:7577789389193872879:0] peer: [2:7577789388791543346:0] socket: 280 qp: -1
2025-11-28T14:53:00.425280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789389193872887:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:00.425299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:00.425325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789389193872872:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:00.425586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789389193872787:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.425624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:00.425654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789389193872787:2048] [node 1] ICP22 created new session: [6:7577789389193872889:2048]
2025-11-28T14:53:00.425686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789389193872889:2048] [node 1] ICS09 handshake done sender: [6:7577789389193872872:2120] self: [6:7577789389193872885:0] peer: [1:7577789385968988366:0] socket: 317 qp: -1
2025-11-28T14:53:00.425712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789389193872889:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:00.425738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:00.460734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:00.460903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789388791543369:2048] [node 1] ICS07 socket disconnect 286 reason# EndOfStream
2025-11-28T14:53:00.460930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789388791543369:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.461018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:00.461058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789388791543369:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:00.461902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:00.462144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789385559684100:2048] [node 1] ICS07 socket disconnect 290 reason# EndOfStream
2025-11-28T14:53:00.462203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789385559684100:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.462296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:00.462349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789385559684100:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:00.463127Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789385559684004:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.463144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789388791543272:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:00.463204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789388791543369:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:00.463480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789385559684100:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.463849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789385559684004:2048] [node 1] ICP30 unregister session Session# [3:7577789385559684100:2048] VirtualId# [3:7577789385559684092:0]
2025-11-28T14:53:00.464057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:00.464106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789385559684004:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.464131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.464104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:00.464235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789388810259411:2048] [node 1] ICS07 socket disconnect 287 reason# EndOfStream
2025-11-28T14:53:00.464256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789388810259411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.464301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:00.464345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789388810259411:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:00.464528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789388791543272:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.464637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789388791543369:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.464678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789388791543272:2048] [node 1] ICP30 unregister session Session# [2:7577789388791543369:2048] VirtualId# [2:7577789388791543360:0]
2025-11-28T14:53:00.464856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:00.464896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789388791543272:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.464908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.465024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789388810259323:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.465089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789388810259411:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.465118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789388810259323:2048] [node 1] ICP30 unregister session Session# [4:7577789388810259411:2048] VirtualId# [4:7577789388810259405:0]
2025-11-28T14:53:00.465234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:00.465263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789388810259323:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.465271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.465367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:00.465610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789388857653911:2048] [node 1] ICS01 socket: 318 reason# EndOfStream
2025-11-28T14:53:00.465655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789388857653812:2048] [node 1] ICP30 unregister session Session# [5:7577789388857653911:2048] VirtualId# [5:7577789388857653906:0]
2025-11-28T14:53:00.465686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789388857653911:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.465771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:00.465825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:00.466139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:00.466264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789389193872889:2048] [node 1] ICS01 socket: 317 reason# EndOfStream
2025-11-28T14:53:00.466294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789389193872787:2048] [node 1] ICP30 unregister session Session# [6:7577789389193872889:2048] VirtualId# [6:7577789389193872885:0]
2025-11-28T14:53:00.466317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789389193872889:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.466376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:00.466413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:00.474034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789385559684004:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:00.475036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789388810259323:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:00.527224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:00.527452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789385559684103:2049] [node 2] ICS01 socket: 294 reason# EndOfStream
2025-11-28T14:53:00.527498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789385559684005:2049] [node 2] ICP30 unregister session Session# [3:7577789385559684103:2049] VirtualId# [3:7577789385559684093:0]
2025-11-28T14:53:00.527535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789385559684103:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.527647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:00.527709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:00.529151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:00.529287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789388810259425:2049] [node 2] ICS01 socket: 319 reason# EndOfStream
2025-11-28T14:53:00.529317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789388810259324:2049] [node 2] ICP30 unregister session Session# [4:7577789388810259425:2049] VirtualId# [4:7577789388810259420:0]
2025-11-28T14:53:00.529336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789388810259425:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.529383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:00.529414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:00.529898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:00.530132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789388857653893:2049] [node 2] ICS07 socket disconnect 296 reason# EndOfStream
2025-11-28T14:53:00.530170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789388857653893:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.530243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:00.530289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789388857653893:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:00.530552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:00.530707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789389193872887:2049] [node 2] ICS01 socket: 280 reason# EndOfStream
2025-11-28T14:53:00.530746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789389193872788:2049] [node 2] ICP30 unregister session Session# [6:7577789389193872887:2049] VirtualId# [6:7577789389193872879:0]
2025-11-28T14:53:00.530784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789389193872887:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.530872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:00.530910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:00.530983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789388857653813:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5310) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.531021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789388857653893:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.531042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789388857653813:2049] [node 2] ICP30 unregister session Session# [5:7577789388857653893:2049] VirtualId# [5:7577789388857653891:0]
2025-11-28T14:53:00.531092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:00.531132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789388857653813:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5310) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.531146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5310) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.540578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789388857653813:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:00.581312Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:00.581476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789388810259409:2050] [node 3] ICS01 socket: 283 reason# EndOfStream
2025-11-28T14:53:00.581507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789388810259325:2050] [node 3] ICP30 unregister session Session# [4:7577789388810259409:2050] VirtualId# [4:7577789388810259404:0]
2025-11-28T14:53:00.581529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789388810259409:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.581592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:00.581629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:00.582180Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:00.582382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789388857653912:2050] [node 3] ICS01 socket: 314 reason# EndOfStream
2025-11-28T14:53:00.582429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789388857653814:2050] [node 3] ICP30 unregister session Session# [5:7577789388857653912:2050] VirtualId# [5:7577789388857653908:0]
2025-11-28T14:53:00.582459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789388857653912:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.582530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:00.582577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:00.582793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:00.582957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789389193872875:2050] [node 3] ICS01 socket: 311 reason# EndOfStream
2025-11-28T14:53:00.583000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789389193872789:2050] [node 3] ICP30 unregister session Session# [6:7577789389193872875:2050] VirtualId# [6:7577789389193872868:0]
2025-11-28T14:53:00.583026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789389193872875:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.583104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:00.583143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:00.645006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:00.645243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789388857653905:2108] [node 4] ICS01 socket: 269 reason# EndOfStream
2025-11-28T14:53:00.645295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789388857653876:2108] [node 4] ICP30 unregister session Session# [5:7577789388857653905:2108] VirtualId# [5:7577789388857653898:0]
2025-11-28T14:53:00.645340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789388857653905:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.645434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:00.645491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:00.645882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:00.646034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789389193872880:2108] [node 4] ICS07 socket disconnect 313 reason# EndOfStream
2025-11-28T14:53:00.646069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789389193872880:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.646154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:00.646192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789389193872880:2108] [node 4] ICS15 start handshake
2025-11-28T14:53:00.646791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789389193872851:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24036) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.646906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789389193872880:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.646946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789389193872851:2108] [node 4] ICP30 unregister session Session# [6:7577789389193872880:2108] VirtualId# [6:7577789389193872870:0]
2025-11-28T14:53:00.647052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:00.647080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789389193872851:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.647091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.657286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789389193872851:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:53:00.673736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:00.673896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789389193872882:2106] [node 5] ICS07 socket disconnect 315 reason# EndOfStream
2025-11-28T14:53:00.673922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789389193872882:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:00.673996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:00.674033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789389193872882:2106] [node 5] ICS15 start handshake
2025-11-28T14:53:00.674680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789389193872850:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7793) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:00.674772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789389193872882:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:00.674816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789389193872850:2106] [node 5] ICP30 unregister session Session# [6:7577789389193872882:2106] VirtualId# [6:7577789389193872873:0]
2025-11-28T14:53:00.674905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:53:00.674931Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789389193872850:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7793) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.674941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7793) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:00.684967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789389193872850:2106] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:00.755694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789386049051882:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:00.756011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789386049051881:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:00.773001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789387948797780:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.773038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789387948797781:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:00.792725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789388741372697:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:00.792876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789388741372696:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.811416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389274522192:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.811446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389274522193:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:00.811702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389274522194:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:00.828313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789385515649804:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.828341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789385515649805:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:00.828382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789385515649806:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:00.846728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789388230838043:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:00.846731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789388230838042:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:00.846745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789388230838044:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:00.936282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789386049051944:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:00.936807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789388230838104:2106] [node 5] ICP01 ready to work
2025-11-28T14:53:00.936906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789385515649867:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:00.937047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389274522256:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:00.937436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789387948797852:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:00.937494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789388741372759:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:00.937461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389274522255:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:00.936338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789386049051945:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:00.937725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789388741372761:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:00.937072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789388230838106:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:00.937878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789388741372760:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:00.936864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789386049051946:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:00.937839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789385515649868:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:00.937690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789387948797853:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:00.937700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789387948797854:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.949961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789388230838118:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
2025-11-28T14:53:00.951794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789388741372773:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50225
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
2025-11-28T14:53:00.957336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789385515649880:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19558
2025-11-28T14:53:00.957468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789388741372774:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50225
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.959075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789388230838118:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.959493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789388741372774:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.959849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789386049051958:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24596
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.960010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789388230838044:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:53:00.959896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789385515649880:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.960455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789386049051960:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24596
2025-11-28T14:53:00.960694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789388741372766:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.961046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389274522266:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27259
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.961509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789386049051959:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24596
2025-11-28T14:53:00.960737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789388741372766:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.961139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389274522265:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27259
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.961512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051958:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.960999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789388741372777:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50225
2025-11-28T14:53:00.962719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789387948797866:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61301
2025-11-28T14:53:00.962743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389274522265:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.961653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789386049051961:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24596
2025-11-28T14:53:00.961148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789388741372773:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.964018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389274522266:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.964735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789387948797868:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61301
2025-11-28T14:53:00.964726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789389274522194:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.962095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789388741372696:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.964962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789387948797867:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61301
2025-11-28T14:53:00.965193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389274522267:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27259
2025-11-28T14:53:00.961829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051960:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.965160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789387948797866:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.966366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789387948797867:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.966210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789389274522192:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.964562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789388741372766:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:00.963184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051950:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.966378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789387948797780:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.966440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789388741372780:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50225
2025-11-28T14:53:00.966475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789387948797868:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.963210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789386049051950:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.966588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789388741372761:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.963238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051959:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.966958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389274522267:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.965178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051961:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.969536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789389274522193:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.966777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:00.966816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789386049051966:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24596
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.966867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789386049051950:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:00.966835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789388741372761:2109] [node 6] ICP22 created new session: [3:7577789388741372782:2109]
2025-11-28T14:53:00.971176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789387948797860:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.966971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051954:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.967095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789388741372768:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.971218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789387948797860:2120] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.967127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789388741372768:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.967003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789386049051954:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.971416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789387948797861:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.972809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789388230838122:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
2025-11-28T14:53:00.967437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789388741372782:2109] [node 6] ICS09 handshake done sender: [3:7577789388741372766:2111] self: [3:7577789388741372764:0] peer: [6:7577789388230838121:0] socket: 266 qp: -1
2025-11-28T14:53:00.967431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051949:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.971437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789387948797861:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.967510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789388741372782:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:00.967461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789386049051949:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.973010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789387948797861:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:00.968256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.969889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789386049051882:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.968267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789388741372777:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.973219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789387948797872:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61301
2025-11-28T14:53:00.970322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:00.970328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789388741372697:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.973259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789387948797875:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61301
2025-11-28T14:53:00.974984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389274522276:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27259
2025-11-28T14:53:00.970567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789386049051882:2049] [node 3] ICP22 created new session: [1:7577789386049051971:2049]
2025-11-28T14:53:00.975562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789388230838118:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:00.973421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789387948797860:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:00.971397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789386049051971:2049] [node 3] ICS09 handshake done sender: [1:7577789386049051950:2111] self: [1:7577789386049051947:0] peer: [3:7577789388741372779:0] socket: 269 qp: -1
2025-11-28T14:53:00.971128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789388741372768:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:00.975520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789387948797781:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.971443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789386049051971:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:00.972876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789388741372780:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.975551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:00.971490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:00.973683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789388741372759:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.975588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789387948797781:2049] [node 3] ICP22 created new session: [2:7577789387948797879:2049]
2025-11-28T14:53:00.973695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789388741372783:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50225
2025-11-28T14:53:00.973483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051966:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.973825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:00.975915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789387948797853:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.973777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789386049051954:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:00.977256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389274522277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27259
2025-11-28T14:53:00.977235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789388230838044:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.975930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789387948797872:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.974105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789386049051949:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:00.973925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789388741372759:2107] [node 4] ICP22 created new session: [3:7577789388741372788:2107]
2025-11-28T14:53:00.977299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:00.978667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389274522265:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:00.976353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:00.974109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789388741372788:2107] [node 4] ICS09 handshake done sender: [3:7577789388741372768:2112] self: [3:7577789388741372765:0] peer: [4:7577789389274522271:0] socket: 265 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.975270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789386049051944:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.978806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389274522278:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27259
2025-11-28T14:53:00.978129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789388230838044:2050] [node 3] ICP22 created new session: [6:7577789388230838124:2050]
2025-11-28T14:53:00.978595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789388230838123:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
2025-11-28T14:53:00.976696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789387948797879:2049] [node 3] ICS09 handshake done sender: [2:7577789387948797861:2121] self: [2:7577789387948797856:0] peer: [3:7577789388741372787:0] socket: 293 qp: -1
2025-11-28T14:53:00.979971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789389274522194:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.975321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:00.974993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789388741372788:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:00.976763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789387948797853:2107] [node 4] ICP22 created new session: [2:7577789387948797880:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.978761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789388230838124:2050] [node 3] ICS09 handshake done sender: [6:7577789388230838118:2114] self: [6:7577789388230838121:0] peer: [3:7577789388741372764:0] socket: 270 qp: -1
2025-11-28T14:53:00.980020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:00.975038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:00.975589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789386049051944:2107] [node 4] ICP22 created new session: [1:7577789386049051976:2107]
2025-11-28T14:53:00.976839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789387948797879:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.979484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789388230838124:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:00.976637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789386049051881:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:00.980211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389274522194:2050] [node 3] ICP22 created new session: [4:7577789389274522279:2050]
2025-11-28T14:53:00.978877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789388741372696:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5216) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789388741372773:2114] held: no
2025-11-28T14:53:00.976878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:00.981695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389274522266:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:00.980164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:00.982549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789388741372789:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50225
2025-11-28T14:53:00.982257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789389274522192:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5216) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789389274522266:2115] held: no
2025-11-28T14:53:00.976676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789386049051976:2107] [node 4] ICS09 handshake done sender: [1:7577789386049051954:2113] self: [1:7577789386049051952:0] peer: [4:7577789389274522272:0] socket: 267 qp: -1
2025-11-28T14:53:00.986326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789388230838125:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
2025-11-28T14:53:00.976910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789387948797880:2107] [node 4] ICS09 handshake done sender: [2:7577789387948797860:2120] self: [2:7577789387948797855:0] peer: [4:7577789389274522273:0] socket: 292 qp: -1
2025-11-28T14:53:00.987152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789388741372773:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:00.982312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789389274522279:2050] [node 3] ICS09 handshake done sender: [4:7577789389274522265:2114] self: [4:7577789389274522271:0] peer: [3:7577789388741372765:0] socket: 277 qp: -1
2025-11-28T14:53:00.978040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789387948797880:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:00.987085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789388230838123:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.976887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789386049051976:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:00.987234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789388741372777:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:00.987132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789385515649884:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19558
2025-11-28T14:53:00.982363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789389274522279:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:00.978076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:00.976923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:00.983935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:00.985494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389274522267:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:00.987451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789389274522192:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.988755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789388741372697:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.987782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789388230838042:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.976923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:00.976977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789386049051881:2048] [node 2] ICP22 created new session: [1:7577789386049051978:2048]
No RDMA context found for GID: 2025-11-28T14:53:00.978601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789386049051978:2048] [node 2] ICS09 handshake done sender: [1:7577789386049051949:2110] self: [1:7577789386049051948:0] peer: [2:7577789387948797871:0] socket: 268 qp: -1
2025-11-28T14:53:00.987620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:00.979141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789387948797878:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61301
2025-11-28T14:53:00.978854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789386049051978:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:00.987873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389274522192:2048] [node 1] ICP22 created new session: [4:7577789389274522282:2048]
::ffff:127.1.0.0
2025-11-28T14:53:00.982593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789387948797866:2123] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:00.978902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:00.988272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789389274522193:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.988783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789388741372696:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.988626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789389274522282:2048] [node 1] ICS09 handshake done sender: [4:7577789389274522266:2115] self: [4:7577789389274522272:0] peer: [1:7577789386049051952:0] socket: 284 qp: -1
2025-11-28T14:53:00.988201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789388230838125:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.989108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051953:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.986549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789387948797780:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5216) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789387948797866:2123] held: no
2025-11-28T14:53:00.989442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789385515649885:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19558
2025-11-28T14:53:00.989074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789388230838043:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.989133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789386049051953:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.988817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:00.987162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789387948797875:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.988662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789389274522282:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:00.993543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789385515649885:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.988830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:00.988666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:00.989249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789387948797780:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:00.988866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789388741372697:2049] [node 2] ICP22 created new session: [3:7577789388741372792:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.994957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789386049051953:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:00.988718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389274522193:2049] [node 2] ICP22 created new session: [4:7577789389274522283:2049]
2025-11-28T14:53:00.995067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789385515649804:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.989462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:00.989033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789388741372696:2048] [node 1] ICP22 created new session: [3:7577789388741372793:2048]
2025-11-28T14:53:00.988760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789389274522283:2049] [node 2] ICS09 handshake done sender: [4:7577789389274522267:2116] self: [4:7577789389274522273:0] peer: [2:7577789387948797855:0] socket: 298 qp: -1
2025-11-28T14:53:00.995979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789386049051882:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17357871ns) {HandleHandshakeStatus:329(17354399ns) {DropIncomingHandshake:442(14333ns) DropOutgoingHandshake:457(433ns) SwitchToState:179(372ns) ProcessPendingSessionEvents:495(14079365ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13467230ns max# 1340531ns}}}
2025-11-28T14:53:00.991941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789387948797780:2048] [node 1] ICP22 created new session: [2:7577789387948797885:2048]
2025-11-28T14:53:00.988786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789389274522283:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:00.989166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789388741372793:2048] [node 1] ICS09 handshake done sender: [3:7577789388741372773:2114] self: [3:7577789388741372779:0] peer: [1:7577789386049051947:0] socket: 274 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.996449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789385515649886:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19558
2025-11-28T14:53:00.992236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789387948797885:2048] [node 1] ICS09 handshake done sender: [2:7577789387948797866:2123] self: [2:7577789387948797871:0] peer: [1:7577789386049051948:0] socket: 289 qp: -1
2025-11-28T14:53:00.989202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789388741372793:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:00.988819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:00.992323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789387948797862:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.997195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789386049051963:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.989415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789388741372792:2049] [node 2] ICS09 handshake done sender: [3:7577789388741372777:2117] self: [3:7577789388741372787:0] peer: [2:7577789387948797856:0] socket: 296 qp: -1
2025-11-28T14:53:00.989101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:00.997640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789385515649884:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.992349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789387948797862:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.997237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789386049051963:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:00.997966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789385515649805:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:00.990532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:53:00.997579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389274522280:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27259
2025-11-28T14:53:00.992397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789387948797885:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:00.992122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789388741372792:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:00.992173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:01.001093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789388230838131:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
2025-11-28T14:53:00.993014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789388741372761:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18257613ns) {HandleHandshakeStatus:329(18254364ns) {DropIncomingHandshake:442(2312ns) DropOutgoingHandshake:457(945ns) SwitchToState:179(304ns) ProcessPendingSessionEvents:495(13853038ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12633691ns max# 1096079ns}}}
2025-11-28T14:53:00.993069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:01.001051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789386049051945:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.000580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389274522280:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.994843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789387948797853:2107] [node 4] ICP18 handshake (actor [2:7577789387948797872:2127]) is too old
2025-11-28T14:53:01.001106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789388230838130:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:00.993510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789388741372759:2107] [node 4] ICP18 handshake (actor [3:7577789388741372780:2118]) is too old
2025-11-28T14:53:01.001110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:00.997359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789387948797862:2122] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:01.001373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789386049051945:2109] [node 6] ICP22 created new session: [1:7577789390344019278:2109]
2025-11-28T14:53:01.003307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789388230838132:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
2025-11-28T14:53:00.997934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789387948797852:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.001927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789390344019278:2109] [node 6] ICS09 handshake done sender: [1:7577789386049051953:2112] self: [1:7577789386049051951:0] peer: [6:7577789388230838128:0] socket: 319 qp: -1
2025-11-28T14:53:01.003686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789385515649889:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19558
2025-11-28T14:53:01.003338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789388230838043:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30191) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789388230838125:2118] held: no
2025-11-28T14:53:01.004215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789389274522194:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17107385ns) {HandleHandshakeStatus:329(17105202ns) {DropIncomingHandshake:442(735ns) DropOutgoingHandshake:457(4736ns) SwitchToState:179(433ns) ProcessPendingSessionEvents:495(11216016ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10504786ns max# 1356721ns}}}
2025-11-28T14:53:00.997993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:01.004328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789388230838123:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:01.004213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789385515649886:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.002843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789390344019278:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:00.998119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789387948797852:2109] [node 6] ICP22 created new session: [2:7577789387948797888:2109]
2025-11-28T14:53:01.004680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789388230838132:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.002912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789386049051944:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(20631575ns) {HandleHandshakeStatus:329(20630326ns) {DropIncomingHandshake:442(14962ns) DropOutgoingHandshake:457(320ns) SwitchToState:179(123ns) ProcessPendingSessionEvents:495(16709951ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16594126ns max# 1403779ns}}}
2025-11-28T14:53:00.998716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789387948797781:2049] [node 3] ICP18 handshake (actor [2:7577789387948797875:2128]) is too old
2025-11-28T14:53:01.004561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789385515649889:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.007031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789388741372776:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:00.999396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789387948797888:2109] [node 6] ICS09 handshake done sender: [2:7577789387948797862:2122] self: [2:7577789387948797858:0] peer: [6:7577789388230838129:0] socket: 264 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.003063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:01.005157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789385515649806:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.008467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389274522264:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.007082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789388741372776:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.003902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789386049051881:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19169449ns) {HandleHandshakeStatus:329(19161844ns) {DropIncomingHandshake:442(14496ns) DropOutgoingHandshake:457(738ns) SwitchToState:179(380ns) ProcessPendingSessionEvents:495(15614486ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15510390ns max# 1528307ns}}}
2025-11-28T14:53:01.007030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789385515649868:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.008507Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789389274522264:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.000540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789387948797888:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:01.009114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789388230838042:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.007454Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789385515649805:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30191) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789385515649884:2116] held: no
2025-11-28T14:53:01.009180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:01.008188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789386049051963:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:01.009572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789389810617186:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19558
No RDMA context found for GID: 2025-11-28T14:53:01.001027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
2025-11-28T14:53:01.001534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789387948797870:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.009423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789388230838042:2048] [node 1] ICP22 created new session: [6:7577789392525805429:2048]
2025-11-28T14:53:01.011080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789389810617189:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19558
2025-11-28T14:53:01.010404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789392525805429:2048] [node 1] ICS09 handshake done sender: [6:7577789388230838123:2117] self: [6:7577789388230838128:0] peer: [1:7577789386049051951:0] socket: 322 qp: -1
2025-11-28T14:53:01.001558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789387948797870:2126] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.011154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389274522264:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:01.011820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789386049051946:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.003847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789387948797870:2126] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:01.012442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:01.010417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789388230838125:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:01.013049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389274522275:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.012771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789386049051946:2108] [node 5] ICP22 created new session: [1:7577789390344019281:2108]
2025-11-28T14:53:01.013270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789385515649884:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:01.004292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789387948797854:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.010459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789392525805429:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:01.013247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789390344019281:2108] [node 5] ICS09 handshake done sender: [1:7577789386049051963:2118] self: [1:7577789386049051962:0] peer: [5:7577789385515649887:0] socket: 301 qp: -1
2025-11-28T14:53:01.013087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789389274522275:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.004559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:01.004714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789387948797854:2108] [node 5] ICP22 created new session: [2:7577789392243765187:2108]
2025-11-28T14:53:01.010495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789388230838106:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.013426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789385515649885:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:01.013401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789390344019281:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.014409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789385515649805:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.010699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:01.004896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789392243765187:2108] [node 5] ICS09 handshake done sender: [2:7577789387948797870:2126] self: [2:7577789387948797869:0] peer: [5:7577789385515649888:0] socket: 323 qp: -1
2025-11-28T14:53:01.013447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:01.011477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789388230838043:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.015003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789385515649804:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.015896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789389274522255:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.011525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:01.015892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789388741372697:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18680216ns) {HandleHandshakeStatus:329(18675549ns) {DropIncomingHandshake:442(1109ns) DropOutgoingHandshake:457(5160ns) SwitchToState:179(554ns) ProcessPendingSessionEvents:495(13558304ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12041782ns max# 1611270ns}}}
2025-11-28T14:53:01.015055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:01.005161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789392243765187:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:01.011812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789388230838043:2049] [node 2] ICP22 created new session: [6:7577789392525805432:2049]
2025-11-28T14:53:01.016072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:01.015244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789385515649804:2048] [node 1] ICP22 created new session: [5:7577789389810617192:2048]
2025-11-28T14:53:01.011858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789392525805432:2049] [node 2] ICS09 handshake done sender: [6:7577789388230838125:2118] self: [6:7577789388230838129:0] peer: [2:7577789387948797858:0] socket: 302 qp: -1
2025-11-28T14:53:01.005664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:01.016095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789389274522193:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19478046ns) {HandleHandshakeStatus:329(19475567ns) {DropIncomingHandshake:442(1577ns) DropOutgoingHandshake:457(5582ns) SwitchToState:179(344ns) ProcessPendingSessionEvents:495(15252818ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15142621ns max# 2664599ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.015424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:01.016766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389274522255:2108] [node 5] ICP22 created new session: [4:7577789393569489586:2108]
2025-11-28T14:53:01.015438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789389810617192:2048] [node 1] ICS09 handshake done sender: [5:7577789385515649885:2117] self: [5:7577789385515649887:0] peer: [1:7577789386049051962:0] socket: 290 qp: -1
2025-11-28T14:53:01.017245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789393569489586:2108] [node 5] ICS09 handshake done sender: [4:7577789389274522264:2113] self: [4:7577789389274522259:0] peer: [5:7577789389810617188:0] socket: 310 qp: -1
2025-11-28T14:53:01.015584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789389810617190:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19558
2025-11-28T14:53:01.011887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789392525805432:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:01.017426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789393569489586:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:01.015602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789385515649805:2049] [node 2] ICP22 created new session: [5:7577789389810617193:2049]
2025-11-28T14:53:01.017467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:01.015604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789389810617192:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:01.011918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:01.015726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:01.017318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789392525805434:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
2025-11-28T14:53:01.015875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789389810617193:2049] [node 2] ICS09 handshake done sender: [5:7577789385515649884:2116] self: [5:7577789385515649888:0] peer: [2:7577789387948797869:0] socket: 324 qp: -1
2025-11-28T14:53:01.016046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789389810617193:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:01.016080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:01.018464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789388741372776:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:01.018676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789385515649889:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:01.019043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789388230838042:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:01.019314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789385515649868:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.019372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:01.019372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789388741372760:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.019532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389274522275:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:01.019532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789385515649868:2108] [node 4] ICP22 created new session: [5:7577789389810617198:2108]
2025-11-28T14:53:01.019609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.019550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789392525805434:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.019649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789388741372760:2108] [node 5] ICP22 created new session: [3:7577789393036340093:2108]
2025-11-28T14:53:01.019892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789389810617198:2108] [node 4] ICS09 handshake done sender: [5:7577789385515649889:2119] self: [5:7577789389810617188:0] peer: [4:7577789389274522259:0] socket: 329 qp: -1
2025-11-28T14:53:01.020229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789392525805435:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
2025-11-28T14:53:01.020284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789388230838104:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.020083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789389810617198:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:01.019717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789393036340093:2108] [node 5] ICS09 handshake done sender: [3:7577789388741372776:2116] self: [3:7577789388741372775:0] peer: [5:7577789389810617187:0] socket: 299 qp: -1
2025-11-28T14:53:01.020098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789385515649804:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:01.021139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789388230838132:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:01.019930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789393036340093:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:01.020092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:01.021416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789389274522256:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.021436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789388230838106:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.020345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: 2025-11-28T14:53:01.021579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-11-28T14:53:01.021815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389274522256:2109] [node 6] ICP22 created new session: [4:7577789393569489590:2109]
2025-11-28T14:53:01.021481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:01.021675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789388230838106:2107] [node 4] ICP22 created new session: [6:7577789392525805437:2107]
2025-11-28T14:53:01.021877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789393569489590:2109] [node 6] ICS09 handshake done sender: [4:7577789389274522275:2117] self: [4:7577789389274522274:0] peer: [6:7577789392525805430:0] socket: 330 qp: -1
2025-11-28T14:53:01.021885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789389810617196:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19558
2025-11-28T14:53:01.021791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789392525805437:2107] [node 4] ICS09 handshake done sender: [6:7577789388230838132:2121] self: [6:7577789392525805430:0] peer: [4:7577789389274522274:0] socket: 331 qp: -1
2025-11-28T14:53:01.021925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789393569489590:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:01.021829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789392525805437:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:01.021860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:01.022013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.023517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789393569489589:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27259
2025-11-28T14:53:01.024903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789385515649806:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12740) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789385515649886:2118] held: no
2025-11-28T14:53:01.024912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789385515649874:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.024937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789385515649874:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.026050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789385515649886:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:01.026352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789385515649806:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.026399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:01.026443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789385515649806:2050] [node 3] ICP22 created new session: [5:7577789389810617201:2050]
2025-11-28T14:53:01.026500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789389810617201:2050] [node 3] ICS09 handshake done sender: [5:7577789385515649886:2118] self: [5:7577789389810617187:0] peer: [3:7577789388741372775:0] socket: 326 qp: -1
2025-11-28T14:53:01.026526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789389810617201:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:01.026553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:01.026699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789385515649874:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.026892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789385515649867:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.026856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789392525805439:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5649
2025-11-28T14:53:01.027013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:01.027086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789385515649867:2109] [node 6] ICP22 created new session: [5:7577789389810617203:2109]
2025-11-28T14:53:01.027145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789389810617203:2109] [node 6] ICS09 handshake done sender: [5:7577789385515649874:2111] self: [5:7577789385515649869:0] peer: [6:7577789392525805436:0] socket: 300 qp: -1
2025-11-28T14:53:01.027186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789389810617203:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:01.027270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:01.027275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789392525805434:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:01.027611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789388230838104:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26188) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789392525805434:2122] held: no
2025-11-28T14:53:01.028973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789388230838104:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.029007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:01.029035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789388230838104:2106] [node 5] ICP22 created new session: [6:7577789392525805440:2106]
2025-11-28T14:53:01.029066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789392525805440:2106] [node 5] ICS09 handshake done sender: [6:7577789392525805434:2122] self: [6:7577789392525805436:0] peer: [5:7577789385515649869:0] socket: 287 qp: -1
2025-11-28T14:53:01.029086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789392525805440:2106] [node 5] ICS10 traffic start
2025-11-28T14:53:01.029114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:01.083917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:01.084206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789387948797885:2048] [node 1] ICS01 socket: 289 reason# EndOfStream
2025-11-28T14:53:01.084268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789387948797780:2048] [node 1] ICP30 unregister session Session# [2:7577789387948797885:2048] VirtualId# [2:7577789387948797871:0]
2025-11-28T14:53:01.084315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789387948797885:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.084465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.084540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.085361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:01.085575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789388741372793:2048] [node 1] ICS01 socket: 274 reason# EndOfStream
2025-11-28T14:53:01.085639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789388741372696:2048] [node 1] ICP30 unregister session Session# [3:7577789388741372793:2048] VirtualId# [3:7577789388741372779:0]
2025-11-28T14:53:01.085674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789388741372793:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.085751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.085803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.088001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:01.088167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789389274522282:2048] [node 1] ICS01 socket: 284 reason# EndOfStream
2025-11-28T14:53:01.088219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789389274522192:2048] [node 1] ICP30 unregister session Session# [4:7577789389274522282:2048] VirtualId# [4:7577789389274522272:0]
2025-11-28T14:53:01.088257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789389274522282:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.088339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.088398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.089233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:01.089458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789389810617192:2048] [node 1] ICS07 socket disconnect 290 reason# EndOfStream
2025-11-28T14:53:01.089489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789389810617192:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.089563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.089602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789389810617192:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:01.090357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789385515649804:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5216) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:01.090399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789389810617192:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:01.090399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:01.090505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789385515649804:2048] [node 1] ICP30 unregister session Session# [5:7577789389810617192:2048] VirtualId# [5:7577789385515649887:0]
2025-11-28T14:53:01.090568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.090621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789385515649804:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.090642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.090729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789392525805429:2048] [node 1] ICS07 socket disconnect 322 reason# EndOfStream
2025-11-28T14:53:01.090770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789392525805429:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.090842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.090882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789392525805429:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:01.091659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789388230838042:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5216) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:01.091750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789392525805429:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:01.091792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789388230838042:2048] [node 1] ICP30 unregister session Session# [6:7577789392525805429:2048] VirtualId# [6:7577789388230838128:0]
2025-11-28T14:53:01.091893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.091927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789388230838042:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.091964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.100707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789385515649804:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:01.101724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789388230838042:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:01.166041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:01.166315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789388741372792:2049] [node 2] ICS07 socket disconnect 296 reason# EndOfStream
2025-11-28T14:53:01.166352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789388741372792:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.166472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:01.166525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789388741372792:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:01.169051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:01.169153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789388741372697:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30191) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:01.169200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789388741372792:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:01.169300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789389274522283:2049] [node 2] ICS07 socket disconnect 298 reason# EndOfStream
2025-11-28T14:53:01.169352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789389274522283:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.169468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:01.169513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789389274522283:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:01.170662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789389274522193:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30191) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:01.170487Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789388741372697:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30191) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:01.170609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789388741372792:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:01.170651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789388741372697:2049] [node 2] ICP30 unregister session Session# [3:7577789388741372792:2049] VirtualId# [3:7577789388741372787:0]
2025-11-28T14:53:01.170870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789389274522283:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:01.170897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:01.170933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789388741372697:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.170925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789389274522193:2049] [node 2] ICP30 unregister session Session# [4:7577789389274522283:2049] VirtualId# [4:7577789389274522273:0]
2025-11-28T14:53:01.170948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.171031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:01.171012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:01.171086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789389274522193:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.171103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30191) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.171254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789389810617193:2049] [node 2] ICS01 socket: 324 reason# EndOfStream
2025-11-28T14:53:01.171345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789385515649805:2049] [node 2] ICP30 unregister session Session# [5:7577789389810617193:2049] VirtualId# [5:7577789385515649888:0]
2025-11-28T14:53:01.171402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789389810617193:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.171501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:01.171568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:01.172394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:01.172614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789392525805432:2049] [node 2] ICS01 socket: 302 reason# EndOfStream
2025-11-28T14:53:01.172654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789388230838043:2049] [node 2] ICP30 unregister session Session# [6:7577789392525805432:2049] VirtualId# [6:7577789388230838129:0]
2025-11-28T14:53:01.172693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789392525805432:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.172762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:01.172808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:01.180610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789388741372697:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:01.180600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789389274522193:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:01.240424Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:01.240708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789389274522279:2050] [node 3] ICS07 socket disconnect 277 reason# EndOfStream
2025-11-28T14:53:01.240748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789389274522279:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.240847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:01.240915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789389274522279:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:01.241708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789389274522194:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12740) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:01.241819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789389274522279:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:01.241874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789389274522194:2050] [node 3] ICP30 unregister session Session# [4:7577789389274522279:2050] VirtualId# [4:7577789389274522271:0]
2025-11-28T14:53:01.241669Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:01.241888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789389810617201:2050] [node 3] ICS01 socket: 326 reason# EndOfStream
2025-11-28T14:53:01.241924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789385515649806:2050] [node 3] ICP30 unregister session Session# [5:7577789389810617201:2050] VirtualId# [5:7577789389810617187:0]
2025-11-28T14:53:01.241958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789389810617201:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.242023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:01.242049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:01.242066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789389274522194:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12740) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.242081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12740) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.242088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:01.242830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:01.243030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789388230838124:2050] [node 3] ICS07 socket disconnect 270 reason# EndOfStream
2025-11-28T14:53:01.243064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789388230838124:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.243141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:01.243202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789388230838124:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:01.243944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789388230838044:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12740) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:01.244053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789388230838124:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:01.244104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789388230838044:2050] [node 3] ICP30 unregister session Session# [6:7577789388230838124:2050] VirtualId# [6:7577789388230838121:0]
2025-11-28T14:53:01.244146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:01.244178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789388230838044:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12740) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.244192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12740) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.254325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789388230838044:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:01.304919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:01.305094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789389810617198:2108] [node 4] ICS07 socket disconnect 329 reason# EndOfStream
2025-11-28T14:53:01.305124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789389810617198:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.305196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:01.305234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789389810617198:2108] [node 4] ICS15 start handshake
2025-11-28T14:53:01.305829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789385515649868:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31594) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:01.305909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789389810617198:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:01.305942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789385515649868:2108] [node 4] ICP30 unregister session Session# [5:7577789389810617198:2108] VirtualId# [5:7577789389810617188:0]
2025-11-28T14:53:01.306019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:01.306046Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789385515649868:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.306056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.306164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:01.306367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789392525805437:2107] [node 4] ICS07 socket disconnect 331 reason# EndOfStream
2025-11-28T14:53:01.306398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789392525805437:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.306460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:01.306504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789392525805437:2107] [node 4] ICS15 start handshake
2025-11-28T14:53:01.307160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789388230838106:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31594) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:01.307280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789392525805437:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:01.307328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789388230838106:2107] [node 4] ICP30 unregister session Session# [6:7577789392525805437:2107] VirtualId# [6:7577789392525805430:0]
2025-11-28T14:53:01.307371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:01.307401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789388230838106:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.307432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.315731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789385515649868:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:53:01.317785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789388230838106:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:53:01.388760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:01.388988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789392525805440:2106] [node 5] ICS01 socket: 287 reason# EndOfStream
2025-11-28T14:53:01.389037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789388230838104:2106] [node 5] ICP30 unregister session Session# [6:7577789392525805440:2106] VirtualId# [6:7577789392525805436:0]
2025-11-28T14:53:01.389074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789392525805440:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.389175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:01.389225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:01.520788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789391945200865:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:01.521303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789391945200866:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:01.540122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789392786359293:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:01.540184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789392786359294:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:01.562878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789391496690405:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:01.562918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789391496690406:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:01.584817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389888219849:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:01.584943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389888219850:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:01.585140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389888219851:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:01.615480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789390160247721:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:01.615518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789390160247723:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:01.615733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789390160247722:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:01.637352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789391978218519:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:01.637408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789391978218521:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:01.637438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789391978218520:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:01.730638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789392786359357:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:01.730676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789391945200928:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:01.731592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789391496690468:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:01.731875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389888219912:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:01.731565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789391945200929:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:01.731152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789392786359358:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:01.732273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789391978218580:2106] [node 4] ICP01 ready to work
2025-11-28T14:53:01.732413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789390160247784:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:01.732783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789389888219913:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:01.731636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789391945200930:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:01.732694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789390160247785:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:01.732530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789391496690469:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:01.732420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789392786359359:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:01.732848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789391978218582:2107] [node 5] ICP01 ready to work
2025-11-28T14:53:01.734758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789391496690470:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:01.746618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789389888219851:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40752534ns) {Configure:104(40741918ns) {ProcessConfigured:135(869029ns) {StartInitialHandshake:157(867929ns) {DropHandshakes:470(1168ns) {DropIncomingHandshake:442(285ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(328ns) PrepareNewSessionHandshake:477(1167ns) {SwitchToState:179(72ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.752327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789391945200944:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22112
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.754324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789391496690481:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.755822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789392786359370:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6995
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.755887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789391496690481:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.756470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789390160247800:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55423
2025-11-28T14:53:01.756150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789391945200945:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22112
2025-11-28T14:53:01.755829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789392786359371:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6995
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.757133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789390160247797:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55423
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.757677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789391496690485:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.756532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789391945200946:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22112
2025-11-28T14:53:01.758485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389888219924:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16996
2025-11-28T14:53:01.760786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789391496690405:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.756188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789392786359375:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6995
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.757564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789390160247801:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55423
2025-11-28T14:53:01.756650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200944:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.763278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389888219927:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16996
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.763897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789391496690486:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
2025-11-28T14:53:01.756669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789392786359371:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.763294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200935:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.757572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789390160247802:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55423
2025-11-28T14:53:01.758527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789392786359370:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.765020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789391496690485:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.764915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389888219929:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16996
2025-11-28T14:53:01.763343Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789391945200935:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.762647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789390160247801:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.759145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789392786359293:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.765408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789391496690406:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.763282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789390160247800:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.765028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389888219924:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.764922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200936:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.763308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789392786359375:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.765873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389888219927:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.766136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789392786359376:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6995
2025-11-28T14:53:01.764946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789391945200936:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.766252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789392786359365:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.766197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789391496690486:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.765867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200946:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.763348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789390160247802:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.766273Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789392786359365:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.768103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789389888219850:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.766453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200937:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.766269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789391496690479:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.767637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789392786359366:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.766296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789391496690479:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.763401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789390160247797:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.767682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789392786359366:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.766469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789391945200937:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.764980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789390160247722:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.768391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389888219929:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.768357Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789391496690479:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:01.766468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200945:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.768117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789392786359365:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:01.770005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789391496690470:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.765411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789390160247723:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.766938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789391945200947:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22112
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.768152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789392786359376:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.766998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789391945200935:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:01.770050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:01.766136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789390160247721:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.775055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218597:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.775030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789390160247722:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789390160247801:2117] held: no
2025-11-28T14:53:01.768652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789392786359367:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.769777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789391945200936:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:01.770248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789391496690470:2108] [node 5] ICP22 created new session: [3:7577789391496690492:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.768679Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789392786359367:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.771159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789391496690491:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.770007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789391945200937:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.776112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789391978218597:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.777024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789390160247721:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24662) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789390160247797:2115] held: no
2025-11-28T14:53:01.770407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789392786359366:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:01.770015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200947:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:53:01.770066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789391945200866:2049] [node 3] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:53:01.770755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789392786359358:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.777050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789391978218521:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.777376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789390160247807:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55423
2025-11-28T14:53:01.770397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.771371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789391496690492:2108] [node 5] ICS09 handshake done sender: [3:7577789391496690479:2113] self: [3:7577789391496690473:0] peer: [5:7577789390160247805:0] socket: 290 qp: -1
2025-11-28T14:53:01.777607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389888219933:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16996
2025-11-28T14:53:01.770798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.777539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789390160247808:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55423
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.770837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789392786359358:2108] [node 5] ICP22 created new session: [2:7577789392786359385:2108]
2025-11-28T14:53:01.770445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789391945200866:2049] [node 3] ICP22 created new session: [1:7577789391945200954:2049]
2025-11-28T14:53:01.772419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789391496690492:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:01.771599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789392786359367:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:01.778301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789389888219850:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789389888219927:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.772109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789392786359385:2108] [node 5] ICS09 handshake done sender: [2:7577789392786359365:2111] self: [2:7577789392786359360:0] peer: [5:7577789390160247803:0] socket: 284 qp: -1
2025-11-28T14:53:01.771394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789391945200954:2049] [node 3] ICS09 handshake done sender: [1:7577789391945200935:2110] self: [1:7577789391945200931:0] peer: [3:7577789391496690487:0] socket: 271 qp: -1
2025-11-28T14:53:01.772479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:01.772711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789392786359385:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:01.771534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789391945200865:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.771566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:01.782007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218599:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.780996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789390160247809:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55423
2025-11-28T14:53:01.771664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789391945200930:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.775087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789391496690405:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24662) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789391496690481:2115] held: no
2025-11-28T14:53:01.773345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.772123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789391945200865:2048] [node 2] ICP22 created new session: [1:7577789391945200955:2048]
2025-11-28T14:53:01.773681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789392786359294:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.781034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789390160247801:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:01.777387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789391496690478:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.783291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789391978218599:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.783191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389888219935:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16996
2025-11-28T14:53:01.781507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789390160247802:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:01.772154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.775272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789392786359357:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.783897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789391978218519:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.783843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389888219934:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16996
2025-11-28T14:53:01.777411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789391496690478:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.772703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789391945200954:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:01.775519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:01.785063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389888219934:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.782892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789390160247722:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.786030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218600:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.775712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:01.772394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789391945200930:2108] [node 5] ICP22 created new session: [1:7577789391945200957:2108]
2025-11-28T14:53:01.785905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789389888219849:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.777538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789391496690494:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.775757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789392786359294:2049] [node 3] ICP22 created new session: [2:7577789392786359388:2049]
2025-11-28T14:53:01.780898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789391496690406:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789391496690485:2116] held: no
2025-11-28T14:53:01.782922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:01.775717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789392786359357:2107] [node 4] ICP22 created new session: [2:7577789392786359389:2107]
2025-11-28T14:53:01.776016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789392786359388:2049] [node 3] ICS09 handshake done sender: [2:7577789392786359366:2112] self: [2:7577789392786359362:0] peer: [3:7577789391496690488:0] socket: 288 qp: -1
2025-11-28T14:53:01.783284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789390160247722:2049] [node 2] ICP22 created new session: [5:7577789390160247811:2049]
2025-11-28T14:53:01.786431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389888219935:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.782080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789391496690481:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:01.773007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789391945200955:2048] [node 2] ICS09 handshake done sender: [1:7577789391945200936:2111] self: [1:7577789391945200933:0] peer: [2:7577789392786359377:0] socket: 270 qp: -1
2025-11-28T14:53:01.776172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789392786359388:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:01.776909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:01.787083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218601:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.777235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789392786359389:2107] [node 4] ICS09 handshake done sender: [2:7577789392786359367:2113] self: [2:7577789392786359361:0] peer: [4:7577789389888219930:0] socket: 287 qp: -1
2025-11-28T14:53:01.786553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389888219927:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:01.777443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789392786359389:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:01.782481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789391496690478:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:01.773049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:01.783472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789390160247723:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.786789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789389888219851:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.777594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:01.783514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.781060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789392786359384:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6995
2025-11-28T14:53:01.773306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789391945200955:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:01.787207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789389888219850:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.783519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789391496690405:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.790421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789391978218597:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.773334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:01.782398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789392786359370:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:01.784057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789390160247811:2049] [node 2] ICS09 handshake done sender: [5:7577789390160247801:2117] self: [5:7577789390160247803:0] peer: [2:7577789392786359360:0] socket: 286 qp: -1
2025-11-28T14:53:01.783553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:01.787232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:01.791437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218603:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.775236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789391945200957:2108] [node 5] ICS09 handshake done sender: [1:7577789391945200937:2112] self: [1:7577789391945200932:0] peer: [5:7577789390160247806:0] socket: 272 qp: -1
2025-11-28T14:53:01.787262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389888219850:2049] [node 2] ICP22 created new session: [4:7577789389888219939:2049]
2025-11-28T14:53:01.784022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789392786359293:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.784272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789390160247723:2050] [node 3] ICP22 created new session: [5:7577789390160247812:2050]
2025-11-28T14:53:01.784035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789391496690469:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.775260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789391945200957:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:01.788095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789389888219939:2049] [node 2] ICS09 handshake done sender: [4:7577789389888219927:2116] self: [4:7577789389888219930:0] peer: [2:7577789392786359361:0] socket: 302 qp: -1
2025-11-28T14:53:01.791638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218604:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.784286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789391496690405:2048] [node 1] ICP22 created new session: [3:7577789391496690498:2048]
2025-11-28T14:53:01.784068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:01.784629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789390160247811:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:01.788312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789389888219939:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:01.791919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789391978218521:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.784641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789392786359293:2048] [node 1] ICP22 created new session: [2:7577789392786359393:2048]
2025-11-28T14:53:01.775296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:01.784435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:01.788788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:01.784660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:01.782355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789391945200963:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22112
2025-11-28T14:53:01.791972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:01.785045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789392786359393:2048] [node 1] ICS09 handshake done sender: [2:7577789392786359370:2116] self: [2:7577789392786359377:0] peer: [1:7577789391945200933:0] socket: 276 qp: -1
2025-11-28T14:53:01.784711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789390160247812:2050] [node 3] ICS09 handshake done sender: [5:7577789390160247802:2118] self: [5:7577789390160247805:0] peer: [3:7577789391496690473:0] socket: 298 qp: -1
2025-11-28T14:53:01.784702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789391496690498:2048] [node 1] ICS09 handshake done sender: [3:7577789391496690481:2115] self: [3:7577789391496690487:0] peer: [1:7577789391945200931:0] socket: 274 qp: -1
2025-11-28T14:53:01.792273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789391978218603:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.785111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200940:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.793361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389888219922:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.784732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789391496690498:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:01.792301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789391978218521:2050] [node 3] ICP22 created new session: [6:7577789391978218605:2050]
2025-11-28T14:53:01.793387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789389888219922:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.785133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789391945200940:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.785126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789390160247812:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:01.786043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789392786359393:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:01.795887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789389888219851:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15386) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789389888219935:2120] held: no
2025-11-28T14:53:01.784931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789391496690469:2109] [node 6] ICP22 created new session: [3:7577789391496690499:2109]
2025-11-28T14:53:01.792711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789391978218605:2050] [node 3] ICS09 handshake done sender: [6:7577789391978218597:2115] self: [6:7577789391978218598:0] peer: [3:7577789391496690472:0] socket: 326 qp: -1
2025-11-28T14:53:01.785160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:01.798162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389888219922:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:01.786421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200963:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.792750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789391978218605:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:01.786077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.784949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:01.785859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789390160247797:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:01.792887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:01.786532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789391945200940:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:01.797749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789392786359358:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17849997ns) {HandleHandshakeStatus:329(17847723ns) {DropIncomingHandshake:442(9155ns) DropOutgoingHandshake:457(186ns) SwitchToState:179(767ns) ProcessPendingSessionEvents:495(12074839ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11983627ns max# 1462813ns}}}
2025-11-28T14:53:01.793576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789391978218600:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.787276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789391945200939:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.798814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389888219941:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16996
2025-11-28T14:53:01.793593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789391978218519:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24662) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789391978218599:2116] held: no
2025-11-28T14:53:01.786092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789391496690499:2109] [node 6] ICS09 handshake done sender: [3:7577789391496690478:2112] self: [3:7577789391496690472:0] peer: [6:7577789391978218598:0] socket: 324 qp: -1
2025-11-28T14:53:01.786871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789390160247721:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.787296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789391945200939:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.794815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789391978218599:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:01.786260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789391496690499:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:01.788914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789391945200929:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.786917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:01.796956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789391978218582:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.800267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789389888219849:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24662) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789389888219934:2119] held: no
2025-11-28T14:53:01.788962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:01.786452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789391496690485:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:01.798374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789391978218520:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.787224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789390160247721:2048] [node 1] ICP22 created new session: [5:7577789390160247817:2048]
2025-11-28T14:53:01.800286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789389888219942:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16996
2025-11-28T14:53:01.789097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789391945200929:2109] [node 6] ICP22 created new session: [1:7577789391945200965:2109]
2025-11-28T14:53:01.786594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:01.802603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789391978218519:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.800293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789389888219912:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.790764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789391945200965:2109] [node 6] ICS09 handshake done sender: [1:7577789391945200940:2114] self: [1:7577789391945200938:0] peer: [6:7577789391978218602:0] socket: 277 qp: -1
2025-11-28T14:53:01.803122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789392786359369:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.788280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789390160247817:2048] [node 1] ICS09 handshake done sender: [5:7577789390160247797:2115] self: [5:7577789390160247806:0] peer: [1:7577789391945200932:0] socket: 275 qp: -1
2025-11-28T14:53:01.788920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789390160247817:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.802639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:01.789034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789390160247816:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55423
2025-11-28T14:53:01.788055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789391496690406:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.790446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:01.802953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789391978218519:2048] [node 1] ICP22 created new session: [6:7577789391978218611:2048]
2025-11-28T14:53:01.791279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789391945200965:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:01.788714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:01.791914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789390160247816:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.803137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789391978218611:2048] [node 1] ICS09 handshake done sender: [6:7577789391978218599:2116] self: [6:7577789391978218602:0] peer: [1:7577789391945200938:0] socket: 280 qp: -1
2025-11-28T14:53:01.788755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789391496690406:2049] [node 2] ICP22 created new session: [3:7577789391496690503:2049]
2025-11-28T14:53:01.788878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789391496690480:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.791932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:01.803167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789391978218611:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:01.792483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789390160247785:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.788912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789391496690480:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.800342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:01.804719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:01.792784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789391945200866:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(16199942ns) {HandleHandshakeStatus:329(16198220ns) {DropIncomingHandshake:442(11462ns) DropOutgoingHandshake:457(309ns) SwitchToState:179(423ns) ProcessPendingSessionEvents:495(10919893ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10854277ns max# 1591505ns}}}
2025-11-28T14:53:01.790767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789391496690503:2049] [node 2] ICS09 handshake done sender: [3:7577789391496690485:2116] self: [3:7577789391496690488:0] peer: [2:7577789392786359362:0] socket: 289 qp: -1
2025-11-28T14:53:01.803155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789392786359369:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.798286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789390160247795:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.793011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789391945200939:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:01.791275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789391496690501:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
2025-11-28T14:53:01.795943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789391945200865:2048] [node 2] ICP18 handshake (actor [1:7577789391945200963:2119]) is too old
2025-11-28T14:53:01.798309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789390160247795:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.804637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789392786359395:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6995
2025-11-28T14:53:01.791449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789391496690503:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:01.796114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789391945200928:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.791993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:01.805556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789392786359294:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(21474094ns) {HandleHandshakeStatus:329(21472494ns) {DropIncomingHandshake:442(9375ns) DropOutgoingHandshake:457(243ns) SwitchToState:179(126ns) ProcessPendingSessionEvents:495(17346504ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16828564ns max# 3384416ns}}}
2025-11-28T14:53:01.802606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789390160247820:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55423
2025-11-28T14:53:01.801876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389888219912:2108] [node 5] ICP22 created new session: [4:7577789389888219945:2108]
2025-11-28T14:53:01.791998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789391496690480:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:01.805701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789392786359369:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:01.802661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789390160247795:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:01.793365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789391496690468:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.796160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:01.804381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789390160247816:2122] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-28T14:53:01.802652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789389888219945:2108] [node 5] ICS09 handshake done sender: [4:7577789389888219922:2113] self: [4:7577789389888219916:0] peer: [5:7577789390160247819:0] socket: 306 qp: -1
::ffff:127.1.0.0
2025-11-28T14:53:01.793406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:01.805524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789390160247784:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.796209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789391945200928:2107] [node 4] ICP22 created new session: [1:7577789391945200968:2107]
2025-11-28T14:53:01.802815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789389888219945:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:01.804333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:01.805578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:01.794704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789391496690468:2107] [node 4] ICP22 created new session: [3:7577789391496690506:2107]
2025-11-28T14:53:01.805673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389888219935:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:01.796258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789391945200968:2107] [node 4] ICS09 handshake done sender: [1:7577789391945200939:2113] self: [1:7577789391945200934:0] peer: [4:7577789389888219937:0] socket: 295 qp: -1
2025-11-28T14:53:01.806279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218610:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.806291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789392786359395:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.795554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789391496690506:2107] [node 4] ICS09 handshake done sender: [3:7577789391496690480:2114] self: [3:7577789391496690477:0] peer: [4:7577789389888219938:0] socket: 329 qp: -1
2025-11-28T14:53:01.806207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789389888219851:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.805627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789390160247784:2109] [node 6] ICP22 created new session: [5:7577789390160247822:2109]
2025-11-28T14:53:01.796288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789391945200968:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:01.806324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218612:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.806341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789392786359357:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21722835ns) {HandleHandshakeStatus:329(21721097ns) {DropIncomingHandshake:442(12555ns) DropOutgoingHandshake:457(435ns) SwitchToState:179(223ns) ProcessPendingSessionEvents:495(15962858ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14524370ns max# 1386516ns}}}
2025-11-28T14:53:01.796337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:01.795602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789391496690506:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:01.795944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:01.806331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389888219934:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:01.805907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789390160247822:2109] [node 6] ICS09 handshake done sender: [5:7577789390160247795:2114] self: [5:7577789390160247788:0] peer: [6:7577789391978218607:0] socket: 334 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.797534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789391496690470:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17419752ns) {HandleHandshakeStatus:329(17417764ns) {DropIncomingHandshake:442(362ns) DropOutgoingHandshake:457(658ns) SwitchToState:179(223ns) ProcessPendingSessionEvents:495(13892402ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13184521ns max# 1724128ns}}}
2025-11-28T14:53:01.805944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789390160247822:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:01.806893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789392786359359:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.807028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789389888219849:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.807118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789391978218603:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:01.806130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:01.807383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:01.807061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:01.808140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789392786359359:2109] [node 6] ICP22 created new session: [2:7577789392786359398:2109]
2025-11-28T14:53:01.807382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218614:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.809255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789392786359398:2109] [node 6] ICS09 handshake done sender: [2:7577789392786359369:2115] self: [2:7577789392786359364:0] peer: [6:7577789391978218609:0] socket: 278 qp: -1
2025-11-28T14:53:01.807499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789391978218582:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.809296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789392786359398:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:01.808216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:01.807345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389888219851:2050] [node 3] ICP22 created new session: [4:7577789389888219947:2050]
2025-11-28T14:53:01.806673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789390160247785:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.808450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789391978218582:2107] [node 5] ICP22 created new session: [6:7577789391978218615:2107]
2025-11-28T14:53:01.807744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789389888219947:2050] [node 3] ICS09 handshake done sender: [4:7577789389888219935:2120] self: [4:7577789389888219938:0] peer: [3:7577789391496690477:0] socket: 330 qp: -1
2025-11-28T14:53:01.809792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:01.807680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:01.811699Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789392786359293:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18531831ns) {HandleHandshakeStatus:329(18529928ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(2358ns) SwitchToState:179(163ns) ProcessPendingSessionEvents:495(15707928ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15597556ns max# 1237930ns}}}
2025-11-28T14:53:01.808137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789390160247785:2108] [node 4] ICP22 created new session: [5:7577789390160247824:2108]
2025-11-28T14:53:01.809690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789391978218615:2107] [node 5] ICS09 handshake done sender: [6:7577789391978218603:2119] self: [6:7577789391978218607:0] peer: [5:7577789390160247788:0] socket: 336 qp: -1
2025-11-28T14:53:01.808353Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789390160247723:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(18108717ns) {HandleHandshakeStatus:329(18106535ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(4568ns) SwitchToState:179(444ns) ProcessPendingSessionEvents:495(14708871ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14607980ns max# 1650543ns}}}
2025-11-28T14:53:01.810283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789391978218615:2107] [node 5] ICS10 traffic start
2025-11-28T14:53:01.808500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789390160247824:2108] [node 4] ICS09 handshake done sender: [5:7577789390160247816:2122] self: [5:7577789390160247819:0] peer: [4:7577789389888219916:0] socket: 333 qp: -1
2025-11-28T14:53:01.811399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:01.808223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789389888219947:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:01.809170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789390160247824:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:01.812090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789391978218520:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789391978218600:2117] held: no
2025-11-28T14:53:01.812982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789391978218610:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.809378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:01.808258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:01.808492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389888219849:2048] [node 1] ICP22 created new session: [4:7577789389888219948:2048]
2025-11-28T14:53:01.813200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789391978218600:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:01.813786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789391496690469:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(21601567ns) {HandleHandshakeStatus:329(21599257ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(316ns) SwitchToState:179(294ns) ProcessPendingSessionEvents:495(17582230ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16790193ns max# 1660877ns}}}
2025-11-28T14:53:01.814305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789391945200929:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16717958ns) {HandleHandshakeStatus:329(16716030ns) {DropIncomingHandshake:442(13711ns) DropOutgoingHandshake:457(250ns) SwitchToState:179(337ns) ProcessPendingSessionEvents:495(12766702ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12674511ns max# 2211909ns}}}
2025-11-28T14:53:01.809272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789389888219948:2048] [node 1] ICS09 handshake done sender: [4:7577789389888219934:2119] self: [4:7577789389888219937:0] peer: [1:7577789391945200934:0] socket: 299 qp: -1
2025-11-28T14:53:01.809412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:01.809548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789389888219948:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:01.809852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:01.814616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789391978218580:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:01.814941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789389888219923:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:01.814984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789389888219923:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:01.814900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789391978218520:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.815061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789390160247785:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:01.815005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:01.815051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789391978218520:2049] [node 2] ICP22 created new session: [6:7577789391978218618:2049]
2025-11-28T14:53:01.815222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789391978218618:2049] [node 2] ICS09 handshake done sender: [6:7577789391978218600:2117] self: [6:7577789391978218609:0] peer: [2:7577789392786359364:0] socket: 294 qp: -1
2025-11-28T14:53:01.815263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789391978218618:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:01.815303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:01.816724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789389888219923:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:01.816935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789389888219913:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:01.817007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789391978218620:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63509
2025-11-28T14:53:01.817009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:01.817062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789389888219913:2109] [node 6] ICP22 created new session: [4:7577789389888219952:2109]
2025-11-28T14:53:01.817112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789389888219952:2109] [node 6] ICS09 handshake done sender: [4:7577789389888219923:2114] self: [4:7577789389888219917:0] peer: [6:7577789391978218617:0] socket: 328 qp: -1
2025-11-28T14:53:01.817144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789389888219952:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:01.817293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789391978218610:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:01.817188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:01.817587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789391978218580:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25666) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789391978218610:2121] held: no
2025-11-28T14:53:01.818935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789391978218580:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:01.819001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:01.819052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789391978218580:2106] [node 4] ICP22 created new session: [6:7577789391978218621:2106]
2025-11-28T14:53:01.819102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789391978218621:2106] [node 4] ICS09 handshake done sender: [6:7577789391978218610:2121] self: [6:7577789391978218617:0] peer: [4:7577789389888219917:0] socket: 332 qp: -1
2025-11-28T14:53:01.819129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789391978218621:2106] [node 4] ICS10 traffic start
2025-11-28T14:53:01.819167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:01.855684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:01.855947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789392786359393:2048] [node 1] ICS07 socket disconnect 276 reason# EndOfStream
2025-11-28T14:53:01.855989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789392786359393:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.856111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.856175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789392786359393:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:01.856768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:01.856989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789391496690498:2048] [node 1] ICS01 socket: 274 reason# EndOfStream
2025-11-28T14:53:01.857045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789391496690405:2048] [node 1] ICP30 unregister session Session# [3:7577789391496690498:2048] VirtualId# [3:7577789391496690487:0]
2025-11-28T14:53:01.857093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789391496690498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.857175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.857230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.858495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789392786359293:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24662) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:01.858537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789392786359393:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:01.859217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:01.859447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789389888219948:2048] [node 1] ICS01 socket: 299 reason# EndOfStream
2025-11-28T14:53:01.859492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789389888219849:2048] [node 1] ICP30 unregister session Session# [4:7577789389888219948:2048] VirtualId# [4:7577789389888219937:0]
2025-11-28T14:53:01.859527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789392786359293:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24662) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:01.859578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789392786359393:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:01.859612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789392786359293:2048] [node 1] ICP30 unregister session Session# [2:7577789392786359393:2048] VirtualId# [2:7577789392786359377:0]
2025-11-28T14:53:01.859531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789389888219948:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.859874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.859902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789392786359293:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24662) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.859614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.859921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24662) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:01.859679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.860106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:01.860260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789390160247817:2048] [node 1] ICS01 socket: 275 reason# EndOfStream
2025-11-28T14:53:01.860292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789390160247721:2048] [node 1] ICP30 unregister session Session# [5:7577789390160247817:2048] VirtualId# [5:7577789390160247806:0]
2025-11-28T14:53:01.860311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789390160247817:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.860380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.860441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.861052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:01.861267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789391978218611:2048] [node 1] ICS01 socket: 280 reason# EndOfStream
2025-11-28T14:53:01.861317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789391978218519:2048] [node 1] ICP30 unregister session Session# [6:7577789391978218611:2048] VirtualId# [6:7577789391978218602:0]
2025-11-28T14:53:01.861371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789391978218611:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.861459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:01.861522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:01.931746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:01.931920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789391496690503:2049] [node 2] ICS01 socket: 289 reason# EndOfStream
2025-11-28T14:53:01.931965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789391496690406:2049] [node 2] ICP30 unregister session Session# [3:7577789391496690503:2049] VirtualId# [3:7577789391496690488:0]
2025-11-28T14:53:01.931989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789391496690503:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.932062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:01.932125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:01.935286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:01.935510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789389888219939:2049] [node 2] ICS01 socket: 302 reason# EndOfStream
2025-11-28T14:53:01.935569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789389888219850:2049] [node 2] ICP30 unregister session Session# [4:7577789389888219939:2049] VirtualId# [4:7577789389888219930:0]
2025-11-28T14:53:01.935602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789389888219939:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.935700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:01.935755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:01.936212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:01.936435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789390160247811:2049] [node 2] ICS01 socket: 286 reason# EndOfStream
2025-11-28T14:53:01.936477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789390160247722:2049] [node 2] ICP30 unregister session Session# [5:7577789390160247811:2049] VirtualId# [5:7577789390160247803:0]
2025-11-28T14:53:01.936507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789390160247811:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.936590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:01.936631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:01.937383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:01.937651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789391978218618:2049] [node 2] ICS01 socket: 294 reason# EndOfStream
2025-11-28T14:53:01.937708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789391978218520:2049] [node 2] ICP30 unregister session Session# [6:7577789391978218618:2049] VirtualId# [6:7577789391978218609:0]
2025-11-28T14:53:01.937739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789391978218618:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:01.937832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:01.937885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:02.015515Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:02.015765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789389888219947:2050] [node 3] ICS01 socket: 330 reason# EndOfStream
2025-11-28T14:53:02.015815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789389888219851:2050] [node 3] ICP30 unregister session Session# [4:7577789389888219947:2050] VirtualId# [4:7577789389888219938:0]
2025-11-28T14:53:02.015856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789389888219947:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.015958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:02.016028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:02.016492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:02.016689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789390160247812:2050] [node 3] ICS07 socket disconnect 298 reason# EndOfStream
2025-11-28T14:53:02.016722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789390160247812:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.016790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:02.016855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789390160247812:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:02.017342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:02.017510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789391978218605:2050] [node 3] ICS01 socket: 326 reason# EndOfStream
2025-11-28T14:53:02.017560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789391978218521:2050] [node 3] ICP30 unregister session Session# [6:7577789391978218605:2050] VirtualId# [6:7577789391978218598:0]
2025-11-28T14:53:02.017588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789391978218605:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.017682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:02.017626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789390160247723:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15386) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.017716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:02.017708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789390160247812:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.017752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789390160247723:2050] [node 3] ICP30 unregister session Session# [5:7577789390160247812:2050] VirtualId# [5:7577789390160247805:0]
2025-11-28T14:53:02.017916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:02.017962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789390160247723:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.017978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.027455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789390160247723:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:02.080011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:02.080253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789390160247824:2108] [node 4] ICS07 socket disconnect 333 reason# EndOfStream
2025-11-28T14:53:02.080289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789390160247824:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.080393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:02.080463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789390160247824:2108] [node 4] ICS15 start handshake
2025-11-28T14:53:02.081273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789390160247785:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25666) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.081365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789390160247824:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.081434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789390160247785:2108] [node 4] ICP30 unregister session Session# [5:7577789390160247824:2108] VirtualId# [5:7577789390160247819:0]
2025-11-28T14:53:02.081355Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:02.081586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:02.081642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789391978218621:2106] [node 4] ICS01 socket: 332 reason# EndOfStream
2025-11-28T14:53:02.081627Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789390160247785:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25666) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.081708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789391978218580:2106] [node 4] ICP30 unregister session Session# [6:7577789391978218621:2106] VirtualId# [6:7577789391978218617:0]
2025-11-28T14:53:02.081641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25666) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.081742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789391978218621:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.081852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:02.081932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:02.140195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:02.140480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789391978218615:2107] [node 5] ICS07 socket disconnect 336 reason# EndOfStream
2025-11-28T14:53:02.140536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789391978218615:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.140662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:02.140745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789391978218615:2107] [node 5] ICS15 start handshake
2025-11-28T14:53:02.141450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789391978218582:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32728) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.141506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789391978218615:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.141560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789391978218582:2107] [node 5] ICP30 unregister session Session# [6:7577789391978218615:2107] VirtualId# [6:7577789391978218607:0]
2025-11-28T14:53:02.141653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:53:02.141680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789391978218582:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.141689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:02.243247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789394184894319:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:02.243284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789394184894320:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:02.259726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789397396267077:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.259766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789397396267078:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:02.276987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394833892050:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.277021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394833892051:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:02.294653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397044522002:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.294709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397044522003:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:02.294737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397044522004:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:02.310949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789396198106614:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.310986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789396198106615:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:02.311010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789396198106616:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:02.327217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789397309444465:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.327263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789397309444466:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:02.327282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789397309444467:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:02.414420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789397396267150:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:02.414469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789394184894409:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:02.415025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394833892140:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:02.415108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789397396267151:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:02.415178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789396198106680:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:02.415169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789397309444536:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:02.415205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397044522067:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:02.415384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789394184894410:2130] [node 5] ICP01 ready to work
2025-11-28T14:53:02.415641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789396198106681:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:02.415736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397044522068:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:02.415802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394833892141:2132] [node 5] ICP01 ready to work
2025-11-28T14:53:02.416063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789394184894411:2122] [node 6] ICP01 ready to work
2025-11-28T14:53:02.416517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394833892142:2134] [node 6] ICP01 ready to work
2025-11-28T14:53:02.417821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789397309444539:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:02.417903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789397396267154:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.426148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789397396267163:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17727
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.426606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789397396267167:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17727
2025-11-28T14:53:02.426651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789397396267168:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17727
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.432030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397044522082:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46612
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.432646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397044522082:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.432637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789397396267163:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.432489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789394184894422:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
2025-11-28T14:53:02.432831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789397396267167:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.432901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789397044522003:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.432823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789394184894423:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.433069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789394184894424:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
2025-11-28T14:53:02.433094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789397396267169:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17727
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.433195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789397396267158:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.433197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789394184894425:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
2025-11-28T14:53:02.433218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789397396267158:2122] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.433457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894422:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.433501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394833892153:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38
2025-11-28T14:53:02.433574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894423:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.433690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394833892154:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.433801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789397396267168:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.433822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394833892155:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.433911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894425:2145] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.433947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394833892156:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.434324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894424:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.434325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789397396267172:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17727
2025-11-28T14:53:02.434426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394833892153:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.434469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789397396267158:2122] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.434596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394833892154:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.434670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789394184894429:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
2025-11-28T14:53:02.434609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397044522084:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.434784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394833892155:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.434798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397044522085:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46612
2025-11-28T14:53:02.434819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789397396267150:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.434896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394833892156:2145] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.434876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.434932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789397396267150:2107] [node 4] ICP22 created new session: [2:7577789397396267175:2107]
2025-11-28T14:53:02.434991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789397396267175:2107] [node 4] ICS09 handshake done sender: [2:7577789397396267158:2122] self: [2:7577789397396267157:0] peer: [4:7577789397044522083:0] socket: 287 qp: -1
2025-11-28T14:53:02.435023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789397396267175:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:02.435059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.435229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394833892159:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38
2025-11-28T14:53:02.435353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397044522082:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.435598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789397044522003:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.435656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.435691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397044522003:2049] [node 2] ICP22 created new session: [4:7577789397044522087:2049]
2025-11-28T14:53:02.435721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397044522087:2049] [node 2] ICS09 handshake done sender: [4:7577789397044522082:2117] self: [4:7577789397044522083:0] peer: [2:7577789397396267157:0] socket: 295 qp: -1
2025-11-28T14:53:02.435709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789394833892051:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.435744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397044522087:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:02.435766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.437251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789396198106695:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45552
2025-11-28T14:53:02.438129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789397396267156:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.438153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789397396267156:2121] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.439777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397044522086:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.440333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789396198106696:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45552
2025-11-28T14:53:02.440770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789397396267169:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.440880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789397396267156:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:02.441713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397044522084:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.441727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789396198106696:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.441716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789397396267078:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.441750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:02.441789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789397396267078:2049] [node 3] ICP22 created new session: [2:7577789397396267179:2049]
2025-11-28T14:53:02.441812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789397044522002:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.441842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789396198106695:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.441823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789397396267179:2049] [node 3] ICS09 handshake done sender: [2:7577789397396267156:2121] self: [2:7577789397396267155:0] peer: [3:7577789394833892163:0] socket: 286 qp: -1
2025-11-28T14:53:02.441842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789397396267179:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:02.441868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:02.441988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789396198106680:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.442037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789396198106615:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.442027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894417:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.442072Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789394184894417:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.442548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397044522076:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.442571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789397044522076:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.443125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789394184894417:2139] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.443488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397044522091:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.443578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789397396267160:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.443600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789397396267160:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.443663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789396198106699:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45552
2025-11-28T14:53:02.443666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789394184894409:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.444027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.444082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789394184894409:2107] [node 4] ICP22 created new session: [1:7577789394184894435:2107]
2025-11-28T14:53:02.444089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789397396267077:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.444110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397044522076:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:02.444130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789394184894435:2107] [node 4] ICS09 handshake done sender: [1:7577789394184894417:2139] self: [1:7577789394184894416:0] peer: [4:7577789397044522090:0] socket: 311 qp: -1
2025-11-28T14:53:02.444172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789394184894435:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:02.444223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:02.444281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789397396267172:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.444469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397044522086:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.444510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789397044522067:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.444457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394833892159:2146] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.444534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789396198106696:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:02.444570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.444614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397044522067:2108] [node 5] ICP22 created new session: [4:7577789397044522093:2108]
2025-11-28T14:53:02.444677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397044522093:2108] [node 5] ICS09 handshake done sender: [4:7577789397044522076:2115] self: [4:7577789397044522075:0] peer: [5:7577789396198106697:0] socket: 321 qp: -1
2025-11-28T14:53:02.444671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789397396267078:2049] [node 3] ICP18 handshake (actor [2:7577789397396267172:2129]) is too old
2025-11-28T14:53:02.444705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397044522093:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:02.444762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.444666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394833892165:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38
2025-11-28T14:53:02.444981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789396198106680:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.445034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.445065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789396198106680:2108] [node 4] ICP22 created new session: [5:7577789396198106701:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.445100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789396198106701:2108] [node 4] ICS09 handshake done sender: [5:7577789396198106696:2119] self: [5:7577789396198106697:0] peer: [4:7577789397044522075:0] socket: 322 qp: -1
2025-11-28T14:53:02.445150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789396198106701:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:02.445176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:02.445190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789397396267160:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:02.445232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444551:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
2025-11-28T14:53:02.445508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789397396267151:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.445574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.445599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789397396267151:2108] [node 5] ICP22 created new session: [2:7577789397396267184:2108]
2025-11-28T14:53:02.445647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789397396267184:2108] [node 5] ICS09 handshake done sender: [2:7577789397396267160:2123] self: [2:7577789397396267159:0] peer: [5:7577789396198106698:0] socket: 289 qp: -1
2025-11-28T14:53:02.445681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789397396267184:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:02.445702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.446204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894429:2146] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.446461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397044522084:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:02.446685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789394833892051:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789394833892155:2144] held: no
2025-11-28T14:53:02.446888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894413:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.446915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789394184894413:2137] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.448434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789397044522002:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.448486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789394833892050:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.448531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:02.448596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397044522002:2048] [node 1] ICP22 created new session: [4:7577789397044522095:2048]
2025-11-28T14:53:02.448602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394833892155:2144] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.448632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397044522095:2048] [node 1] ICS09 handshake done sender: [4:7577789397044522084:2118] self: [4:7577789397044522090:0] peer: [1:7577789394184894416:0] socket: 313 qp: -1
2025-11-28T14:53:02.448797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397044522095:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.448825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:02.449278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789397309444551:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.449382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789397396267186:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17727
2025-11-28T14:53:02.449391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789397309444466:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.449478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789394184894413:2137] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.449516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789397044522002:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:02.449597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789394833892051:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.449647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.450188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394833892051:2049] [node 2] ICP22 created new session: [3:7577789394833892167:2049]
2025-11-28T14:53:02.450242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394833892167:2049] [node 2] ICS09 handshake done sender: [3:7577789394833892155:2144] self: [3:7577789394833892163:0] peer: [2:7577789397396267155:0] socket: 302 qp: -1
2025-11-28T14:53:02.450305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789397396267162:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.450281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394833892167:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:02.450328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789397396267162:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.450322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:02.450694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894415:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.450725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789394184894415:2138] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.450758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789397396267169:2128] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.451391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789396198106700:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45552
2025-11-28T14:53:02.451458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789394184894319:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.451548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.451597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789394184894319:2048] [node 2] ICP22 created new session: [1:7577789394184894440:2048]
2025-11-28T14:53:02.451650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789394184894440:2048] [node 2] ICS09 handshake done sender: [1:7577789394184894413:2137] self: [1:7577789394184894412:0] peer: [2:7577789397396267181:0] socket: 304 qp: -1
2025-11-28T14:53:02.451674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789397396267077:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.451687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789394184894440:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:02.451767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:02.451801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:02.451842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789397396267077:2048] [node 1] ICP22 created new session: [2:7577789397396267188:2048]
2025-11-28T14:53:02.451865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789397396267188:2048] [node 1] ICS09 handshake done sender: [2:7577789397396267169:2128] self: [2:7577789397396267181:0] peer: [1:7577789394184894412:0] socket: 306 qp: -1
2025-11-28T14:53:02.451890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789397396267188:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.451907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.452063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444553:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
2025-11-28T14:53:02.452088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789396198106615:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789396198106695:2118] held: no
2025-11-28T14:53:02.453166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789396198106695:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.453155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789397044522004:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.453629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789396198106615:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.453710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.453747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789396198106615:2049] [node 2] ICP22 created new session: [5:7577789396198106706:2049]
2025-11-28T14:53:02.453704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444554:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
2025-11-28T14:53:02.453786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789396198106706:2049] [node 2] ICS09 handshake done sender: [5:7577789396198106695:2118] self: [5:7577789396198106698:0] peer: [2:7577789397396267159:0] socket: 320 qp: -1
2025-11-28T14:53:02.453835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789396198106706:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:02.453888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:02.453853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789397396267162:2124] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:02.454578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394833892148:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.454628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789394833892148:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.454687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789397309444551:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.455553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789397309444466:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.455607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.455643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789397309444466:2049] [node 2] ICP22 created new session: [6:7577789397309444555:2049]
2025-11-28T14:53:02.455671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789397309444555:2049] [node 2] ICS09 handshake done sender: [6:7577789397309444551:2123] self: [6:7577789397309444552:0] peer: [2:7577789397396267161:0] socket: 330 qp: -1
2025-11-28T14:53:02.455690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789397309444555:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:02.455733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.455864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394833892148:2139] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.455854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789397396267154:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.455925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:02.455964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397044522099:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46612
2025-11-28T14:53:02.455958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789397396267154:2109] [node 6] ICP22 created new session: [2:7577789397396267190:2109]
2025-11-28T14:53:02.455987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789397396267190:2109] [node 6] ICS09 handshake done sender: [2:7577789397396267162:2124] self: [2:7577789397396267161:0] peer: [6:7577789397309444552:0] socket: 329 qp: -1
2025-11-28T14:53:02.455994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789396198106703:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45552
2025-11-28T14:53:02.456280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789394833892140:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.456010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789397396267190:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:02.456339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.456519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397044522086:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:02.456038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:02.456532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394833892140:2107] [node 4] ICP22 created new session: [3:7577789394833892171:2107]
2025-11-28T14:53:02.456595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394833892171:2107] [node 4] ICS09 handshake done sender: [3:7577789394833892148:2139] self: [3:7577789394833892147:0] peer: [4:7577789397044522098:0] socket: 316 qp: -1
2025-11-28T14:53:02.456679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394833892171:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:02.456724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:02.456744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789397044522004:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.456771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:02.456797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397044522004:2050] [node 3] ICP22 created new session: [4:7577789397044522100:2050]
2025-11-28T14:53:02.456822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397044522100:2050] [node 3] ICS09 handshake done sender: [4:7577789397044522086:2120] self: [4:7577789397044522098:0] peer: [3:7577789394833892147:0] socket: 318 qp: -1
2025-11-28T14:53:02.456849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397044522100:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:02.456870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.457036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789396198106705:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45552
2025-11-28T14:53:02.459033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789396198106703:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.459136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789396198106614:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.459142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789394184894415:2138] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:02.459478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789397309444554:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.459525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894419:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.459554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789394184894419:2140] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.459856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444557:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.460172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789397309444536:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.460211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789394184894320:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.460238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444558:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
2025-11-28T14:53:02.460269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:02.460284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789396198106705:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.460328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789394184894320:2049] [node 3] ICP22 created new session: [1:7577789394184894444:2049]
2025-11-28T14:53:02.460343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789397309444557:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.460386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789394184894444:2049] [node 3] ICS09 handshake done sender: [1:7577789394184894415:2138] self: [1:7577789394184894414:0] peer: [3:7577789394833892166:0] socket: 305 qp: -1
2025-11-28T14:53:02.460382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789396198106616:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:53:02.460418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789394184894444:2049] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-28T14:53:02.460465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:02.460498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397044522102:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46612
2025-11-28T14:53:02.460643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397044522078:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.460670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789397044522078:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.460920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394833892150:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.460956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789394833892150:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.461752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397044522078:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:02.462891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789397309444539:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.463134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789397044522068:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.463162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:53:02.463192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397044522068:2109] [node 6] ICP22 created new session: [4:7577789397044522104:2109]
2025-11-28T14:53:02.463220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397044522104:2109] [node 6] ICS09 handshake done sender: [4:7577789397044522078:2116] self: [4:7577789397044522077:0] peer: [6:7577789397309444560:0] socket: 333 qp: -1
2025-11-28T14:53:02.463243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397044522104:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:02.463262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.463273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394833892173:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38
2025-11-28T14:53:02.463323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789396198106691:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.463335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444561:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
2025-11-28T14:53:02.463348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789396198106691:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.464133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789397309444554:2125] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.464397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444563:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
2025-11-28T14:53:02.464451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789397309444536:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.464476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.464456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397044522102:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.464569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789397309444536:2107] [node 4] ICP22 created new session: [6:7577789397309444564:2107]
2025-11-28T14:53:02.464883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789397309444564:2107] [node 4] ICS09 handshake done sender: [6:7577789397309444554:2125] self: [6:7577789397309444560:0] peer: [4:7577789397044522077:0] socket: 334 qp: -1
2025-11-28T14:53:02.464903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789397309444564:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.464938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:02.464990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789397044522068:2109] [node 6] ICP18 handshake (actor [4:7577789397044522102:2123]) is too old
2025-11-28T14:53:02.465024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789396198106710:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45552
2025-11-28T14:53:02.465377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789396198106691:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:02.465960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789396198106681:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.465986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789394184894419:2140] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:02.466026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:02.466085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789396198106681:2109] [node 6] ICP22 created new session: [5:7577789396198106713:2109]
2025-11-28T14:53:02.466134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789396198106713:2109] [node 6] ICS09 handshake done sender: [5:7577789396198106691:2117] self: [5:7577789396198106690:0] peer: [6:7577789397309444562:0] socket: 337 qp: -1
2025-11-28T14:53:02.466141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394833892150:2140] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:02.466165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789396198106713:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:02.466210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:02.466279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789394184894410:2130] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.466456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.466516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789394184894410:2130] [node 5] ICP22 created new session: [1:7577789394184894447:2130]
2025-11-28T14:53:02.466578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789394184894447:2130] [node 5] ICS09 handshake done sender: [1:7577789394184894419:2140] self: [1:7577789394184894418:0] peer: [5:7577789396198106708:0] socket: 281 qp: -1
2025-11-28T14:53:02.466583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394833892159:2146] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:02.466618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789394184894447:2130] [node 5] ICS10 traffic start
2025-11-28T14:53:02.466654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.466871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789394833892050:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789394833892159:2146] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.467253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789396198106712:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45552
2025-11-28T14:53:02.467490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789394833892050:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.467537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:02.467746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394833892050:2048] [node 1] ICP22 created new session: [3:7577789394833892176:2048]
2025-11-28T14:53:02.467869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394833892176:2048] [node 1] ICS09 handshake done sender: [3:7577789394833892159:2146] self: [3:7577789394833892166:0] peer: [1:7577789394184894414:0] socket: 307 qp: -1
2025-11-28T14:53:02.467920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394833892176:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.467949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:02.468238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789396198106703:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:02.468514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789396198106705:2123] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.468735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444566:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
2025-11-28T14:53:02.469945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789397309444558:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.470118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789394833892141:2132] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.470150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789396198106614:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.470170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789397309444557:2126] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:02.470192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:02.470179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.470246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789396198106614:2048] [node 1] ICP22 created new session: [5:7577789396198106715:2048]
2025-11-28T14:53:02.470228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394833892141:2132] [node 5] ICP22 created new session: [3:7577789394833892178:2132]
2025-11-28T14:53:02.470293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789396198106715:2048] [node 1] ICS09 handshake done sender: [5:7577789396198106703:2122] self: [5:7577789396198106708:0] peer: [1:7577789394184894418:0] socket: 312 qp: -1
2025-11-28T14:53:02.470354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394833892178:2132] [node 5] ICS09 handshake done sender: [3:7577789394833892150:2140] self: [3:7577789394833892149:0] peer: [5:7577789396198106709:0] socket: 332 qp: -1
2025-11-28T14:53:02.470364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789396198106715:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.470397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394833892178:2132] [node 5] ICS10 traffic start
2025-11-28T14:53:02.470393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:02.470415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789397309444539:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.470443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.470456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.470478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789397309444539:2108] [node 5] ICP22 created new session: [6:7577789397309444568:2108]
2025-11-28T14:53:02.470531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789397309444568:2108] [node 5] ICS09 handshake done sender: [6:7577789397309444557:2126] self: [6:7577789397309444562:0] peer: [5:7577789396198106690:0] socket: 288 qp: -1
2025-11-28T14:53:02.470555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789397309444568:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:02.470617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.470931Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789396198106614:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:02.471721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789396198106616:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.471761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:02.471788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789396198106616:2050] [node 3] ICP22 created new session: [5:7577789396198106717:2050]
2025-11-28T14:53:02.471823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789396198106717:2050] [node 3] ICS09 handshake done sender: [5:7577789396198106705:2123] self: [5:7577789396198106709:0] peer: [3:7577789394833892149:0] socket: 284 qp: -1
2025-11-28T14:53:02.471849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789396198106717:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:02.471886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:02.472108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789397309444465:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.472606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789394184894421:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.472636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789394184894421:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.473600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789394184894421:2141] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.473728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444571:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
2025-11-28T14:53:02.473941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789394184894411:2122] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.474000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:02.474064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789394184894411:2122] [node 6] ICP22 created new session: [1:7577789394184894450:2122]
2025-11-28T14:53:02.474122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789394184894450:2122] [node 6] ICS09 handshake done sender: [1:7577789394184894421:2141] self: [1:7577789394184894420:0] peer: [6:7577789397309444570:0] socket: 283 qp: -1
2025-11-28T14:53:02.474154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789394184894450:2122] [node 6] ICS10 traffic start
2025-11-28T14:53:02.474218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:02.474255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789397309444558:2127] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:02.474478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789397309444465:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.474505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:02.474529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789397309444465:2048] [node 1] ICP22 created new session: [6:7577789397309444572:2048]
2025-11-28T14:53:02.474558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789397309444572:2048] [node 1] ICS09 handshake done sender: [6:7577789397309444558:2127] self: [6:7577789397309444570:0] peer: [1:7577789394184894420:0] socket: 292 qp: -1
2025-11-28T14:53:02.474596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789397309444572:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.474646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.476515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789396198106719:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45552
2025-11-28T14:53:02.477689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789397309444563:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.477811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789397309444467:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.478057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394833892152:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.478094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789394833892152:2141] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.478916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394833892152:2141] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:02.478996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789397309444575:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12670
2025-11-28T14:53:02.479205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789394833892142:2134] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.479277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:02.479283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789397309444563:2129] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:02.479331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394833892142:2134] [node 6] ICP22 created new session: [3:7577789394833892181:2134]
2025-11-28T14:53:02.479440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789397309444467:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.479445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394833892181:2134] [node 6] ICS09 handshake done sender: [3:7577789394833892152:2141] self: [3:7577789394833892151:0] peer: [6:7577789397309444574:0] socket: 342 qp: -1
2025-11-28T14:53:02.479481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394833892181:2134] [node 6] ICS10 traffic start
2025-11-28T14:53:02.479475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:02.479504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789397309444467:2050] [node 3] ICP22 created new session: [6:7577789397309444576:2050]
2025-11-28T14:53:02.479522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:02.479526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789397309444576:2050] [node 3] ICS09 handshake done sender: [6:7577789397309444563:2129] self: [6:7577789397309444574:0] peer: [3:7577789394833892151:0] socket: 344 qp: -1
2025-11-28T14:53:02.479548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789397309444576:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:02.479570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:02.501492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:02.501651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789397396267188:2048] [node 1] ICS07 socket disconnect 306 reason# EndOfStream
2025-11-28T14:53:02.501678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789397396267188:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.501743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:02.501782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789397396267188:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:02.502574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:02.502823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789394833892176:2048] [node 1] ICS01 socket: 307 reason# EndOfStream
2025-11-28T14:53:02.502876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789394833892050:2048] [node 1] ICP30 unregister session Session# [3:7577789394833892176:2048] VirtualId# [3:7577789394833892166:0]
2025-11-28T14:53:02.502915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789394833892176:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.503030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:02.503081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:02.503859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789397396267077:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Socket error# Connection reset by peer state# SendExRequestHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
2025-11-28T14:53:02.503895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789397396267188:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:02.504378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789397396267077:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.504422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789397396267188:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.504442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789397396267077:2048] [node 1] ICP30 unregister session Session# [2:7577789397396267188:2048] VirtualId# [2:7577789397396267181:0]
2025-11-28T14:53:02.504472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:02.504489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789397396267077:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.504497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.504521Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:02.504699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789397044522095:2048] [node 1] ICS07 socket disconnect 313 reason# EndOfStream
2025-11-28T14:53:02.504722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789397044522095:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.504771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:02.504805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789397044522095:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:02.505213Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:02.505380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789397044522002:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.505396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789397309444572:2048] [node 1] ICS07 socket disconnect 292 reason# EndOfStream
2025-11-28T14:53:02.505413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789397044522095:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.505422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789397309444572:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.505433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789397044522002:2048] [node 1] ICP30 unregister session Session# [4:7577789397044522095:2048] VirtualId# [4:7577789397044522090:0]
2025-11-28T14:53:02.505463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:02.505498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789397309444572:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:02.505596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:02.505620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789397044522002:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.505628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.506009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789397309444465:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.506038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789397309444572:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.506056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789397309444465:2048] [node 1] ICP30 unregister session Session# [6:7577789397309444572:2048] VirtualId# [6:7577789397309444570:0]
2025-11-28T14:53:02.506029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:02.506121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:02.506139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789397309444465:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.506164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.506272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789396198106715:2048] [node 1] ICS07 socket disconnect 312 reason# EndOfStream
2025-11-28T14:53:02.506325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789396198106715:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.506427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:02.506482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789396198106715:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:02.507246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789396198106614:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.507299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789396198106715:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.507331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789396198106614:2048] [node 1] ICP30 unregister session Session# [5:7577789396198106715:2048] VirtualId# [5:7577789396198106708:0]
2025-11-28T14:53:02.507482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:02.507584Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789396198106614:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.507614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.514822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789397396267077:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:02.515854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789397044522002:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:02.515857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789397309444465:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:02.517911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789396198106614:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:02.548041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:02.548306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789394833892167:2049] [node 2] ICS01 socket: 302 reason# EndOfStream
2025-11-28T14:53:02.548355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789394833892051:2049] [node 2] ICP30 unregister session Session# [3:7577789394833892167:2049] VirtualId# [3:7577789394833892163:0]
2025-11-28T14:53:02.548407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789394833892167:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.548512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:02.548598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:02.551126Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:02.551397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789397044522087:2049] [node 2] ICS07 socket disconnect 295 reason# EndOfStream
2025-11-28T14:53:02.551461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789397044522087:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.551600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:02.551666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789397044522087:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:02.552010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:02.552277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789396198106706:2049] [node 2] ICS01 socket: 320 reason# EndOfStream
2025-11-28T14:53:02.552334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789396198106615:2049] [node 2] ICP30 unregister session Session# [5:7577789396198106706:2049] VirtualId# [5:7577789396198106698:0]
2025-11-28T14:53:02.552396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789396198106706:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.552480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:02.552552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:02.552714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789397044522003:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16197) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.552776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789397044522087:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.552814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789397044522003:2049] [node 2] ICP30 unregister session Session# [4:7577789397044522087:2049] VirtualId# [4:7577789397044522083:0]
2025-11-28T14:53:02.552938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:02.552983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789397044522003:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16197) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.552937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:02.553003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16197) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.553188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789397309444555:2049] [node 2] ICS01 socket: 330 reason# EndOfStream
2025-11-28T14:53:02.553239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789397309444466:2049] [node 2] ICP30 unregister session Session# [6:7577789397309444555:2049] VirtualId# [6:7577789397309444552:0]
2025-11-28T14:53:02.553293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789397309444555:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.553378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:02.553441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:02.562989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789397044522003:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:02.577431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:02.577631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789397044522100:2050] [node 3] ICS07 socket disconnect 318 reason# EndOfStream
2025-11-28T14:53:02.577673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789397044522100:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.577743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:02.577778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789397044522100:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:02.578463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:02.578488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789397044522004:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9728) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.578536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789397044522100:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.578559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789397044522004:2050] [node 3] ICP30 unregister session Session# [4:7577789397044522100:2050] VirtualId# [4:7577789397044522098:0]
2025-11-28T14:53:02.578596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:02.578616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789397044522004:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.578641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9728) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.578724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789396198106717:2050] [node 3] ICS01 socket: 284 reason# EndOfStream
2025-11-28T14:53:02.578782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789396198106616:2050] [node 3] ICP30 unregister session Session# [5:7577789396198106717:2050] VirtualId# [5:7577789396198106709:0]
2025-11-28T14:53:02.578815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789396198106717:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.578912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:02.578975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:02.579433Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:02.579716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789397309444576:2050] [node 3] ICS01 socket: 344 reason# EndOfStream
2025-11-28T14:53:02.579768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789397309444467:2050] [node 3] ICP30 unregister session Session# [6:7577789397309444576:2050] VirtualId# [6:7577789397309444574:0]
2025-11-28T14:53:02.579804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789397309444576:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.579927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:02.579981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:02.622812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:02.623057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789396198106701:2108] [node 4] ICS07 socket disconnect 322 reason# EndOfStream
2025-11-28T14:53:02.623085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789396198106701:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.623200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:02.623265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789396198106701:2108] [node 4] ICS15 start handshake
2025-11-28T14:53:02.623682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:02.623883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789397309444564:2107] [node 4] ICS07 socket disconnect 334 reason# EndOfStream
2025-11-28T14:53:02.623925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789397309444564:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.624040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:02.624104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789397309444564:2107] [node 4] ICS15 start handshake
2025-11-28T14:53:02.624132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789396198106680:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5302) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.624188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789396198106701:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.624231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789396198106680:2108] [node 4] ICP30 unregister session Session# [5:7577789396198106701:2108] VirtualId# [5:7577789396198106697:0]
2025-11-28T14:53:02.624338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:02.624389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789396198106680:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5302) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.624404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5302) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.624866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789397309444536:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5302) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:02.624906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789397309444564:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:02.624937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789397309444536:2107] [node 4] ICP30 unregister session Session# [6:7577789397309444564:2107] VirtualId# [6:7577789397309444560:0]
2025-11-28T14:53:02.625017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:02.625040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789397309444536:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5302) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.625061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5302) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:02.634654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789397309444536:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:53:02.652046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:02.652194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789397309444568:2108] [node 5] ICS01 socket: 288 reason# EndOfStream
2025-11-28T14:53:02.652228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789397309444539:2108] [node 5] ICP30 unregister session Session# [6:7577789397309444568:2108] VirtualId# [6:7577789397309444562:0]
2025-11-28T14:53:02.652256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789397309444568:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:02.652340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:02.652423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:02.718293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789395653107698:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:02.718300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789395653107699:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:02.736340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789395329427904:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.736390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789395329427905:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:02.753799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394422895068:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.753845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394422895069:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:02.772553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397169734952:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.772619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397169734954:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:02.772689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397169734953:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:02.794417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789395537179110:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.794452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789395537179112:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:02.794588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789395537179111:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:02.816142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789393993435015:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:02.816172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789393993435016:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:02.816183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789393993435017:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:02.896939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789395653107788:2131] [node 5] ICP01 ready to work
2025-11-28T14:53:02.900857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789395329427967:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:02.900646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394422895140:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:02.902138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789393993435092:2106] [node 5] ICP01 ready to work
2025-11-28T14:53:02.902519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789395653107789:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:02.902205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789395329427968:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:02.902541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397169735033:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:02.902535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789395537179173:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:02.903160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789393993435095:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:02.902879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394422895141:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:02.903217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789395653107790:2132] [node 6] ICP01 ready to work
2025-11-28T14:53:02.902711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789395537179174:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:02.902683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789395329427969:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:02.903246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789397169735034:2126] [node 6] ICP01 ready to work
2025-11-28T14:53:02.903334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789394422895142:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.914308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789395329427978:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32099
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.914304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394422895151:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42864
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.916396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789395329427985:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32099
No RDMA context found for GID: 2025-11-28T14:53:02.917479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397169735048:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25706
::ffff:127.1.0.0
2025-11-28T14:53:02.917272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789395329427984:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32099
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.917514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394422895157:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42864
2025-11-28T14:53:02.918355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394422895155:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42864
2025-11-28T14:53:02.918491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394422895159:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42864
2025-11-28T14:53:02.919022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394422895151:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.919325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394422895155:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.919571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789395329427985:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.919936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394422895157:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.920059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789395329427984:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.920084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789395537179188:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18292
2025-11-28T14:53:02.920207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789395329427986:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32099
2025-11-28T14:53:02.920220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789395329427978:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
2025-11-28T14:53:02.921481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789395653107800:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.921810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789395653107799:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.922895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789395653107801:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-28T14:53:02.922927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107799:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.923114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789395653107802:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-28T14:53:02.923187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789395653107803:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-28T14:53:02.923733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107800:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.924201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107801:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.924356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394422895159:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.924481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107802:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.924927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107803:2145] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:53:02.924930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789395329427986:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.924986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394422895163:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42864
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.925089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789395537179189:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18292
2025-11-28T14:53:02.925105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789394422895069:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.925151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789395537179188:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.925154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789395329427990:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32099
2025-11-28T14:53:02.925299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789395329427971:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.925298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789395537179112:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.925276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397169735048:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.925327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789395329427971:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.925811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789397169734953:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.925845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394422895148:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.925882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789394422895148:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.925952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789395329427975:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.925977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789395329427975:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.926317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789395329427971:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:02.927387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394422895148:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:02.927389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394422895165:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42864
2025-11-28T14:53:02.927551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789395329427975:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:02.927876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789395329427905:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.928006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.928064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789395329427905:2049] [node 3] ICP22 created new session: [2:7577789395329427994:2049]
2025-11-28T14:53:02.928305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789395329427994:2049] [node 3] ICS09 handshake done sender: [2:7577789395329427971:2110] self: [2:7577789395329427970:0] peer: [3:7577789394422895164:0] socket: 299 qp: -1
2025-11-28T14:53:02.928344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789395329427994:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:02.928878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397169735050:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25706
2025-11-28T14:53:02.928878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789395537179191:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18292
2025-11-28T14:53:02.928898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789394422895141:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.928642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:02.928977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.929025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394422895141:2108] [node 5] ICP22 created new session: [3:7577789394422895167:2108]
2025-11-28T14:53:02.929114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394422895167:2108] [node 5] ICS09 handshake done sender: [3:7577789394422895148:2121] self: [3:7577789394422895147:0] peer: [5:7577789395537179190:0] socket: 307 qp: -1
2025-11-28T14:53:02.929156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394422895167:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:02.929159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394422895159:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.929188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.929340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397169735048:2133] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.929287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789395537179188:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:02.930181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789395537179189:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.930189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789397169734953:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.930188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789395329427967:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.930230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789395537179112:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.930327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.930351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.930288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:02.930376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789395329427967:2107] [node 4] ICP22 created new session: [2:7577789395329427996:2107]
2025-11-28T14:53:02.930332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789395537179112:2050] [node 3] ICP22 created new session: [5:7577789395537179192:2050]
2025-11-28T14:53:02.930406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397169734953:2049] [node 2] ICP22 created new session: [4:7577789397169735051:2049]
2025-11-28T14:53:02.930362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789395537179192:2050] [node 3] ICS09 handshake done sender: [5:7577789395537179188:2115] self: [5:7577789395537179190:0] peer: [3:7577789394422895147:0] socket: 308 qp: -1
2025-11-28T14:53:02.930546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789395329427996:2107] [node 4] ICS09 handshake done sender: [2:7577789395329427975:2112] self: [2:7577789395329427974:0] peer: [4:7577789397169735049:0] socket: 302 qp: -1
2025-11-28T14:53:02.930542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397169735051:2049] [node 2] ICS09 handshake done sender: [4:7577789397169735048:2133] self: [4:7577789397169735049:0] peer: [2:7577789395329427974:0] socket: 303 qp: -1
2025-11-28T14:53:02.930386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789395537179192:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:02.930574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789395329427996:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:02.930566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397169735051:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:02.931072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:02.930428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:02.930658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:02.932741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789395537179110:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.932846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789394422895069:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.933210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.933325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394422895069:2049] [node 2] ICP22 created new session: [3:7577789394422895169:2049]
2025-11-28T14:53:02.933376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394422895169:2049] [node 2] ICS09 handshake done sender: [3:7577789394422895159:2127] self: [3:7577789394422895164:0] peer: [2:7577789395329427970:0] socket: 304 qp: -1
2025-11-28T14:53:02.933402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394422895169:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:02.933428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:02.933483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107796:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.933520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789395653107796:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.933695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789395537179194:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18292
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.934272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789395653107796:2139] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:02.934403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435109:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
2025-11-28T14:53:02.934468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789394422895069:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:02.934563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789395653107788:2131] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.934602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789395537179194:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.934616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.934709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789395653107788:2131] [node 5] ICP22 created new session: [1:7577789395653107815:2131]
2025-11-28T14:53:02.934886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789395653107815:2131] [node 5] ICS09 handshake done sender: [1:7577789395653107796:2139] self: [1:7577789395653107795:0] peer: [5:7577789395537179195:0] socket: 319 qp: -1
2025-11-28T14:53:02.934923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789395653107815:2131] [node 5] ICS10 traffic start
2025-11-28T14:53:02.934983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.934982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789395537179174:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.935081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789395329427990:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.935149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394422895163:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.935298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789395537179197:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18292
2025-11-28T14:53:02.935348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397169735042:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.935382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789397169735042:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.935580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789395329427904:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.936185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435110:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
2025-11-28T14:53:02.936705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397169735054:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25706
2025-11-28T14:53:02.937026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397169735042:2131] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:02.937177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789395537179110:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789395537179189:2116] held: no
2025-11-28T14:53:02.937080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107794:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.937110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789395653107794:2138] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.937853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789397169735033:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:53:02.937897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.938129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397169735033:2108] [node 5] ICP22 created new session: [4:7577789397169735057:2108]
2025-11-28T14:53:02.938178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397169735057:2108] [node 5] ICS09 handshake done sender: [4:7577789397169735042:2131] self: [4:7577789397169735041:0] peer: [5:7577789395537179198:0] socket: 291 qp: -1
2025-11-28T14:53:02.938215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789394422895068:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.938223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397169735056:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25706
2025-11-28T14:53:02.938229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397169735057:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:02.938264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.938362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397169735054:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.938446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789393993435109:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.938495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107793:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.938527Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789395653107793:2137] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.938635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789393993435092:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.938722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789397169734952:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.940067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789395537179189:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:02.940800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107798:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.940834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789395653107798:2140] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.941342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789395653107794:2138] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.941418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435112:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.941613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397169735056:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.941480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789393993435110:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.941607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789395653107698:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.941625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789395537179110:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.941746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:02.941633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789395329428001:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32099
2025-11-28T14:53:02.941643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435113:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
2025-11-28T14:53:02.941788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.941664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789393993435016:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.941870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789395537179182:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.941834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789395653107698:2048] [node 2] ICP22 created new session: [1:7577789395653107821:2048]
2025-11-28T14:53:02.941820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789393993435112:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.941910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789393993435017:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:53:02.941874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789395653107821:2048] [node 2] ICS09 handshake done sender: [1:7577789395653107794:2138] self: [1:7577789395653107792:0] peer: [2:7577789395329428000:0] socket: 318 qp: -1
::ffff:127.1.0.0
2025-11-28T14:53:02.941888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789395537179110:2048] [node 1] ICP22 created new session: [5:7577789395537179201:2048]
2025-11-28T14:53:02.941924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789395653107821:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:02.941989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789395329427980:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.942032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789395329427980:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.942493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394422895158:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.941909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789395537179182:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.942230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789393993435113:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.941955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:02.942637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789395329427990:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:02.942525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789394422895158:2126] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.941936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789395537179201:2048] [node 1] ICS09 handshake done sender: [5:7577789395537179189:2116] self: [5:7577789395537179195:0] peer: [1:7577789395653107795:0] socket: 320 qp: -1
2025-11-28T14:53:02.943141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789393993435095:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.942184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789395653107793:2137] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.942591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789394422895174:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42864
2025-11-28T14:53:02.943636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789395329427904:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.943679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:02.941998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789395537179201:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.943934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789395329427904:2048] [node 1] ICP22 created new session: [2:7577789395329428003:2048]
2025-11-28T14:53:02.942163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:02.944336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789395329428003:2048] [node 1] ICS09 handshake done sender: [2:7577789395329427990:2119] self: [2:7577789395329428000:0] peer: [1:7577789395653107792:0] socket: 322 qp: -1
2025-11-28T14:53:02.944349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394422895163:2128] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.944424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789395329428003:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.944261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789395537179199:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18292
2025-11-28T14:53:02.944429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789397169735044:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.944519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789394422895068:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.944487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:02.944458Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789397169735044:2132] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:53:02.945534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
::ffff:127.1.0.0
2025-11-28T14:53:02.945458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789395537179200:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18292
2025-11-28T14:53:02.945813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789397169734954:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.945762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394422895068:2048] [node 1] ICP22 created new session: [3:7577789394422895176:2048]
2025-11-28T14:53:02.945929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394422895176:2048] [node 1] ICS09 handshake done sender: [3:7577789394422895163:2128] self: [3:7577789394422895172:0] peer: [1:7577789395653107791:0] socket: 321 qp: -1
2025-11-28T14:53:02.945963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397169735061:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25706
2025-11-28T14:53:02.946635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397169735044:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:02.946500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394422895176:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.946674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789395653107699:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.947038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789397169735034:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.946645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:02.946733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:02.947214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:02.947273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397169735034:2126] [node 6] ICP22 created new session: [4:7577789397169735064:2126]
2025-11-28T14:53:02.946857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789395653107699:2049] [node 3] ICP22 created new session: [1:7577789395653107823:2049]
2025-11-28T14:53:02.947363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789395329427980:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:02.947151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789395653107823:2049] [node 3] ICS09 handshake done sender: [1:7577789395653107793:2137] self: [1:7577789395653107791:0] peer: [3:7577789394422895172:0] socket: 317 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.947292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789395653107823:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:02.947391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397169735064:2126] [node 6] ICS09 handshake done sender: [4:7577789397169735044:2132] self: [4:7577789397169735043:0] peer: [6:7577789393993435116:0] socket: 340 qp: -1
2025-11-28T14:53:02.947430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397169735064:2126] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-11-28T14:53:02.947330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0
2025-11-28T14:53:02.947475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:02.947582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435117:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
2025-11-28T14:53:02.947621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394422895158:2126] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:02.947612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435119:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
2025-11-28T14:53:02.947967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789395537179199:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.948066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789395329427969:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.948122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:02.948170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789395329427969:2109] [node 6] ICP22 created new session: [2:7577789395329428005:2109]
2025-11-28T14:53:02.948233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789395329428005:2109] [node 6] ICS09 handshake done sender: [2:7577789395329427980:2115] self: [2:7577789395329427979:0] peer: [6:7577789393993435114:0] socket: 330 qp: -1
2025-11-28T14:53:02.948202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789395537179194:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.948434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789395537179182:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:02.948265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789395329428005:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:02.948415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:02.948800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789394422895150:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.948900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435118:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
2025-11-28T14:53:02.948832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789394422895150:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.949253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789395537179174:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.949325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.949641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397169735066:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25706
2025-11-28T14:53:02.949359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789395537179174:2108] [node 4] ICP22 created new session: [5:7577789395537179205:2108]
2025-11-28T14:53:02.949638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789393993435112:2133] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:02.949745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789395537179205:2108] [node 4] ICS09 handshake done sender: [5:7577789395537179194:2118] self: [5:7577789395537179198:0] peer: [4:7577789397169735041:0] socket: 294 qp: -1
2025-11-28T14:53:02.949778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789395537179205:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:02.949977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:02.950355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789394422895142:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.950207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789393993435110:2132] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.950237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789395537179111:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.950406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:02.950511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394422895142:2109] [node 6] ICP22 created new session: [3:7577789394422895178:2109]
2025-11-28T14:53:02.950326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789395537179173:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.950378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:02.950624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789393993435017:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.950631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394422895178:2109] [node 6] ICS09 handshake done sender: [3:7577789394422895158:2126] self: [3:7577789394422895156:0] peer: [6:7577789393993435115:0] socket: 339 qp: -1
2025-11-28T14:53:02.950690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789395653107798:2140] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:02.950803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394422895178:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:02.950828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:02.950566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789395537179173:2109] [node 6] ICP22 created new session: [5:7577789395537179208:2109]
2025-11-28T14:53:02.950850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:02.950598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789395537179208:2109] [node 6] ICS09 handshake done sender: [5:7577789395537179182:2113] self: [5:7577789395537179181:0] peer: [6:7577789393993435111:0] socket: 301 qp: -1
2025-11-28T14:53:02.950880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789393993435016:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.950882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789393993435017:2050] [node 3] ICP22 created new session: [6:7577789393993435121:2050]
2025-11-28T14:53:02.950675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789395537179208:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:02.950924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789393993435121:2050] [node 3] ICS09 handshake done sender: [6:7577789393993435112:2133] self: [6:7577789393993435115:0] peer: [3:7577789394422895156:0] socket: 341 qp: -1
2025-11-28T14:53:02.950709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:02.950953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789393993435121:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:02.950955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.951105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:02.951440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397169735054:2135] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:02.951109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789393993435016:2049] [node 2] ICP22 created new session: [6:7577789393993435122:2049]
2025-11-28T14:53:02.951576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789397169734952:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.951184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789393993435122:2049] [node 2] ICS09 handshake done sender: [6:7577789393993435110:2132] self: [6:7577789393993435114:0] peer: [2:7577789395329427979:0] socket: 331 qp: -1
2025-11-28T14:53:02.951218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789393993435122:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:02.951247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:02.951764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:02.951807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397169734952:2048] [node 1] ICP22 created new session: [4:7577789397169735067:2048]
2025-11-28T14:53:02.951900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397169735067:2048] [node 1] ICS09 handshake done sender: [4:7577789397169735054:2135] self: [4:7577789397169735060:0] peer: [1:7577789395653107797:0] socket: 333 qp: -1
2025-11-28T14:53:02.952003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397169735067:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.952068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:02.953113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789395653107789:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.953157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.953266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789395653107789:2107] [node 4] ICP22 created new session: [1:7577789395653107826:2107]
2025-11-28T14:53:02.953332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789395653107826:2107] [node 4] ICS09 handshake done sender: [1:7577789395653107798:2140] self: [1:7577789395653107797:0] peer: [4:7577789397169735060:0] socket: 332 qp: -1
2025-11-28T14:53:02.953431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789395653107826:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:02.953528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:02.953924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789395329427977:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.953954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789395329427977:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.954256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789394422895150:2122] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.955002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789397169735069:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25706
2025-11-28T14:53:02.955052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789393993435113:2134] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.955221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789394422895140:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.955465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789395329427977:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:02.955292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.955572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435120:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
2025-11-28T14:53:02.955440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789394422895140:2107] [node 4] ICP22 created new session: [3:7577789394422895181:2107]
2025-11-28T14:53:02.955595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789397169735056:2136] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:02.955581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789394422895181:2107] [node 4] ICS09 handshake done sender: [3:7577789394422895150:2122] self: [3:7577789394422895149:0] peer: [4:7577789397169735062:0] socket: 334 qp: -1
2025-11-28T14:53:02.955617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789394422895181:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.955755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789395329427968:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.955654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:02.956389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.956546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789395537179210:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18292
2025-11-28T14:53:02.956673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789395329427968:2108] [node 5] ICP22 created new session: [2:7577789395329428008:2108]
2025-11-28T14:53:02.958664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789395329428008:2108] [node 5] ICS09 handshake done sender: [2:7577789395329427977:2113] self: [2:7577789395329427976:0] peer: [5:7577789395537179207:0] socket: 313 qp: -1
2025-11-28T14:53:02.958738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789395537179111:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25469) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789395537179199:2120] held: no
2025-11-28T14:53:02.958716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789395329428008:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:02.958768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.959562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789397169734954:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.959636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789393993435095:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.959683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:02.959662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:02.959705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789397169734954:2050] [node 3] ICP22 created new session: [4:7577789397169735070:2050]
2025-11-28T14:53:02.959729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789393993435095:2107] [node 4] ICP22 created new session: [6:7577789393993435126:2107]
2025-11-28T14:53:02.959745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789397169735070:2050] [node 3] ICS09 handshake done sender: [4:7577789397169735056:2136] self: [4:7577789397169735062:0] peer: [3:7577789394422895149:0] socket: 337 qp: -1
2025-11-28T14:53:02.959781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789393993435126:2107] [node 4] ICS09 handshake done sender: [6:7577789393993435113:2134] self: [6:7577789393993435116:0] peer: [4:7577789397169735043:0] socket: 342 qp: -1
2025-11-28T14:53:02.959806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789393993435126:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:02.963593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:02.959773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789397169735070:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:02.959796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:02.960316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789397169734954:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:02.964174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789395537179199:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:02.964355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789395537179111:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.964418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:02.964447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789395537179111:2049] [node 2] ICP22 created new session: [5:7577789395537179211:2049]
2025-11-28T14:53:02.964607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789395537179211:2049] [node 2] ICS09 handshake done sender: [5:7577789395537179199:2120] self: [5:7577789395537179207:0] peer: [2:7577789395329427976:0] socket: 336 qp: -1
2025-11-28T14:53:02.964666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789395537179211:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:02.964694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:02.965265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789393993435109:2131] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.965825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435125:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
2025-11-28T14:53:02.965839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789393993435092:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.965932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:02.965986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789393993435092:2106] [node 5] ICP22 created new session: [6:7577789393993435128:2106]
2025-11-28T14:53:02.966037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789393993435128:2106] [node 5] ICS09 handshake done sender: [6:7577789393993435109:2131] self: [6:7577789393993435111:0] peer: [5:7577789395537179181:0] socket: 306 qp: -1
2025-11-28T14:53:02.966117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789393993435128:2106] [node 5] ICS10 traffic start
2025-11-28T14:53:02.966154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:02.966352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789393993435125:2139] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.966493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789393993435015:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:02.966754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789395653107805:2146] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:02.966788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789395653107805:2146] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:02.967806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789395653107805:2146] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:02.967966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789395653107790:2132] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:02.968014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:02.968044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789395653107790:2132] [node 6] ICP22 created new session: [1:7577789395653107829:2132]
2025-11-28T14:53:02.968014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789393993435132:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14634
2025-11-28T14:53:02.968084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789395653107829:2132] [node 6] ICS09 handshake done sender: [1:7577789395653107805:2146] self: [1:7577789395653107804:0] peer: [6:7577789393993435131:0] socket: 288 qp: -1
2025-11-28T14:53:02.968111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789395653107829:2132] [node 6] ICS10 traffic start
2025-11-28T14:53:02.968139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:02.968221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789393993435125:2139] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:02.968736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789393993435015:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:02.968804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:02.968867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789393993435015:2048] [node 1] ICP22 created new session: [6:7577789393993435133:2048]
2025-11-28T14:53:02.968927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789393993435133:2048] [node 1] ICS09 handshake done sender: [6:7577789393993435125:2139] self: [6:7577789393993435131:0] peer: [1:7577789395653107804:0] socket: 283 qp: -1
2025-11-28T14:53:02.968965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789393993435133:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:02.969019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:02.969888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789393993435015:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:03.010871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:03.011024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789395329428003:2048] [node 1] ICS07 socket disconnect 322 reason# EndOfStream
2025-11-28T14:53:03.011048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789395329428003:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.011112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.011147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789395329428003:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:03.012127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:03.012289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789394422895176:2048] [node 1] ICS07 socket disconnect 321 reason# EndOfStream
2025-11-28T14:53:03.012326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789394422895176:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.012422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.012471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789394422895176:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:03.013574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789395329427904:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:03.013616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789395329428003:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:03.014857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:03.015009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789397169735067:2048] [node 1] ICS07 socket disconnect 333 reason# EndOfStream
2025-11-28T14:53:03.015040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789397169735067:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.015012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789394422895068:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:03.015022Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789395329427904:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.015045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789394422895176:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:03.015088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.015103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789395329428003:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.015355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789395329427904:2048] [node 1] ICP30 unregister session Session# [2:7577789395329428003:2048] VirtualId# [2:7577789395329428000:0]
2025-11-28T14:53:03.015113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789397169735067:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:03.015603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.015651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789395329427904:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.015672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.016049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789397169734952:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.016089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789394422895068:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.016111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789397169735067:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.016226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789397169734952:2048] [node 1] ICP30 unregister session Session# [4:7577789397169735067:2048] VirtualId# [4:7577789397169735060:0]
2025-11-28T14:53:03.016234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789394422895176:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.016255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.016280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789397169734952:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.016266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789394422895068:2048] [node 1] ICP30 unregister session Session# [3:7577789394422895176:2048] VirtualId# [3:7577789394422895172:0]
2025-11-28T14:53:03.016290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.016317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.016349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789394422895068:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.016361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.016537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:03.016903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789395537179201:2048] [node 1] ICS01 socket: 320 reason# EndOfStream
2025-11-28T14:53:03.016954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789395537179110:2048] [node 1] ICP30 unregister session Session# [5:7577789395537179201:2048] VirtualId# [5:7577789395537179195:0]
2025-11-28T14:53:03.016993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789395537179201:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.017106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.017160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.017498Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:03.017667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789393993435133:2048] [node 1] ICS07 socket disconnect 283 reason# EndOfStream
2025-11-28T14:53:03.017689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789393993435133:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.017750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.017782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789393993435133:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:03.018285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789393993435015:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.018317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789393993435133:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.018334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789393993435015:2048] [node 1] ICP30 unregister session Session# [6:7577789393993435133:2048] VirtualId# [6:7577789393993435131:0]
2025-11-28T14:53:03.018432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.018457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789393993435015:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.018471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.025798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789394422895068:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:03.025823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789397169734952:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:03.025859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789395329427904:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:03.027874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789393993435015:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:03.093647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:03.093869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789394422895169:2049] [node 2] ICS07 socket disconnect 304 reason# EndOfStream
2025-11-28T14:53:03.093910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789394422895169:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.094011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:03.094063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789394422895169:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:03.096055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789394422895069:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25469) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:03.096117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789394422895169:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:03.097576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:03.097625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789394422895069:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25469) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.097768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789394422895169:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.097813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789394422895069:2049] [node 2] ICP30 unregister session Session# [3:7577789394422895169:2049] VirtualId# [3:7577789394422895164:0]
2025-11-28T14:53:03.097878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789397169735051:2049] [node 2] ICS01 socket: 303 reason# EndOfStream
2025-11-28T14:53:03.097966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:03.098019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789397169734953:2049] [node 2] ICP30 unregister session Session# [4:7577789397169735051:2049] VirtualId# [4:7577789397169735049:0]
2025-11-28T14:53:03.098019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789394422895069:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.098077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789397169735051:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.098038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.098209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:03.098273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:03.098632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:03.098788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789395537179211:2049] [node 2] ICS01 socket: 336 reason# EndOfStream
2025-11-28T14:53:03.098817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789395537179111:2049] [node 2] ICP30 unregister session Session# [5:7577789395537179211:2049] VirtualId# [5:7577789395537179207:0]
2025-11-28T14:53:03.098838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789395537179211:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.098904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:03.098943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:03.099500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:03.099633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789393993435122:2049] [node 2] ICS07 socket disconnect 331 reason# EndOfStream
2025-11-28T14:53:03.099657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789393993435122:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.099714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:03.099746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789393993435122:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:03.100265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789393993435016:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25469) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.100335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789393993435122:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.100401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789393993435016:2049] [node 2] ICP30 unregister session Session# [6:7577789393993435122:2049] VirtualId# [6:7577789393993435114:0]
2025-11-28T14:53:03.100535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:03.100574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789393993435016:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.100584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.110824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789393993435016:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:03.149879Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:03.150121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789397169735070:2050] [node 3] ICS01 socket: 337 reason# EndOfStream
2025-11-28T14:53:03.150181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789397169734954:2050] [node 3] ICP30 unregister session Session# [4:7577789397169735070:2050] VirtualId# [4:7577789397169735062:0]
2025-11-28T14:53:03.150220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789397169735070:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.150335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:03.150391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:03.151844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:03.152189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789395537179192:2050] [node 3] ICS07 socket disconnect 308 reason# EndOfStream
2025-11-28T14:53:03.152231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789395537179192:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.152377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:03.152442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789395537179192:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:03.152794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:03.153001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789393993435121:2050] [node 3] ICS07 socket disconnect 341 reason# EndOfStream
2025-11-28T14:53:03.153038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789393993435121:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.153132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:03.153184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789393993435121:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:03.153438Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789395537179112:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28839) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.153534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789395537179192:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.153697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789395537179112:2050] [node 3] ICP30 unregister session Session# [5:7577789395537179192:2050] VirtualId# [5:7577789395537179190:0]
2025-11-28T14:53:03.153779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:03.153856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789395537179112:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.153883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.154076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789393993435017:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28839) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.154140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789393993435121:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.154281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789393993435017:2050] [node 3] ICP30 unregister session Session# [6:7577789393993435121:2050] VirtualId# [6:7577789393993435115:0]
2025-11-28T14:53:03.154391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:03.154427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789393993435017:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.154440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.164083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789393993435017:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:03.164082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789395537179112:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:03.240476Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:03.240812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789395537179205:2108] [node 4] ICS07 socket disconnect 294 reason# EndOfStream
2025-11-28T14:53:03.240847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789395537179205:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.240968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:03.241009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789395537179205:2108] [node 4] ICS15 start handshake
2025-11-28T14:53:03.242423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789395537179174:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.242523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789395537179205:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.242405Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:03.242597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789395537179174:2108] [node 4] ICP30 unregister session Session# [5:7577789395537179205:2108] VirtualId# [5:7577789395537179198:0]
2025-11-28T14:53:03.242633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789393993435126:2107] [node 4] ICS07 socket disconnect 342 reason# EndOfStream
2025-11-28T14:53:03.242675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789393993435126:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.242716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:03.242756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789395537179174:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.242759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:03.242771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.242828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789393993435126:2107] [node 4] ICS15 start handshake
2025-11-28T14:53:03.243570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789393993435095:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.243664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789393993435126:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.243694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789393993435095:2107] [node 4] ICP30 unregister session Session# [6:7577789393993435126:2107] VirtualId# [6:7577789393993435116:0]
2025-11-28T14:53:03.243792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:03.243833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789393993435095:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.243850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.252177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789395537179174:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:53:03.254127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789393993435095:2107] [node 4] ICP33 wake up from error state
2025-11-28T14:53:03.296931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:03.297129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789393993435128:2106] [node 5] ICS07 socket disconnect 306 reason# EndOfStream
2025-11-28T14:53:03.297156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789393993435128:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.297225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:03.297259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789393993435128:2106] [node 5] ICS15 start handshake
2025-11-28T14:53:03.297923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789393993435092:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29767) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.298070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789393993435128:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.298110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789393993435092:2106] [node 5] ICP30 unregister session Session# [6:7577789393993435128:2106] VirtualId# [6:7577789393993435111:0]
2025-11-28T14:53:03.298270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:53:03.298298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789393993435092:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29767) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.298307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29767) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:03.497754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789400256646319:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:03.497787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789400256646320:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:03.515282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789401245364679:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:03.515365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789401245364680:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:03.535488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789401431477804:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:03.535543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789401431477805:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:03.558464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789401006720760:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:03.558506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789401006720761:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:03.558546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789401006720762:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:03.581406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789400454691324:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:03.581442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789400454691326:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:03.581558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789400454691325:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:03.603846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789398967992830:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:03.603920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789398967992831:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:03.604079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789398967992832:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:03.686383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789398967992889:2105] [node 5] ICP01 ready to work
2025-11-28T14:53:03.721140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789401006720823:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:03.708758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789400256646382:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:03.720165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789401245364742:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:03.742176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789401006720826:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:03.733199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789400256646383:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:03.737986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789400454691387:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:03.742172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789401431477868:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:03.742218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789400454691388:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:03.740772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789398967992903:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:03.737955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789400256646384:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:03.741800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789401245364744:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:03.750381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789401431477869:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:03.743437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789401245364743:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:03.750428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789401431477870:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:03.753967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789401006720761:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27361193ns) {Configure:104(27356218ns) {ProcessConfigured:135(592645ns) {StartInitialHandshake:157(325762ns) {DropHandshakes:470(4009ns) {DropIncomingHandshake:442(759ns) DropOutgoingHandshake:457(563ns)} GenerateSessionVirtualId:988(1531ns) PrepareNewSessionHandshake:477(1818ns) {SwitchToState:179(769ns)}}}}}
2025-11-28T14:53:03.756069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789398967992889:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(35416616ns) {Configure:104(35398057ns) {ProcessConfigured:135(1065445ns) {StartInitialHandshake:157(1060210ns) {DropHandshakes:470(2978ns) {DropIncomingHandshake:442(1050ns) DropOutgoingHandshake:457(497ns)} GenerateSessionVirtualId:988(782ns) PrepareNewSessionHandshake:477(2880ns) {SwitchToState:179(957ns)}}}}}
2025-11-28T14:53:03.764536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789401431477804:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(59247125ns) {Configure:104(59239092ns) {ProcessConfigured:135(514520ns) {StartInitialHandshake:157(512276ns) {DropHandshakes:470(1842ns) {DropIncomingHandshake:442(371ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(513ns) PrepareNewSessionHandshake:477(2545ns) {SwitchToState:179(1395ns)}}}}}
2025-11-28T14:53:03.760650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789401006720823:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27343548ns) {Configure:104(27337427ns) {ProcessConfigured:135(2563814ns) {StartInitialHandshake:157(2560724ns) {DropHandshakes:470(1879ns) {DropIncomingHandshake:442(848ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(606ns) PrepareNewSessionHandshake:477(2579ns) {SwitchToState:179(1049ns)}}}}}
2025-11-28T14:53:03.764046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789400256646320:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27902851ns) {Configure:104(27894836ns) {ProcessConfigured:135(287800ns) {StartInitialHandshake:157(286238ns) {DropHandshakes:470(1727ns) {DropIncomingHandshake:442(498ns) DropOutgoingHandshake:457(584ns)} GenerateSessionVirtualId:988(534ns) PrepareNewSessionHandshake:477(937ns) {SwitchToState:179(284ns)}}}}}
2025-11-28T14:53:03.766802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789401245364680:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17271450ns) {Configure:104(17266950ns) {ProcessConfigured:135(297393ns) {StartInitialHandshake:157(295643ns) {DropHandshakes:470(1066ns) {DropIncomingHandshake:442(278ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(442ns) PrepareNewSessionHandshake:477(1392ns) {SwitchToState:179(747ns)}}}}}
2025-11-28T14:53:03.765130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789400256646319:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19979833ns) {Configure:104(19965185ns) {ProcessConfigured:135(290318ns) {StartInitialHandshake:157(288336ns) {DropHandshakes:470(1258ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(321ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(1960ns) {SwitchToState:179(887ns)}}}}}
2025-11-28T14:53:03.771117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789401431477805:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17520679ns) {Configure:104(17342956ns) {ProcessConfigured:135(2051642ns) {StartInitialHandshake:157(2050097ns) {DropHandshakes:470(1318ns) {DropIncomingHandshake:442(230ns) DropOutgoingHandshake:457(94ns)} GenerateSessionVirtualId:988(407ns) PrepareNewSessionHandshake:477(1829ns) {SwitchToState:179(344ns)}}}}}
2025-11-28T14:53:03.791378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789400256646384:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20002214ns) {Configure:104(19997294ns) {ProcessConfigured:135(3522761ns) {StartInitialHandshake:157(3519108ns) {DropHandshakes:470(2612ns) {DropIncomingHandshake:442(498ns) DropOutgoingHandshake:457(769ns)} GenerateSessionVirtualId:988(743ns) PrepareNewSessionHandshake:477(7123ns) {SwitchToState:179(5405ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.791624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789398967992903:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16240753ns) {Configure:104(16230455ns) {ProcessConfigured:135(905308ns) {StartInitialHandshake:157(900802ns) {DropHandshakes:470(3661ns) {DropIncomingHandshake:442(1157ns) DropOutgoingHandshake:457(599ns)} GenerateSessionVirtualId:988(1553ns) PrepareNewSessionHandshake:477(2849ns) {SwitchToState:179(1119ns)}}}}}
2025-11-28T14:53:03.791889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789400256646392:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48728
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.792084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789401245364743:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20922140ns) {Configure:104(20917818ns) {ProcessConfigured:135(1230505ns) {StartInitialHandshake:157(1228137ns) {DropHandshakes:470(3467ns) {DropIncomingHandshake:442(948ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(1152ns) PrepareNewSessionHandshake:477(1525ns) {SwitchToState:179(784ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.795176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789401431477869:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21080316ns) {Configure:104(21075648ns) {ProcessConfigured:135(955607ns) {StartInitialHandshake:157(950408ns) {DropHandshakes:470(3110ns) {DropIncomingHandshake:442(1076ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(833ns) PrepareNewSessionHandshake:477(2477ns) {SwitchToState:179(1224ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.798520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992912:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.799124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646392:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.792103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789401245364744:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19757598ns) {Configure:104(19752790ns) {ProcessConfigured:135(4217948ns) {StartInitialHandshake:157(4215782ns) {DropHandshakes:470(1556ns) {DropIncomingHandshake:442(490ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:477(2428ns) {SwitchToState:179(898ns)}}}}}
2025-11-28T14:53:03.795745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789401431477870:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20576182ns) {Configure:104(20272187ns) {ProcessConfigured:135(926626ns) {StartInitialHandshake:157(923181ns) {DropHandshakes:470(1312ns) {DropIncomingHandshake:442(286ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:477(6426ns) {SwitchToState:179(4658ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.798999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992913:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.801455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789398967992912:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.798565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789401431477881:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25441
2025-11-28T14:53:03.801886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789400454691402:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37185
2025-11-28T14:53:03.798725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789401245364751:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21566
2025-11-28T14:53:03.801393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789400256646399:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48728
2025-11-28T14:53:03.802346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789401006720838:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6983
2025-11-28T14:53:03.800887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789401431477882:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25441
2025-11-28T14:53:03.802578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789398967992903:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.803768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789401245364751:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.802521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789401431477881:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.804484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789398967992913:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.804025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789401431477882:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.814066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789400256646401:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48728
2025-11-28T14:53:03.814189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789401431477887:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25441
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.814188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992919:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.814163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789400454691392:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.814326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789398967992889:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.814225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789401431477886:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25441
2025-11-28T14:53:03.813108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789401006720830:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.814205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789400454691392:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.813164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789401006720830:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.814405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789400454691403:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37185
2025-11-28T14:53:03.814403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789401245364759:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21566
2025-11-28T14:53:03.814398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789400256646402:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48728
2025-11-28T14:53:03.814537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789401006720830:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:03.814801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789401245364760:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21566
2025-11-28T14:53:03.815407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789401431477887:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.816080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789401006720823:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.815673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789401431477804:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.817457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789398967992919:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.818021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646387:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.816138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:03.818281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789400454691392:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:03.816307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789401006720823:2109] [node 6] ICP22 created new session: [4:7577789401006720840:2109]
No RDMA context found for GID: 2025-11-28T14:53:03.818059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789400256646387:2110] [node 3] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-28T14:53:03.818722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789401245364761:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21566
2025-11-28T14:53:03.820085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789401245364759:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.817433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789401006720840:2109] [node 6] ICS09 handshake done sender: [4:7577789401006720830:2112] self: [4:7577789401006720829:0] peer: [6:7577789398967992920:0] socket: 289 qp: -1
2025-11-28T14:53:03.820428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992922:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.820833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789400256646387:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:03.821872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789400454691402:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.817480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789401006720840:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:03.818035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:03.822364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789401245364679:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.822415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646399:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.822211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789401006720838:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.822465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789398967992830:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.822712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789400454691388:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.822936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646402:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.822934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789401245364760:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.822404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789401006720761:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.822943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:03.824186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789398967992912:2123] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.822994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789400454691388:2109] [node 6] ICP22 created new session: [5:7577789400454691406:2109]
2025-11-28T14:53:03.824180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789401431477886:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.824326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992923:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.824322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789401245364749:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.823937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789400454691403:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.824412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789401431477891:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25441
2025-11-28T14:53:03.824388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789400454691406:2109] [node 6] ICS09 handshake done sender: [5:7577789400454691392:2111] self: [5:7577789400454691391:0] peer: [6:7577789398967992921:0] socket: 294 qp: -1
2025-11-28T14:53:03.824499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646401:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.824494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789401431477805:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.824750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646390:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.824349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789401245364749:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.824394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789400454691326:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.825397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789401431477876:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.824360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789401245364761:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.824808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789398967992903:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.824754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789400256646320:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.825432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789401431477876:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.824741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789401245364763:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21566
2025-11-28T14:53:03.828481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789401431477887:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:03.824444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789400454691406:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:03.824776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789400256646390:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.828492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789401006720844:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6983
2025-11-28T14:53:03.824822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:03.824798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:03.824866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:03.828889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789401431477804:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.826364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646388:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.825478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789398967992903:2108] [node 4] ICP22 created new session: [6:7577789398967992925:2108]
2025-11-28T14:53:03.825471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789401245364747:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.828953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:03.828453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789398967992925:2108] [node 4] ICS09 handshake done sender: [6:7577789398967992912:2123] self: [6:7577789398967992920:0] peer: [4:7577789401006720829:0] socket: 295 qp: -1
2025-11-28T14:53:03.826370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789400256646320:2049] [node 3] ICP22 created new session: [1:7577789400256646406:2049]
2025-11-28T14:53:03.825498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789401245364747:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.828471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789398967992913:2124] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:03.826396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789400256646388:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.828471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789400256646406:2049] [node 3] ICS09 handshake done sender: [1:7577789400256646387:2110] self: [1:7577789400256646385:0] peer: [3:7577789401431477890:0] socket: 302 qp: -1
2025-11-28T14:53:03.830143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789401006720844:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.829302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789401431477876:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:03.828511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789400256646406:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.829310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789401431477804:2048] [node 1] ICP22 created new session: [3:7577789401431477894:2048]
2025-11-28T14:53:03.828748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:03.827751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789401245364763:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.828505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789398967992925:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:03.829742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789401431477894:2048] [node 1] ICS09 handshake done sender: [3:7577789401431477887:2119] self: [3:7577789401431477890:0] peer: [1:7577789400256646385:0] socket: 315 qp: -1
2025-11-28T14:53:03.829705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789400256646390:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:03.829296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789401245364749:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:03.828560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:03.829760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789401245364747:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:03.832125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789400256646383:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.831151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789401245364742:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.829793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789398967992889:2105] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:53:03.832136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789400256646388:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:03.832569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:03.830201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:03.829785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789401431477894:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:03.832682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789400454691409:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37185
2025-11-28T14:53:03.832709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:03.832652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789401245364742:2107] [node 4] ICP22 created new session: [2:7577789401245364770:2107]
2025-11-28T14:53:03.832772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789400256646383:2109] [node 6] ICP22 created new session: [1:7577789400256646411:2109]
2025-11-28T14:53:03.829939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:03.832720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789401245364770:2107] [node 4] ICS09 handshake done sender: [2:7577789401245364749:2111] self: [2:7577789401245364746:0] peer: [4:7577789401006720843:0] socket: 306 qp: -1
2025-11-28T14:53:03.833428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789400454691403:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:03.830736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789398967992889:2105] [node 5] ICP22 created new session: [6:7577789398967992927:2105]
2025-11-28T14:53:03.832766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789401245364770:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:03.832991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789400256646411:2109] [node 6] ICS09 handshake done sender: [1:7577789400256646390:2112] self: [1:7577789400256646389:0] peer: [6:7577789398967992924:0] socket: 310 qp: -1
2025-11-28T14:53:03.832985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789401006720846:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6983
2025-11-28T14:53:03.832849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789398967992927:2105] [node 5] ICS09 handshake done sender: [6:7577789398967992913:2124] self: [6:7577789398967992921:0] peer: [5:7577789400454691391:0] socket: 297 qp: -1
2025-11-28T14:53:03.832570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789401431477868:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.833282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789400256646411:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:03.832984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789401006720845:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6983
2025-11-28T14:53:03.833696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789400454691326:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.832790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789401245364680:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.832671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:03.832800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:03.833403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:03.833296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789398967992927:2105] [node 5] ICS10 traffic start
2025-11-28T14:53:03.835509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:03.832726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789401431477868:2108] [node 5] ICP22 created new session: [3:7577789401431477897:2108]
2025-11-28T14:53:03.836131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789400454691388:2109] [node 6] ICP18 handshake (actor [5:7577789400454691402:2115]) is too old
2025-11-28T14:53:03.833334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:03.832954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789401431477897:2108] [node 5] ICS09 handshake done sender: [3:7577789401431477876:2113] self: [3:7577789401431477874:0] peer: [5:7577789400454691407:0] socket: 319 qp: -1
2025-11-28T14:53:03.836135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789400454691326:2050] [node 3] ICP22 created new session: [5:7577789400454691410:2050]
2025-11-28T14:53:03.835287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789401006720761:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15956) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789401006720838:2115] held: no
2025-11-28T14:53:03.833531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789401245364680:2049] [node 3] ICP22 created new session: [2:7577789401245364773:2049]
2025-11-28T14:53:03.833621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:03.837746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789400454691410:2050] [node 3] ICS09 handshake done sender: [5:7577789400454691403:2116] self: [5:7577789400454691407:0] peer: [3:7577789401431477874:0] socket: 323 qp: -1
2025-11-28T14:53:03.832992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789401431477897:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:03.834459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789401245364773:2049] [node 3] ICS09 handshake done sender: [2:7577789401245364747:2110] self: [2:7577789401245364745:0] peer: [3:7577789401431477892:0] socket: 300 qp: -1
2025-11-28T14:53:03.837821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789400256646319:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.837775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789400454691410:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:03.833371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:03.834489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789401245364773:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:03.837987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:03.836423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789401006720838:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:03.835543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:03.838022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.838181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789400256646319:2048] [node 2] ICP22 created new session: [1:7577789400256646413:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.839628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789401245364772:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21566
2025-11-28T14:53:03.840848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992928:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.839969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789401431477896:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25441
2025-11-28T14:53:03.840230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789401006720761:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.838402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789400256646413:2048] [node 2] ICS09 handshake done sender: [1:7577789400256646388:2111] self: [1:7577789400256646386:0] peer: [2:7577789401245364764:0] socket: 303 qp: -1
2025-11-28T14:53:03.838511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789400256646413:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:03.840273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:03.838532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:03.840783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789401006720761:2049] [node 2] ICP22 created new session: [4:7577789401006720848:2049]
2025-11-28T14:53:03.840924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789401006720848:2049] [node 2] ICS09 handshake done sender: [4:7577789401006720838:2115] self: [4:7577789401006720843:0] peer: [2:7577789401245364746:0] socket: 309 qp: -1
2025-11-28T14:53:03.841014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789401006720848:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:03.841660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789398967992830:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22718) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789398967992919:2126] held: no
2025-11-28T14:53:03.841210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:03.842221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789401431477805:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15956) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789401431477886:2118] held: no
2025-11-28T14:53:03.842234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789400256646320:2049] [node 3] ICP18 handshake (actor [1:7577789400256646401:2117]) is too old
No RDMA context found for GID: 2025-11-28T14:53:03.845607Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789401245364759:2116] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.846886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789400454691413:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37185
2025-11-28T14:53:03.846948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789400454691412:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37185
2025-11-28T14:53:03.846793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789401245364679:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22718) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789401245364759:2116] held: no
2025-11-28T14:53:03.846619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789401006720851:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6983
2025-11-28T14:53:03.847327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789398967992919:2126] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:03.847438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789400454691412:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.849537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789401006720846:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.849858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789401006720760:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.849374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789401431477886:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:03.849894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789401245364679:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.850139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:03.850220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789401245364679:2048] [node 1] ICP22 created new session: [2:7577789401245364777:2048]
2025-11-28T14:53:03.850518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789401245364777:2048] [node 1] ICS09 handshake done sender: [2:7577789401245364759:2116] self: [2:7577789401245364764:0] peer: [1:7577789400256646386:0] socket: 316 qp: -1
2025-11-28T14:53:03.850207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789398967992830:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.850565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789401245364777:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:03.850738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:03.850541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:03.850876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646393:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.850722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789398967992830:2048] [node 1] ICP22 created new session: [6:7577789398967992933:2048]
2025-11-28T14:53:03.850906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789400256646393:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.850885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789398967992933:2048] [node 1] ICS09 handshake done sender: [6:7577789398967992919:2126] self: [6:7577789398967992924:0] peer: [1:7577789400256646389:0] socket: 321 qp: -1
2025-11-28T14:53:03.851265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789400454691387:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.850964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789398967992933:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.852559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789401431477805:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.852648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789400256646416:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48728
2025-11-28T14:53:03.851196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:03.852709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:03.852748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789401431477805:2049] [node 2] ICP22 created new session: [3:7577789401431477902:2049]
2025-11-28T14:53:03.852855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789401431477902:2049] [node 2] ICS09 handshake done sender: [3:7577789401431477886:2118] self: [3:7577789401431477892:0] peer: [2:7577789401245364745:0] socket: 312 qp: -1
2025-11-28T14:53:03.853124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789400454691413:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.852874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789401431477902:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:03.853137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:03.853436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789400256646393:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:03.853467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789400454691415:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37185
2025-11-28T14:53:03.853475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789400454691325:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.853442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646416:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.853604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789401006720837:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.853641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789401006720837:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:53:03.853926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789401245364758:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-28T14:53:03.853974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789401245364758:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.853931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789400256646382:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:53:03.855571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.855557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992932:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.856143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992931:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.856123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789400454691415:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.856074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789400256646382:2107] [node 4] ICP22 created new session: [1:7577789400256646419:2107]
2025-11-28T14:53:03.856119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789401245364758:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:03.855822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789401431477901:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25441
2025-11-28T14:53:03.855722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789401006720853:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6983
2025-11-28T14:53:03.856117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789401006720837:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:03.856228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789400256646419:2107] [node 4] ICS09 handshake done sender: [1:7577789400256646393:2114] self: [1:7577789400256646391:0] peer: [4:7577789401006720852:0] socket: 346 qp: -1
2025-11-28T14:53:03.856309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789398967992932:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.856907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789400256646419:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:03.856931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789400454691324:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.856957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:03.856976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789401245364743:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.857678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:03.857135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789398967992831:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:53:03.857375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789401431477901:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:53:03.857482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789398967992931:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.857851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789400256646395:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.857890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789401245364743:2108] [node 5] ICP22 created new session: [2:7577789401245364780:2108]
2025-11-28T14:53:03.857534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789401006720760:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22718) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789401006720846:2118] held: no
2025-11-28T14:53:03.857879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789400256646395:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.858001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789398967992832:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:03.858071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789401006720826:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.858294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789401245364757:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.858300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789400454691418:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37185
2025-11-28T14:53:03.858526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789401431477880:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.858564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789401431477880:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.858350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:03.858308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789401245364780:2108] [node 5] ICS09 handshake done sender: [2:7577789401245364758:2115] self: [2:7577789401245364753:0] peer: [5:7577789400454691417:0] socket: 327 qp: -1
2025-11-28T14:53:03.858545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789401006720826:2108] [node 5] ICP22 created new session: [4:7577789401006720855:2108]
2025-11-28T14:53:03.858313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789401245364757:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.858360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789401245364780:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:03.861077Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789400454691325:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15956) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789400454691413:2119] held: no
2025-11-28T14:53:03.858641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789401006720851:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:03.858398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:03.861557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789401245364757:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:03.861197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789401006720855:2108] [node 5] ICS09 handshake done sender: [4:7577789401006720837:2114] self: [4:7577789401006720836:0] peer: [5:7577789400454691416:0] socket: 292 qp: -1
2025-11-28T14:53:03.861570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789401431477880:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:03.861543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789401006720855:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:03.861581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:03.861637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789401006720846:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:03.861709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789401245364744:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.862431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789401006720760:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.862234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789400454691412:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:03.862473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:03.862244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789401431477870:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.862440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789401006720762:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.862457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:03.862520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789401245364744:2109] [node 6] ICP22 created new session: [2:7577789401245364783:2109]
2025-11-28T14:53:03.862275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789400256646395:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:03.862546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789401006720760:2048] [node 1] ICP22 created new session: [4:7577789401006720858:2048]
2025-11-28T14:53:03.862938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789401245364783:2109] [node 6] ICS09 handshake done sender: [2:7577789401245364757:2114] self: [2:7577789401245364752:0] peer: [6:7577789398967992935:0] socket: 332 qp: -1
2025-11-28T14:53:03.862972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:03.863016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992937:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.862974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789401006720858:2048] [node 1] ICS09 handshake done sender: [4:7577789401006720846:2118] self: [4:7577789401006720852:0] peer: [1:7577789400256646391:0] socket: 349 qp: -1
2025-11-28T14:53:03.862968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789401245364783:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:03.863020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789401431477870:2109] [node 6] ICP22 created new session: [3:7577789401431477906:2109]
2025-11-28T14:53:03.863643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789400256646384:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.863057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:03.863567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789398967992938:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64622
2025-11-28T14:53:03.863011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789401006720858:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:03.863564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789401431477906:2109] [node 6] ICS09 handshake done sender: [3:7577789401431477880:2115] self: [3:7577789401431477878:0] peer: [6:7577789398967992936:0] socket: 305 qp: -1
2025-11-28T14:53:03.864380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789400454691387:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18203) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789400454691412:2118] held: no
2025-11-28T14:53:03.864991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:03.864322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:03.863573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789401431477879:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.863594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789401431477879:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:03.865211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789400256646384:2108] [node 5] ICP22 created new session: [1:7577789400256646422:2108]
2025-11-28T14:53:03.866017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789400454691420:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37185
2025-11-28T14:53:03.866616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789398967992831:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15956) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789398967992932:2131] held: no
2025-11-28T14:53:03.864338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789401431477906:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:03.865881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789400256646422:2108] [node 5] ICS09 handshake done sender: [1:7577789400256646395:2115] self: [1:7577789400256646394:0] peer: [5:7577789400454691419:0] socket: 322 qp: -1
2025-11-28T14:53:03.866526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789400256646422:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:03.864992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:03.866552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.865223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789401431477879:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:03.868019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789400454691387:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.865995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789401431477869:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:03.867959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:03.868137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789401431477869:2107] [node 4] ICP22 created new session: [3:7577789401431477909:2107]
2025-11-28T14:53:03.868449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:03.868914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789401006720860:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6983
2025-11-28T14:53:03.868558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789401431477909:2107] [node 4] ICS09 handshake done sender: [3:7577789401431477879:2114] self: [3:7577789401431477877:0] peer: [4:7577789401006720857:0] socket: 308 qp: -1
2025-11-28T14:53:03.868727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789401431477909:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:03.868949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789398967992832:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24931) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789398967992931:2130] held: no
2025-11-28T14:53:03.868567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789400454691387:2108] [node 4] ICP22 created new session: [5:7577789400454691422:2108]
2025-11-28T14:53:03.868949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789400454691422:2108] [node 4] ICS09 handshake done sender: [5:7577789400454691412:2118] self: [5:7577789400454691416:0] peer: [4:7577789401006720836:0] socket: 318 qp: -1
2025-11-28T14:53:03.868766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:03.868988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789400454691422:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:03.869656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789398967992931:2130] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:03.869023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:03.869651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789400454691324:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22718) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789400454691415:2120] held: no
2025-11-28T14:53:03.869863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789398967992932:2131] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:03.870754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789400454691413:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:03.872139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789398967992831:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:03.872199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:03.872241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789398967992831:2049] [node 2] ICP22 created new session: [6:7577789398967992939:2049]
2025-11-28T14:53:03.872447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789398967992939:2049] [node 2] ICS09 handshake done sender: [6:7577789398967992932:2131] self: [6:7577789398967992935:0] peer: [2:7577789401245364752:0] socket: 313 qp: -1
2025-11-28T14:53:03.872607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789400454691421:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37185
2025-11-28T14:53:03.872418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789398967992832:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.872586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789398967992939:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:03.872649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:03.872678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:03.872842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789400454691325:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.872798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789398967992832:2050] [node 3] ICP22 created new session: [6:7577789398967992941:2050]
2025-11-28T14:53:03.872925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789398967992941:2050] [node 3] ICS09 handshake done sender: [6:7577789398967992931:2130] self: [6:7577789398967992936:0] peer: [3:7577789401431477878:0] socket: 314 qp: -1
2025-11-28T14:53:03.872964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789398967992941:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:03.873097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:03.872992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:03.873143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789401006720851:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:03.873102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789400454691325:2049] [node 2] ICP22 created new session: [5:7577789400454691424:2049]
2025-11-28T14:53:03.873229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789401006720762:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24931) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789401006720851:2119] held: no
2025-11-28T14:53:03.873137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789400454691424:2049] [node 2] ICS09 handshake done sender: [5:7577789400454691413:2119] self: [5:7577789400454691417:0] peer: [2:7577789401245364753:0] socket: 336 qp: -1
2025-11-28T14:53:03.873209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789400454691424:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:03.873232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:03.873844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789400454691415:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:03.873922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789401006720762:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.874026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:03.874063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789401006720762:2050] [node 3] ICP22 created new session: [4:7577789401006720861:2050]
2025-11-28T14:53:03.874100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789401006720861:2050] [node 3] ICS09 handshake done sender: [4:7577789401006720851:2119] self: [4:7577789401006720857:0] peer: [3:7577789401431477877:0] socket: 324 qp: -1
2025-11-28T14:53:03.874127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789401006720861:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:03.874209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:03.874279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789400454691324:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:03.874331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:03.874421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789400454691324:2048] [node 1] ICP22 created new session: [5:7577789400454691426:2048]
2025-11-28T14:53:03.874516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789400454691426:2048] [node 1] ICS09 handshake done sender: [5:7577789400454691415:2120] self: [5:7577789400454691419:0] peer: [1:7577789400256646394:0] socket: 353 qp: -1
2025-11-28T14:53:03.874564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789400454691426:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:03.874604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:03.929311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:03.929584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789401245364777:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-11-28T14:53:03.929637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789401245364679:2048] [node 1] ICP30 unregister session Session# [2:7577789401245364777:2048] VirtualId# [2:7577789401245364764:0]
2025-11-28T14:53:03.929673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789401245364777:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.929818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.929913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.930642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:03.930813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789401431477894:2048] [node 1] ICS07 socket disconnect 315 reason# EndOfStream
2025-11-28T14:53:03.930846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789401431477894:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.930928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.930988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789401431477894:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:03.933235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:03.933298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789401431477804:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22718) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:03.933345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789401431477894:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:03.933460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789401006720858:2048] [node 1] ICS01 socket: 349 reason# EndOfStream
2025-11-28T14:53:03.933780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789401006720760:2048] [node 1] ICP30 unregister session Session# [4:7577789401006720858:2048] VirtualId# [4:7577789401006720852:0]
2025-11-28T14:53:03.933829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789401006720858:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.933976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.934026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.934578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789401431477804:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22718) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:03.934686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789401431477894:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:03.934722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789401431477804:2048] [node 1] ICP30 unregister session Session# [3:7577789401431477894:2048] VirtualId# [3:7577789401431477890:0]
2025-11-28T14:53:03.934783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.934823Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789401431477804:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22718) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.935052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:03.934839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22718) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:03.935254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789400454691426:2048] [node 1] ICS01 socket: 353 reason# EndOfStream
2025-11-28T14:53:03.935304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789400454691324:2048] [node 1] ICP30 unregister session Session# [5:7577789400454691426:2048] VirtualId# [5:7577789400454691419:0]
2025-11-28T14:53:03.935330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789400454691426:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.935387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.935446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.936206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:03.936458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789398967992933:2048] [node 1] ICS01 socket: 321 reason# EndOfStream
2025-11-28T14:53:03.936535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789398967992830:2048] [node 1] ICP30 unregister session Session# [6:7577789398967992933:2048] VirtualId# [6:7577789398967992924:0]
2025-11-28T14:53:03.936631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789398967992933:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.936731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:03.936782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:03.944398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789401431477804:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:03.996311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:03.996489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789401431477902:2049] [node 2] ICS01 socket: 312 reason# EndOfStream
2025-11-28T14:53:03.996525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789401431477805:2049] [node 2] ICP30 unregister session Session# [3:7577789401431477902:2049] VirtualId# [3:7577789401431477892:0]
2025-11-28T14:53:03.996558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789401431477902:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.996651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:03.996687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:03.999155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:03.999322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789401006720848:2049] [node 2] ICS01 socket: 309 reason# EndOfStream
2025-11-28T14:53:03.999362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789401006720761:2049] [node 2] ICP30 unregister session Session# [4:7577789401006720848:2049] VirtualId# [4:7577789401006720843:0]
2025-11-28T14:53:03.999389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789401006720848:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:03.999448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:03.999493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:04.000275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:04.000405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789400454691424:2049] [node 2] ICS01 socket: 336 reason# EndOfStream
2025-11-28T14:53:04.000431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789400454691325:2049] [node 2] ICP30 unregister session Session# [5:7577789400454691424:2049] VirtualId# [5:7577789400454691417:0]
2025-11-28T14:53:04.000448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789400454691424:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.000498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:04.000550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:04.001308Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:04.001534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789398967992939:2049] [node 2] ICS01 socket: 313 reason# EndOfStream
2025-11-28T14:53:04.001601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789398967992831:2049] [node 2] ICP30 unregister session Session# [6:7577789398967992939:2049] VirtualId# [6:7577789398967992935:0]
2025-11-28T14:53:04.001633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789398967992939:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.001736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:04.001790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:04.049726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:04.049955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789401006720861:2050] [node 3] ICS01 socket: 324 reason# EndOfStream
2025-11-28T14:53:04.050047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789401006720762:2050] [node 3] ICP30 unregister session Session# [4:7577789401006720861:2050] VirtualId# [4:7577789401006720857:0]
2025-11-28T14:53:04.050092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789401006720861:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.050179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:04.050232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:04.050562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:04.050773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789400454691410:2050] [node 3] ICS07 socket disconnect 323 reason# EndOfStream
2025-11-28T14:53:04.050816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789400454691410:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.050900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:04.050953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789400454691410:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:04.051306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:04.051536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789398967992941:2050] [node 3] ICS01 socket: 314 reason# EndOfStream
2025-11-28T14:53:04.051593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789398967992832:2050] [node 3] ICP30 unregister session Session# [6:7577789398967992941:2050] VirtualId# [6:7577789398967992936:0]
2025-11-28T14:53:04.051632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789398967992941:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.051714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:04.051772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:04.051724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789400454691326:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24931) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.051795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789400454691410:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.051866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789400454691326:2050] [node 3] ICP30 unregister session Session# [5:7577789400454691410:2050] VirtualId# [5:7577789400454691407:0]
2025-11-28T14:53:04.051910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:04.051948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789400454691326:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24931) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.051964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24931) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.062157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789400454691326:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:04.104855Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:04.105035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789400454691422:2108] [node 4] ICS01 socket: 318 reason# EndOfStream
2025-11-28T14:53:04.105070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789400454691387:2108] [node 4] ICP30 unregister session Session# [5:7577789400454691422:2108] VirtualId# [5:7577789400454691416:0]
2025-11-28T14:53:04.105101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789400454691422:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.105185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:04.105229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:04.106154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:04.106375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789398967992925:2108] [node 4] ICS07 socket disconnect 295 reason# EndOfStream
2025-11-28T14:53:04.106408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789398967992925:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.106505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:04.106555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789398967992925:2108] [node 4] ICS15 start handshake
2025-11-28T14:53:04.107426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789398967992903:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18203) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.107519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789398967992925:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.107552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789398967992903:2108] [node 4] ICP30 unregister session Session# [6:7577789398967992925:2108] VirtualId# [6:7577789398967992920:0]
2025-11-28T14:53:04.107675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:04.107705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789398967992903:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.107717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.117423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789398967992903:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:53:04.155930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:04.156184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789398967992927:2105] [node 5] ICS07 socket disconnect 297 reason# EndOfStream
2025-11-28T14:53:04.156219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789398967992927:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.156313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:04.156432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789398967992927:2105] [node 5] ICS15 start handshake
2025-11-28T14:53:04.157317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789398967992889:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16785) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.157421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789398967992927:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.157480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789398967992889:2105] [node 5] ICP30 unregister session Session# [6:7577789398967992927:2105] VirtualId# [6:7577789398967992921:0]
2025-11-28T14:53:04.157584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:53:04.157623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789398967992889:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16785) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.157637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16785) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.167655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789398967992889:2105] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:04.294335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789406405281340:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:04.294470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789406405281339:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:04.315819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789405187450618:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:04.316133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789405187450617:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:04.340763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789403672025045:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:04.340790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789403672025046:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:04.361705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789406528160212:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:04.361711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789406528160211:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:04.361720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789406528160213:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:04.384073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789403332954973:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:04.384071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789403332954972:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:04.384315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789403332954974:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:04.405630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789405489474451:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:04.405675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789405489474452:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:04.405715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789405489474453:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:04.469864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789405489474511:2105] [node 5] ICP01 ready to work
2025-11-28T14:53:04.478447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789405187450678:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:04.484282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789403672025106:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:04.488658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789406405281401:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:04.522286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789405187450682:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:04.512965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789406405281412:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:04.535622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789406528160275:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:04.535472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789403672025111:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:04.522162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789406405281413:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:04.537160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789403672025110:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:04.537293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789403332955036:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:04.538999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789403332955037:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:04.539128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789405187450683:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:04.537154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789406528160274:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:04.538858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789405489474515:2109] [node 4] ICP01 ready to work
2025-11-28T14:53:04.540960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789406405281339:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(69924851ns) {Configure:104(69908071ns) {ProcessConfigured:135(274527ns) {StartInitialHandshake:157(271863ns) {DropHandshakes:470(2135ns) {DropIncomingHandshake:442(670ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(558ns) PrepareNewSessionHandshake:477(1145ns) {SwitchToState:179(413ns)}}}}}
2025-11-28T14:53:04.541009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789405489474511:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(70533571ns) {Configure:104(70527982ns) {ProcessConfigured:135(1279211ns) {StartInitialHandshake:157(1276495ns) {DropHandshakes:470(2533ns) {DropIncomingHandshake:442(836ns) DropOutgoingHandshake:457(404ns)} GenerateSessionVirtualId:988(515ns) PrepareNewSessionHandshake:477(2076ns) {SwitchToState:179(497ns)}}}}}
2025-11-28T14:53:04.547390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789403672025045:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(62913709ns) {Configure:104(62907695ns) {ProcessConfigured:135(419766ns) {StartInitialHandshake:157(417053ns) {DropHandshakes:470(933ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:477(1218ns) {SwitchToState:179(169ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-28T14:53:04.555714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789406405281427:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.560010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789403672025123:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36463
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
2025-11-28T14:53:04.560474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789403672025124:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36463
2025-11-28T14:53:04.560471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789406528160284:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18994
2025-11-28T14:53:04.561038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789405187450692:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15229
2025-11-28T14:53:04.561278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789403332955046:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15740
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.561287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789403672025123:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.561246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789406405281428:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-28T14:53:04.561257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789405187450693:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15229
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.562868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789403672025045:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.565213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474529:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.566062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281428:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.566479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789405187450694:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15229
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.567634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789405489474529:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.567305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789403672025125:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36463
2025-11-28T14:53:04.567171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281421:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.568672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789406528160284:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.567875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789403332955046:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.567179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789405187450692:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.569005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789403672025125:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.567923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789405489474453:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.567201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789406405281421:2122] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.567622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789405187450698:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15229
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.569295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789406528160288:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18994
2025-11-28T14:53:04.569176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789403672025110:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16224246ns) {Configure:104(16220004ns) {ProcessConfigured:135(690142ns) {StartInitialHandshake:157(688043ns) {DropHandshakes:470(2390ns) {DropIncomingHandshake:442(648ns) DropOutgoingHandshake:457(401ns)} GenerateSessionVirtualId:988(627ns) PrepareNewSessionHandshake:477(171986ns) {SwitchToState:179(2471ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.568675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789405187450693:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.569607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281427:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.569715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789403672025119:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.569753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789406528160211:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.568877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789405187450694:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.569694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789406405281429:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.570274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789406528160288:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.569746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789403672025119:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.569840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789406405281421:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:04.569149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789405187450617:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.573312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789405187450698:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.570069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789403672025126:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36463
2025-11-28T14:53:04.571460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789406528160291:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18994
2025-11-28T14:53:04.569862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281415:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.570136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789403672025124:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.574073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789406528160291:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.569888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789406405281415:2119] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.573720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789403672025126:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.576412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789405187450687:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-28T14:53:04.575043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789406528160212:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.572151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789406405281430:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-28T14:53:04.574939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789403672025119:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:04.576446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789405187450686:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.574975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789403672025131:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36463
2025-11-28T14:53:04.572989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789406405281431:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-28T14:53:04.578837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474531:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.576452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789405187450687:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.573812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281429:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.576191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789403672025046:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.576463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789405187450686:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.574043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281417:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.578263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789403672025111:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.578358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789405187450687:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:04.580815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789405489474529:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:04.574081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789406405281417:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.579096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789403672025123:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:04.582043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789406528160295:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18994
2025-11-28T14:53:04.581606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789405489474453:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.578831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789405187450705:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15229
2025-11-28T14:53:04.574965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789406405281415:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:04.579242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:04.581032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789405187450686:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:04.582086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:04.576136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281430:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.581388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789405187450692:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:04.583803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789406528160296:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18994
2025-11-28T14:53:04.582545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789405489474453:2050] [node 3] ICP22 created new session: [6:7577789405489474533:2050]
2025-11-28T14:53:04.576203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789406405281340:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.581843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789405187450678:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.583033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789405489474533:2050] [node 3] ICS09 handshake done sender: [6:7577789405489474529:2115] self: [6:7577789405489474530:0] peer: [3:7577789403672025117:0] socket: 315 qp: -1
2025-11-28T14:53:04.582513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.576262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:04.583018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789405187450678:2107] [node 4] ICP22 created new session: [2:7577789405187450708:2107]
2025-11-28T14:53:04.583518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789405187450618:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.576296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789406405281340:2049] [node 3] ICP22 created new session: [1:7577789406405281436:2049]
2025-11-28T14:53:04.583655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789405489474533:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:04.579244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789403672025045:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789403672025123:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.576343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789406405281436:2049] [node 3] ICS09 handshake done sender: [1:7577789406405281421:2122] self: [1:7577789406405281419:0] peer: [3:7577789403672025127:0] socket: 297 qp: -1
2025-11-28T14:53:04.579696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789403672025111:2109] [node 6] ICP22 created new session: [3:7577789403672025135:2109]
2025-11-28T14:53:04.583568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:04.585277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474532:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.576473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789406405281436:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:04.585558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789403332955053:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15740
2025-11-28T14:53:04.583664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789405187450708:2107] [node 4] ICS09 handshake done sender: [2:7577789405187450687:2112] self: [2:7577789405187450684:0] peer: [4:7577789406528160294:0] socket: 328 qp: -1
2025-11-28T14:53:04.576603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:04.585566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:04.580801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789403672025135:2109] [node 6] ICS09 handshake done sender: [3:7577789403672025119:2114] self: [3:7577789403672025117:0] peer: [6:7577789405489474530:0] socket: 301 qp: -1
2025-11-28T14:53:04.587399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789406528160212:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32059) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789406528160291:2117] held: no
2025-11-28T14:53:04.577756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789406405281339:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.589253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789405489474532:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.583703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789405187450708:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:04.580852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789403672025135:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:04.589972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789403332955053:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.589028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789406528160211:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789406528160284:2114] held: no
2025-11-28T14:53:04.577804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:04.589955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789405489474451:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.583779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789405187450618:2049] [node 3] ICP22 created new session: [2:7577789405187450709:2049]
2025-11-28T14:53:04.580998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:04.577918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789406405281339:2048] [node 2] ICP22 created new session: [1:7577789406405281439:2048]
2025-11-28T14:53:04.590698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789406528160291:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:04.583819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789405187450709:2049] [node 3] ICS09 handshake done sender: [2:7577789405187450686:2111] self: [2:7577789405187450685:0] peer: [3:7577789403672025134:0] socket: 327 qp: -1
2025-11-28T14:53:04.583993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789403672025045:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.583836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:04.586140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:04.583844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789405187450617:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.578353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789406405281439:2048] [node 2] ICS09 handshake done sender: [1:7577789406405281415:2119] self: [1:7577789406405281414:0] peer: [2:7577789405187450701:0] socket: 295 qp: -1
2025-11-28T14:53:04.587893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789403672025138:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36463
2025-11-28T14:53:04.583861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789405187450709:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:04.578386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789406405281439:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:04.592133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789403332955054:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15740
2025-11-28T14:53:04.592256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789406528160212:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.587930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789403672025045:2048] [node 1] ICP22 created new session: [3:7577789403672025140:2048]
2025-11-28T14:53:04.584067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:04.592671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789403332954972:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.579015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.592467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789406528160284:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:04.589229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789403672025140:2048] [node 1] ICS09 handshake done sender: [3:7577789403672025123:2115] self: [3:7577789403672025127:0] peer: [1:7577789406405281419:0] socket: 304 qp: -1
2025-11-28T14:53:04.585241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:04.579229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281431:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.592662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:04.596065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789403332955054:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.589266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789403672025140:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:04.585743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789405187450617:2048] [node 1] ICP22 created new session: [2:7577789405187450712:2048]
2025-11-28T14:53:04.580655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789406405281417:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:04.593409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789406528160212:2049] [node 2] ICP22 created new session: [4:7577789406528160299:2049]
2025-11-28T14:53:04.589513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:04.597713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789403332954974:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.583655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789406405281401:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.587899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789405187450712:2048] [node 1] ICS09 handshake done sender: [2:7577789405187450692:2115] self: [2:7577789405187450701:0] peer: [1:7577789406405281414:0] socket: 302 qp: -1
2025-11-28T14:53:04.593512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789406528160299:2049] [node 2] ICS09 handshake done sender: [4:7577789406528160291:2117] self: [4:7577789406528160294:0] peer: [2:7577789405187450684:0] socket: 333 qp: -1
2025-11-28T14:53:04.592661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789403672025126:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:04.588856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789405187450712:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:04.593555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789406528160299:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:04.583830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-28T14:53:04.595047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789403672025046:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32059) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789403672025126:2118] held: no
::ffff:127.1.0.0
2025-11-28T14:53:04.594959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:04.588895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:04.583886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789406405281401:2107] [node 4] ICP22 created new session: [1:7577789406405281445:2107]
2025-11-28T14:53:04.596439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789406528160298:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18994
2025-11-28T14:53:04.585237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789406405281445:2107] [node 4] ICS09 handshake done sender: [1:7577789406405281417:2120] self: [1:7577789406405281416:0] peer: [4:7577789406528160292:0] socket: 294 qp: -1
2025-11-28T14:53:04.598494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789403672025129:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.596763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789406528160297:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18994
2025-11-28T14:53:04.598597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474537:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.585283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789406405281445:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:04.596845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789406528160211:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.598529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789403672025129:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.598080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:04.585606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.601486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789405489474532:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:04.598551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789406528160211:2048] [node 1] ICP22 created new session: [4:7577789406528160301:2048]
2025-11-28T14:53:04.601557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789403672025046:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.590640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281420:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.601750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789403672025129:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.598586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789406528160301:2048] [node 1] ICS09 handshake done sender: [4:7577789406528160284:2114] self: [4:7577789406528160292:0] peer: [1:7577789406405281416:0] socket: 306 qp: -1
2025-11-28T14:53:04.590668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789406405281420:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.602368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474539:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.603558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789403332955057:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15740
2025-11-28T14:53:04.601991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789403672025118:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.593501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789406405281423:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.598821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789406528160301:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:04.602016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789403672025118:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.593529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789406405281423:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.603479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789405489474451:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.598832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789406528160297:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.602413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:04.604709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789405187450678:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16607355ns) {HandleHandshakeStatus:329(16604905ns) {DropIncomingHandshake:442(10130ns) DropOutgoingHandshake:457(213ns) SwitchToState:179(306ns) ProcessPendingSessionEvents:495(14482553ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14400596ns max# 1240871ns}}}
2025-11-28T14:53:04.604780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789403332955058:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15740
2025-11-28T14:53:04.595013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789406405281420:2121] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:04.603390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789403672025046:2049] [node 2] ICP22 created new session: [3:7577789403672025143:2049]
2025-11-28T14:53:04.598851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:04.605024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789403332955059:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15740
2025-11-28T14:53:04.598477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789406405281413:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.603549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789403672025143:2049] [node 2] ICS09 handshake done sender: [3:7577789403672025126:2118] self: [3:7577789403672025134:0] peer: [2:7577789405187450685:0] socket: 331 qp: -1
2025-11-28T14:53:04.604016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474538:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.598664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:04.599175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789406528160298:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.605195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789403332954974:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789403332955054:2118] held: no
2025-11-28T14:53:04.606155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789405187450618:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(16568905ns) {HandleHandshakeStatus:329(16566427ns) {DropIncomingHandshake:442(10449ns) DropOutgoingHandshake:457(186ns) SwitchToState:179(365ns) ProcessPendingSessionEvents:495(13944912ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13850346ns max# 2344794ns}}}
2025-11-28T14:53:04.603593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789403672025143:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:04.598711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789406405281413:2109] [node 6] ICP22 created new session: [1:7577789406405281449:2109]
2025-11-28T14:53:04.604326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:04.603712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789403672025110:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.600444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789406528160213:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.603763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:04.598899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789406405281449:2109] [node 6] ICS09 handshake done sender: [1:7577789406405281420:2121] self: [1:7577789406405281418:0] peer: [6:7577789405489474536:0] socket: 346 qp: -1
2025-11-28T14:53:04.606462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789403332955057:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.603779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:04.604512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789405489474539:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.599061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789406405281449:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:04.607295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789403332955058:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.603808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789403672025110:2108] [node 5] ICP22 created new session: [3:7577789403672025145:2108]
2025-11-28T14:53:04.608196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789406528160290:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:53:04.599251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.604513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789403672025145:2108] [node 5] ICS09 handshake done sender: [3:7577789403672025129:2119] self: [3:7577789403672025128:0] peer: [5:7577789403332955056:0] socket: 324 qp: -1
2025-11-28T14:53:04.608247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789406528160290:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.608185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789405187450691:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.604522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789405489474451:2048] [node 1] ICP22 created new session: [6:7577789405489474540:2048]
2025-11-28T14:53:04.608227Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789405187450691:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.602174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789406405281340:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17224677ns) {HandleHandshakeStatus:329(17222539ns) {DropIncomingHandshake:442(12864ns) DropOutgoingHandshake:457(457ns) SwitchToState:179(613ns) ProcessPendingSessionEvents:495(15254943ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15151486ns max# 1355733ns}}}
2025-11-28T14:53:04.604786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789405489474538:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.609871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789405187450617:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18604628ns) {HandleHandshakeStatus:329(18602462ns) {DropIncomingHandshake:442(987ns) DropOutgoingHandshake:457(5253ns) SwitchToState:179(760ns) ProcessPendingSessionEvents:495(12469561ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12260775ns max# 1476976ns}}}
2025-11-28T14:53:04.603412Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789406405281339:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(17412453ns) {HandleHandshakeStatus:329(17410757ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(1036ns) SwitchToState:179(369ns) ProcessPendingSessionEvents:495(15699417ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15600077ns max# 1381675ns}}}
2025-11-28T14:53:04.604931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789405489474540:2048] [node 1] ICS09 handshake done sender: [6:7577789405489474532:2117] self: [6:7577789405489474536:0] peer: [1:7577789406405281418:0] socket: 347 qp: -1
2025-11-28T14:53:04.609956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789403332955060:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15740
2025-11-28T14:53:04.608312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789406405281401:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(19161876ns) {HandleHandshakeStatus:329(18982436ns) {DropIncomingHandshake:442(18525ns) DropOutgoingHandshake:457(412ns) SwitchToState:179(110ns) ProcessPendingSessionEvents:495(13245074ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12297030ns max# 1648424ns}}}
2025-11-28T14:53:04.609981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789403332955036:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.609848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789406405281423:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:04.604783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789403672025145:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:04.610103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789405187450691:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:04.610434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789403332954973:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.605091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789405489474540:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:04.605181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:04.605182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789403672025111:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20235867ns) {HandleHandshakeStatus:329(20233244ns) {DropIncomingHandshake:442(359ns) DropOutgoingHandshake:457(393ns) SwitchToState:179(157ns) ProcessPendingSessionEvents:495(15905663ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15800291ns max# 1791285ns}}}
2025-11-28T14:53:04.606088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789403672025118:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:04.610474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789406405281339:2048] [node 2] ICP18 handshake (actor [1:7577789406405281431:2128]) is too old
2025-11-28T14:53:04.605159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789405489474515:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.610455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789406528160307:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18994
2025-11-28T14:53:04.609848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789403672025106:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.610703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789406405281412:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.610732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789403332955053:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:04.605507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:04.611106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789405187450682:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.611338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789406528160283:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.605953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789405489474452:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.611101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:04.607313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789405489474453:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(18176847ns) {HandleHandshakeStatus:329(18174869ns) {DropIncomingHandshake:442(696ns) DropOutgoingHandshake:457(5429ns) SwitchToState:179(471ns) ProcessPendingSessionEvents:495(12235185ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12150262ns max# 1523339ns}}}
2025-11-28T14:53:04.611424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789403332955054:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:04.611161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:04.609999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:04.611362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789406528160283:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:04.612035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789406405281412:2108] [node 5] ICP22 created new session: [1:7577789406405281451:2108]
2025-11-28T14:53:04.610406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789403672025106:2107] [node 4] ICP22 created new session: [3:7577789403672025148:2107]
2025-11-28T14:53:04.611442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789403332954972:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.611414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789406528160290:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:04.612029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789405187450682:2109] [node 6] ICP22 created new session: [2:7577789405187450717:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.612088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789405187450700:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.610459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789403672025148:2107] [node 4] ICS09 handshake done sender: [3:7577789403672025118:2113] self: [3:7577789403672025113:0] peer: [4:7577789406528160306:0] socket: 307 qp: -1
2025-11-28T14:53:04.612053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:04.613700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789406528160297:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:04.612278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789406405281451:2108] [node 5] ICS09 handshake done sender: [1:7577789406405281423:2123] self: [1:7577789406405281422:0] peer: [5:7577789403332955055:0] socket: 348 qp: -1
2025-11-28T14:53:04.612436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789403332954972:2048] [node 1] ICP22 created new session: [5:7577789403332955064:2048]
2025-11-28T14:53:04.613645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789403332955064:2048] [node 1] ICS09 handshake done sender: [5:7577789403332955053:2117] self: [5:7577789403332955055:0] peer: [1:7577789406405281422:0] socket: 349 qp: -1
2025-11-28T14:53:04.610624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789403672025148:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:04.612311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789406405281451:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:04.616917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474545:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.614861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789406528160275:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.612118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789405187450700:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.612345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:04.613279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789405187450717:2109] [node 6] ICS09 handshake done sender: [2:7577789405187450691:2114] self: [2:7577789405187450689:0] peer: [6:7577789405489474543:0] socket: 316 qp: -1
2025-11-28T14:53:04.616142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:04.610666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:04.613702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789403332954974:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.616859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789406528160275:2109] [node 6] ICP22 created new session: [4:7577789406528160310:2109]
2025-11-28T14:53:04.618155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789406528160310:2109] [node 6] ICS09 handshake done sender: [4:7577789406528160290:2116] self: [4:7577789406528160289:0] peer: [6:7577789405489474541:0] socket: 330 qp: -1
2025-11-28T14:53:04.613319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789405187450717:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:04.618193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789406528160310:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:04.614711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789403332955064:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:04.613706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:04.618238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:04.618696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789406528160213:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.614977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:04.619503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474544:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.619182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:04.615350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789405187450700:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:04.615165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: 2025-11-28T14:53:04.619416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789406528160213:2050] [node 3] ICP22 created new session: [4:7577789406528160312:2050]
::ffff:127.1.0.0
2025-11-28T14:53:04.615242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789403332954974:2050] [node 3] ICP22 created new session: [5:7577789403332955066:2050]
2025-11-28T14:53:04.617299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789405187450683:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.619467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789406528160312:2050] [node 3] ICS09 handshake done sender: [4:7577789406528160297:2120] self: [4:7577789406528160306:0] peer: [3:7577789403672025113:0] socket: 351 qp: -1
2025-11-28T14:53:04.618194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:04.615556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789403332955066:2050] [node 3] ICS09 handshake done sender: [5:7577789403332955054:2118] self: [5:7577789403332955056:0] peer: [3:7577789403672025128:0] socket: 354 qp: -1
2025-11-28T14:53:04.619494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789406528160312:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:04.616087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789403332955066:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:04.618678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789405187450683:2108] [node 5] ICP22 created new session: [2:7577789405187450721:2108]
2025-11-28T14:53:04.620063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789405489474452:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32059) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789405489474538:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.616126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:04.618787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789405187450721:2108] [node 5] ICS09 handshake done sender: [2:7577789405187450700:2119] self: [2:7577789405187450699:0] peer: [5:7577789403332955062:0] socket: 356 qp: -1
2025-11-28T14:53:04.619724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:04.618991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789405187450721:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:04.620022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789403332955068:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15740
2025-11-28T14:53:04.620975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789403332955057:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:04.619960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789406528160283:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:04.619105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789405187450720:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15229
2025-11-28T14:53:04.620986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789403332955069:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15740
2025-11-28T14:53:04.619131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:04.621447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789406528160274:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.621500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:04.620134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789405187450720:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.621650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789403332955036:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.621983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789406528160274:2108] [node 5] ICP22 created new session: [4:7577789406528160314:2108]
2025-11-28T14:53:04.622313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789406528160314:2108] [node 5] ICS09 handshake done sender: [4:7577789406528160283:2113] self: [4:7577789406528160282:0] peer: [5:7577789403332955061:0] socket: 300 qp: -1
2025-11-28T14:53:04.621992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:04.622034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789403332955036:2108] [node 4] ICP22 created new session: [5:7577789403332955071:2108]
2025-11-28T14:53:04.623380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789406528160314:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:04.623925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789405489474539:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:04.623612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:04.622242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789403332955071:2108] [node 4] ICS09 handshake done sender: [5:7577789403332955057:2119] self: [5:7577789403332955061:0] peer: [4:7577789406528160282:0] socket: 353 qp: -1
2025-11-28T14:53:04.623946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789405489474451:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16757834ns) {HandleHandshakeStatus:329(16755511ns) {DropIncomingHandshake:442(2133ns) DropOutgoingHandshake:457(7600ns) SwitchToState:179(255ns) ProcessPendingSessionEvents:495(11739138ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11663825ns max# 1645488ns}}}
2025-11-28T14:53:04.624212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789405489474515:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12874) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789405489474539:2120] held: no
2025-11-28T14:53:04.622267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789403332955071:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:04.622849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:04.623831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789403332955058:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:04.624414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789403332954973:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.624782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:04.624887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789403332954973:2049] [node 2] ICP22 created new session: [5:7577789403332955073:2049]
2025-11-28T14:53:04.624915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789406528160213:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:04.624940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789403332955073:2049] [node 2] ICS09 handshake done sender: [5:7577789403332955058:2120] self: [5:7577789403332955062:0] peer: [2:7577789405187450699:0] socket: 357 qp: -1
2025-11-28T14:53:04.625023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789403332955073:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:04.625055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:04.625084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789405489474538:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:04.625337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789405187450683:2108] [node 5] ICP18 handshake (actor [2:7577789405187450720:2121]) is too old
2025-11-28T14:53:04.625529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789405489474515:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.625663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:04.625689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789405489474452:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.625705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789405489474515:2109] [node 4] ICP22 created new session: [6:7577789405489474547:2109]
2025-11-28T14:53:04.625714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:04.625783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789405489474547:2109] [node 4] ICS09 handshake done sender: [6:7577789405489474539:2120] self: [6:7577789405489474541:0] peer: [4:7577789406528160289:0] socket: 358 qp: -1
2025-11-28T14:53:04.625790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789405489474452:2049] [node 2] ICP22 created new session: [6:7577789405489474548:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.625817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789405489474547:2109] [node 4] ICS10 traffic start
2025-11-28T14:53:04.625820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789405489474548:2049] [node 2] ICS09 handshake done sender: [6:7577789405489474538:2119] self: [6:7577789405489474543:0] peer: [2:7577789405187450689:0] socket: 355 qp: -1
2025-11-28T14:53:04.625844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789405489474548:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:04.625868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:04.625895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:04.626925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474546:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.627870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789405489474546:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.627961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789405489474511:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:04.628164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789403332955051:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:04.628196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789403332955051:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:04.629265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789403332955051:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:04.629388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789405489474552:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56117
2025-11-28T14:53:04.629478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789403332955037:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:04.629657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789405489474546:2123] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:04.629670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:04.629752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789403332955037:2109] [node 6] ICP22 created new session: [5:7577789403332955076:2109]
2025-11-28T14:53:04.629840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789405489474511:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:04.630106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789403332955076:2109] [node 6] ICS09 handshake done sender: [5:7577789403332955051:2116] self: [5:7577789403332955050:0] peer: [6:7577789405489474551:0] socket: 298 qp: -1
2025-11-28T14:53:04.630153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789403332955076:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:04.630166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:04.630205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:04.630206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789405489474511:2105] [node 5] ICP22 created new session: [6:7577789405489474553:2105]
2025-11-28T14:53:04.630246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789405489474553:2105] [node 5] ICS09 handshake done sender: [6:7577789405489474546:2123] self: [6:7577789405489474551:0] peer: [5:7577789403332955050:0] socket: 314 qp: -1
2025-11-28T14:53:04.630276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789405489474553:2105] [node 5] ICS10 traffic start
2025-11-28T14:53:04.630358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:04.631256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789405489474511:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:04.686898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:04.687078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789405187450712:2048] [node 1] ICS07 socket disconnect 302 reason# EndOfStream
2025-11-28T14:53:04.687107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789405187450712:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.687202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:04.687248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789405187450712:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:04.687920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:04.688130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789403672025140:2048] [node 1] ICS01 socket: 304 reason# EndOfStream
2025-11-28T14:53:04.688184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789403672025045:2048] [node 1] ICP30 unregister session Session# [3:7577789403672025140:2048] VirtualId# [3:7577789403672025127:0]
2025-11-28T14:53:04.688223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789403672025140:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.688306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:04.688356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:04.689415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789405187450617:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:04.689445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789405187450712:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:04.690416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789405187450617:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.690458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789405187450712:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.690522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789405187450617:2048] [node 1] ICP30 unregister session Session# [2:7577789405187450712:2048] VirtualId# [2:7577789405187450701:0]
2025-11-28T14:53:04.690629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:04.690660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789405187450617:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.690670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.692624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:04.692852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789406528160301:2048] [node 1] ICS01 socket: 306 reason# EndOfStream
2025-11-28T14:53:04.692897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789406528160211:2048] [node 1] ICP30 unregister session Session# [4:7577789406528160301:2048] VirtualId# [4:7577789406528160292:0]
2025-11-28T14:53:04.692929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789406528160301:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.693011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:04.693068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:04.693782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:04.694019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789403332955064:2048] [node 1] ICS07 socket disconnect 349 reason# EndOfStream
2025-11-28T14:53:04.694061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789403332955064:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.694176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:04.694251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789403332955064:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:04.694763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:04.694955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789405489474540:2048] [node 1] ICS07 socket disconnect 347 reason# EndOfStream
2025-11-28T14:53:04.694992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789405489474540:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.695065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:04.695085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789403332954972:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.695219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789403332955064:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.695111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789405489474540:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:04.695409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789403332954972:2048] [node 1] ICP30 unregister session Session# [5:7577789403332955064:2048] VirtualId# [5:7577789403332955055:0]
2025-11-28T14:53:04.695551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:04.695598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789403332954972:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.695613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.696026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789405489474451:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.696135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789405489474540:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.696178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789405489474451:2048] [node 1] ICP30 unregister session Session# [6:7577789405489474540:2048] VirtualId# [6:7577789405489474536:0]
2025-11-28T14:53:04.696287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:04.696326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789405489474451:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.696340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.700094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789405187450617:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:04.705226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789403332954972:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:04.706239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789405489474451:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:04.768515Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:04.768774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789403672025143:2049] [node 2] ICS01 socket: 331 reason# EndOfStream
2025-11-28T14:53:04.768821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789403672025046:2049] [node 2] ICP30 unregister session Session# [3:7577789403672025143:2049] VirtualId# [3:7577789403672025134:0]
2025-11-28T14:53:04.768872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789403672025143:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.769059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:04.769249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:04.770017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:04.770204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789406528160299:2049] [node 2] ICS01 socket: 333 reason# EndOfStream
2025-11-28T14:53:04.770246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789406528160212:2049] [node 2] ICP30 unregister session Session# [4:7577789406528160299:2049] VirtualId# [4:7577789406528160294:0]
2025-11-28T14:53:04.770270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789406528160299:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.770324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:04.770357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:04.771260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:04.771393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789403332955073:2049] [node 2] ICS07 socket disconnect 357 reason# EndOfStream
2025-11-28T14:53:04.771413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789403332955073:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.771463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:04.771507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789403332955073:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:04.772391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789403332954973:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32059) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.772444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789403332955073:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.772486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789403332954973:2049] [node 2] ICP30 unregister session Session# [5:7577789403332955073:2049] VirtualId# [5:7577789403332955062:0]
2025-11-28T14:53:04.772358Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:04.772596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:04.772628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789403332954973:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32059) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.772576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789405489474548:2049] [node 2] ICS01 socket: 355 reason# EndOfStream
2025-11-28T14:53:04.772647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32059) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.772604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789405489474452:2049] [node 2] ICP30 unregister session Session# [6:7577789405489474548:2049] VirtualId# [6:7577789405489474543:0]
2025-11-28T14:53:04.772627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789405489474548:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.772693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:04.772754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:04.782003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789403332954973:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:04.852782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:04.852982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789406528160312:2050] [node 3] ICS07 socket disconnect 351 reason# EndOfStream
2025-11-28T14:53:04.853018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789406528160312:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.853113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:04.853153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789406528160312:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:04.853926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789406528160213:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.853985Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:04.854024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789406528160312:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.854063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789406528160213:2050] [node 3] ICP30 unregister session Session# [4:7577789406528160312:2050] VirtualId# [4:7577789406528160306:0]
2025-11-28T14:53:04.854183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:04.854216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789406528160213:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.854206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789403332955066:2050] [node 3] ICS01 socket: 354 reason# EndOfStream
2025-11-28T14:53:04.854230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.854287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789403332954974:2050] [node 3] ICP30 unregister session Session# [5:7577789403332955066:2050] VirtualId# [5:7577789403332955056:0]
2025-11-28T14:53:04.854323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789403332955066:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.854406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:04.854465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:04.855201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:04.855403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789405489474533:2050] [node 3] ICS07 socket disconnect 315 reason# EndOfStream
2025-11-28T14:53:04.855446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789405489474533:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.855529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:04.855583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789405489474533:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:04.856410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789405489474453:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.856509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789405489474533:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.856568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789405489474453:2050] [node 3] ICP30 unregister session Session# [6:7577789405489474533:2050] VirtualId# [6:7577789405489474530:0]
2025-11-28T14:53:04.856699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:04.856729Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789405489474453:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.856740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.867012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789405489474453:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:04.925183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:04.925341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789403332955071:2108] [node 4] ICS01 socket: 353 reason# EndOfStream
2025-11-28T14:53:04.925375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789403332955036:2108] [node 4] ICP30 unregister session Session# [5:7577789403332955071:2108] VirtualId# [5:7577789403332955061:0]
2025-11-28T14:53:04.925402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789403332955071:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.925477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:04.925511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:04.926469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:04.926598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789405489474547:2109] [node 4] ICS01 socket: 358 reason# EndOfStream
2025-11-28T14:53:04.926628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789405489474515:2109] [node 4] ICP30 unregister session Session# [6:7577789405489474547:2109] VirtualId# [6:7577789405489474541:0]
2025-11-28T14:53:04.926647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789405489474547:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.926705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:04.926752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:04.997720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:04.997980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789405489474553:2105] [node 5] ICS07 socket disconnect 314 reason# EndOfStream
2025-11-28T14:53:04.998021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789405489474553:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:04.998118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:04.998170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789405489474553:2105] [node 5] ICS15 start handshake
2025-11-28T14:53:04.999046Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789405489474511:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31805) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:04.999166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789405489474553:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:04.999239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789405489474511:2105] [node 5] ICP30 unregister session Session# [6:7577789405489474553:2105] VirtualId# [6:7577789405489474551:0]
2025-11-28T14:53:04.999293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:53:04.999341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789405489474511:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31805) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:04.999361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31805) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:05.111901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789409043022021:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:05.111949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789409043022022:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:05.136913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789408100577681:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:05.136915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789408100577682:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:05.156401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789409669145276:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:05.156423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789409669145277:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:05.178152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789408751074008:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:05.178205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789408751074009:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:05.178249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789408751074010:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:05.198093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789410774600138:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:05.198131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789410774600140:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:05.198377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789410774600139:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:05.217507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410258517711:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:05.217537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410258517713:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:05.217939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410258517712:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:05.296828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789408100577744:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:05.312690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410258517772:2104] [node 4] ICP01 ready to work
2025-11-28T14:53:05.312722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789408751074075:2110] [node 6] ICP01 ready to work
2025-11-28T14:53:05.318289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410258517773:2106] [node 5] ICP01 ready to work
2025-11-28T14:53:05.315466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789409043022085:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:05.316983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789408100577745:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:05.317037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789409669145339:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:05.317059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789409043022086:2110] [node 6] ICP01 ready to work
2025-11-28T14:53:05.319108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789410774600202:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:05.318063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789408100577746:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:05.317789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789409043022087:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:05.319171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789410774600203:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:05.319135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789408751074076:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:05.318074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789409669145340:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:05.319839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789409669145341:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:05.323612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789409669145276:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22924968ns) {Configure:104(22914632ns) {ProcessConfigured:135(238292ns) {StartInitialHandshake:157(235192ns) {DropHandshakes:470(3328ns) {DropIncomingHandshake:442(809ns) DropOutgoingHandshake:457(517ns)} GenerateSessionVirtualId:988(845ns) PrepareNewSessionHandshake:477(4426ns) {SwitchToState:179(459ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.331056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789408100577757:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6703
2025-11-28T14:53:05.333490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789408100577757:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.335703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789409043022098:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30820
2025-11-28T14:53:05.336514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789408751074082:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55333
2025-11-28T14:53:05.336974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789409043022098:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.338716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789408751074082:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.340421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789408751074010:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.342087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789409669145350:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.342864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789409043022100:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30820
2025-11-28T14:53:05.343116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789410774600217:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17260
2025-11-28T14:53:05.342166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789408751074088:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55333
2025-11-28T14:53:05.343638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517791:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.343819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789408100577765:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6703
2025-11-28T14:53:05.342135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789409669145350:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.344545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789409043022103:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30820
2025-11-28T14:53:05.346422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789408751074088:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.343844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789408100577763:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6703
2025-11-28T14:53:05.344157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517792:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.347862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789408751074009:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.343060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789409669145356:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33872
2025-11-28T14:53:05.344526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789408100577762:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6703
2025-11-28T14:53:05.348336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789410774600217:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.344179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789409669145358:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33872
2025-11-28T14:53:05.348725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789409043022100:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.344526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517793:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.348811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789410774600202:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:53:05.344755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789408100577764:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6703
::ffff:127.1.0.0
2025-11-28T14:53:05.349719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789409043022103:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.344210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789409669145350:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:05.346507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410258517791:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.351228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789408751074083:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.347846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410258517792:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.346467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789408100577765:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.349913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789409043022097:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.344745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789409669145355:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33872
2025-11-28T14:53:05.347848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410258517712:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.344925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789409669145357:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33872
2025-11-28T14:53:05.349915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789409043022095:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.348575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410258517711:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.347598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789408100577681:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.349940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789409043022097:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.348961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789409669145358:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.347590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789408100577764:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.351278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789408751074083:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.349796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789409669145276:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-28T14:53:05.353299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410258517793:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-28T14:53:05.349942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789409043022095:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.356183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789408751074093:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55333
2025-11-28T14:53:05.355571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410258517772:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.348310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789408100577763:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.349977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789409669145341:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.352491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789409043022097:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:05.356518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789408751074083:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:05.358097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517797:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.348723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789408100577752:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.350026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789409669145356:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.356736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789408751074086:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.353137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789409043022093:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.348769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789408100577752:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.351262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:05.353172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789409043022093:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.361262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789410258517711:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789410258517792:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.349742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789408100577751:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.356765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789408751074086:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.356449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789409043022095:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.352449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789409669145355:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.364519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789410774600219:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17260
2025-11-28T14:53:05.360584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789408751074086:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.349776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789408100577751:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.352774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789409669145341:2107] [node 4] ICP22 created new session: [3:7577789409669145361:2107]
2025-11-28T14:53:05.356752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789409043022086:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.364338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789408751074076:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.368936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789410774600220:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.354048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789408100577762:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.354003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789409669145361:2107] [node 4] ICS09 handshake done sender: [3:7577789409669145350:2112] self: [3:7577789409669145348:0] peer: [4:7577789408751074092:0] socket: 302 qp: -1
2025-11-28T14:53:05.370104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517798:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.356831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:05.364463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789408751074009:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12058) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789408751074088:2120] held: no
2025-11-28T14:53:05.355573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789409669145361:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:05.358031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789409043022093:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.354107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789408100577752:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:05.370556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789410258517712:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12058) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789410258517791:2117] held: no
2025-11-28T14:53:05.374735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789410774600219:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.358514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789409043022086:2110] [node 6] ICP22 created new session: [1:7577789409043022110:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.356426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789408100577746:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.355595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789409669145357:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.364525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:05.374315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410258517792:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:05.356625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.365559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789408751074076:2108] [node 5] ICP22 created new session: [4:7577789408751074099:2108]
2025-11-28T14:53:05.359683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789409043022021:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.376020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789410774600140:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.355636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:05.357100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789408100577751:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:05.366725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789408751074075:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.356057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789409669145277:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.377204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517799:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.361285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789409043022110:2110] [node 6] ICS09 handshake done sender: [1:7577789409043022097:2115] self: [1:7577789409043022092:0] peer: [6:7577789410258517795:0] socket: 318 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.357140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789408100577746:2107] [node 4] ICP22 created new session: [2:7577789408100577773:2107]
2025-11-28T14:53:05.376664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789410774600217:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:05.367668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:05.377273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789410774600221:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17260
2025-11-28T14:53:05.361419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789409043022022:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.365541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789409669145276:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789409669145358:2118] held: no
2025-11-28T14:53:05.377335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410258517711:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.357201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789408100577773:2107] [node 4] ICS09 handshake done sender: [2:7577789408100577752:2111] self: [2:7577789408100577749:0] peer: [4:7577789408751074094:0] socket: 306 qp: -1
2025-11-28T14:53:05.362000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789409043022110:2110] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.368925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789408751074082:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:05.380716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:05.380782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410258517711:2048] [node 1] ICP22 created new session: [6:7577789410258517803:2048]
2025-11-28T14:53:05.373431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789409669145367:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33872
2025-11-28T14:53:05.378552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789410774600222:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17260
2025-11-28T14:53:05.382141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410258517793:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:05.357963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789408100577756:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.375065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789409669145277:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12058) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789409669145355:2115] held: no
2025-11-28T14:53:05.382500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410258517791:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:05.362061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:05.368959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789408751074099:2108] [node 5] ICS09 handshake done sender: [4:7577789408751074083:2116] self: [4:7577789408751074079:0] peer: [5:7577789410774600218:0] socket: 329 qp: -1
2025-11-28T14:53:05.378603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789410774600221:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.357998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789408100577756:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.375523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789409669145368:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33872
2025-11-28T14:53:05.362381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:05.382582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410258517803:2048] [node 1] ICS09 handshake done sender: [6:7577789410258517792:2118] self: [6:7577789410258517795:0] peer: [1:7577789409043022092:0] socket: 325 qp: -1
2025-11-28T14:53:05.369013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789408751074099:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:05.378625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789410774600202:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.377155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789409669145358:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:05.369060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:05.363841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789409043022021:2048] [node 2] ICP22 created new session: [1:7577789409043022112:2048]
2025-11-28T14:53:05.358149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789408100577773:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:05.378805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:05.383403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410258517803:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:05.358192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:05.378566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789409669145351:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.379437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789410774600138:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.383684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:05.380709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789410774600202:2108] [node 4] ICP22 created new session: [5:7577789410774600224:2108]
2025-11-28T14:53:05.361995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789408100577745:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.382751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789410774600224:2108] [node 4] ICS09 handshake done sender: [5:7577789410774600217:2116] self: [5:7577789410774600218:0] peer: [4:7577789408751074079:0] socket: 337 qp: -1
2025-11-28T14:53:05.384556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410258517772:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.369313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789408751074075:2110] [node 6] ICP22 created new session: [4:7577789408751074100:2110]
2025-11-28T14:53:05.378601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789409669145351:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.385278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:05.362385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:05.370166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789408751074100:2110] [node 6] ICS09 handshake done sender: [4:7577789408751074086:2118] self: [4:7577789408751074081:0] peer: [6:7577789410258517796:0] socket: 333 qp: -1
2025-11-28T14:53:05.364313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789409043022112:2048] [node 2] ICS09 handshake done sender: [1:7577789409043022095:2113] self: [1:7577789409043022089:0] peer: [2:7577789408100577766:0] socket: 330 qp: -1
2025-11-28T14:53:05.383088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789410774600224:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:05.386255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410258517712:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.379357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789409669145276:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.364351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789409043022112:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:05.362449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789408100577745:2109] [node 6] ICP22 created new session: [2:7577789408100577777:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.383140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:05.386291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410258517772:2104] [node 4] ICP22 created new session: [6:7577789410258517805:2104]
2025-11-28T14:53:05.385270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789410774600222:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.386772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:05.364504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:05.386378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789410774600226:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17260
2025-11-28T14:53:05.387842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410258517805:2104] [node 4] ICS09 handshake done sender: [6:7577789410258517793:2119] self: [6:7577789410258517796:0] peer: [4:7577789408751074081:0] socket: 334 qp: -1
2025-11-28T14:53:05.364576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789409043022022:2049] [node 3] ICP22 created new session: [1:7577789409043022115:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.370512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789408751074100:2110] [node 6] ICS10 traffic start
2025-11-28T14:53:05.364138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789408100577756:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:05.380709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:05.387426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789410774600139:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.370558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:05.387866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410258517712:2049] [node 2] ICP22 created new session: [6:7577789410258517807:2049]
2025-11-28T14:53:05.382143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789409669145276:2048] [node 1] ICP22 created new session: [3:7577789409669145370:2048]
2025-11-28T14:53:05.365019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:05.365017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789408100577777:2109] [node 6] ICS09 handshake done sender: [2:7577789408100577751:2110] self: [2:7577789408100577748:0] peer: [6:7577789410258517794:0] socket: 313 qp: -1
2025-11-28T14:53:05.388863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789410774600140:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20612) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789410774600219:2117] held: no
2025-11-28T14:53:05.388943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410258517807:2049] [node 2] ICS09 handshake done sender: [6:7577789410258517791:2117] self: [6:7577789410258517794:0] peer: [2:7577789408100577748:0] socket: 315 qp: -1
2025-11-28T14:53:05.383114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789409669145370:2048] [node 1] ICS09 handshake done sender: [3:7577789409669145358:2118] self: [3:7577789409669145360:0] peer: [1:7577789409043022090:0] socket: 338 qp: -1
2025-11-28T14:53:05.365590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789408100577777:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:05.392751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789410774600226:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.366814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789409043022115:2049] [node 3] ICS09 handshake done sender: [1:7577789409043022093:2111] self: [1:7577789409043022090:0] peer: [3:7577789409669145360:0] socket: 324 qp: -1
2025-11-28T14:53:05.388979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410258517807:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:05.375159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789408751074010:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.383156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789409669145370:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:05.367728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789409043022115:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:05.365636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:05.388992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410258517805:2104] [node 4] ICS10 traffic start
2025-11-28T14:53:05.394447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789410774600231:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17260
2025-11-28T14:53:05.389187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:05.366847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789408100577772:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6703
2025-11-28T14:53:05.383227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:05.369325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:05.389657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:05.394467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789410774600229:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17260
2025-11-28T14:53:05.390530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517802:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.369542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789408100577682:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.397517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789410774600138:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789410774600221:2119] held: no
2025-11-28T14:53:05.383405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789409669145351:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:05.393883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410258517802:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.369916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:05.386119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789409669145339:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.376637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789408751074096:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55333
2025-11-28T14:53:05.377445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789409043022118:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30820
2025-11-28T14:53:05.394808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410258517713:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.397953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789410774600219:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.386249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:05.370501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789408100577765:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:05.377953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:05.383086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789409043022096:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.394962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517806:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.386420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789409669145355:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:05.383129Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789409043022096:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.378780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789408751074010:2050] [node 3] ICP22 created new session: [4:7577789408751074103:2050]
2025-11-28T14:53:05.386426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789409669145339:2108] [node 5] ICP22 created new session: [3:7577789409669145373:2108]
2025-11-28T14:53:05.398264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410258517806:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.370916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789408100577682:2049] [node 3] ICP22 created new session: [2:7577789408100577780:2049]
2025-11-28T14:53:05.382096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789408751074103:2050] [node 3] ICS09 handshake done sender: [4:7577789408751074082:2115] self: [4:7577789408751074092:0] peer: [3:7577789409669145348:0] socket: 305 qp: -1
2025-11-28T14:53:05.399742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410258517773:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.387792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789409669145277:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.386819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789409043022096:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:05.400851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789410774600232:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17260
2025-11-28T14:53:05.373427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789408100577780:2049] [node 3] ICS09 handshake done sender: [2:7577789408100577756:2114] self: [2:7577789408100577754:0] peer: [3:7577789409669145365:0] socket: 316 qp: -1
2025-11-28T14:53:05.382476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789408751074103:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:05.391171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789409043022087:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.401137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789410774600139:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12058) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789410774600222:2120] held: no
2025-11-28T14:53:05.387829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:05.383115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:05.373467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789408100577780:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:05.387839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789409669145373:2108] [node 5] ICS09 handshake done sender: [3:7577789409669145351:2113] self: [3:7577789409669145349:0] peer: [5:7577789410774600223:0] socket: 346 qp: -1
2025-11-28T14:53:05.383168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789408751074098:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55333
2025-11-28T14:53:05.392240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:05.388613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789409669145277:2049] [node 2] ICP22 created new session: [3:7577789409669145374:2049]
2025-11-28T14:53:05.373503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:05.384783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789408751074088:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:05.373516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789408100577681:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.388928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789409669145373:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:05.392645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789409043022087:2108] [node 5] ICP22 created new session: [1:7577789409043022121:2108]
2025-11-28T14:53:05.386797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789408751074098:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.373574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:05.387443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789408751074009:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.403555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789410774600140:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.394476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789409043022121:2108] [node 5] ICS09 handshake done sender: [1:7577789409043022096:2114] self: [1:7577789409043022091:0] peer: [5:7577789410774600225:0] socket: 317 qp: -1
2025-11-28T14:53:05.388965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.394830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789409043022121:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:05.394869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:05.403567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789410258517713:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20612) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789410258517802:2123] held: no
2025-11-28T14:53:05.373830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789408100577681:2048] [node 1] ICP22 created new session: [2:7577789408100577782:2048]
2025-11-28T14:53:05.376728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789408100577782:2048] [node 1] ICS09 handshake done sender: [2:7577789408100577765:2119] self: [2:7577789408100577766:0] peer: [1:7577789409043022089:0] socket: 339 qp: -1
2025-11-28T14:53:05.388922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789408751074104:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55333
2025-11-28T14:53:05.390731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789409669145374:2049] [node 2] ICS09 handshake done sender: [3:7577789409669145355:2115] self: [3:7577789409669145365:0] peer: [2:7577789408100577754:0] socket: 319 qp: -1
2025-11-28T14:53:05.394897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789409043022120:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30820
2025-11-28T14:53:05.403675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:05.377074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789408100577782:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:05.377311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:05.404047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789410774600213:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.398881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789409043022086:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(35745882ns) {HandleHandshakeStatus:329(35742371ns) {DropIncomingHandshake:442(14113ns) DropOutgoingHandshake:457(400ns) SwitchToState:179(170ns) ProcessPendingSessionEvents:495(24394526ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24282441ns max# 1686206ns}}}
2025-11-28T14:53:05.404077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789410774600213:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.389101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789408100577755:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.399474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789409043022120:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.390549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:05.404082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789410774600140:2050] [node 3] ICP22 created new session: [5:7577789410774600234:2050]
2025-11-28T14:53:05.389130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789408100577755:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.391991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789408100577755:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:05.401299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789409043022094:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.396492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789408100577744:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.390739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789408751074009:2049] [node 2] ICP22 created new session: [4:7577789408751074108:2049]
2025-11-28T14:53:05.404132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789410774600234:2050] [node 3] ICS09 handshake done sender: [5:7577789410774600219:2117] self: [5:7577789410774600223:0] peer: [3:7577789409669145349:0] socket: 348 qp: -1
2025-11-28T14:53:05.397485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:05.401335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789409043022094:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.391480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789408751074108:2049] [node 2] ICS09 handshake done sender: [4:7577789408751074088:2120] self: [4:7577789408751074094:0] peer: [2:7577789408100577749:0] socket: 308 qp: -1
2025-11-28T14:53:05.404554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517812:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.390769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789409669145374:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:05.404174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789410774600234:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:05.397514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789408100577745:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(30492241ns) {HandleHandshakeStatus:329(30489899ns) {DropIncomingHandshake:442(15749ns) DropOutgoingHandshake:457(571ns) SwitchToState:179(160ns) ProcessPendingSessionEvents:495(22801761ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21609024ns max# 1887331ns}}}
2025-11-28T14:53:05.401838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789409043022021:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(32981597ns) {HandleHandshakeStatus:329(32979650ns) {DropIncomingHandshake:442(1089ns) DropOutgoingHandshake:457(1313ns) SwitchToState:179(319ns) ProcessPendingSessionEvents:495(23114518ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21523168ns max# 2716662ns}}}
2025-11-28T14:53:05.404389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:05.391506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789408751074108:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:05.391499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:05.402397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789409043022022:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(32688478ns) {HandleHandshakeStatus:329(32686153ns) {DropIncomingHandshake:442(14223ns) DropOutgoingHandshake:457(268ns) SwitchToState:179(706ns) ProcessPendingSessionEvents:495(19398886ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18283311ns max# 2703256ns}}}
2025-11-28T14:53:05.404403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789409043022094:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:05.393457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:05.406527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789409043022085:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.405726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789410774600221:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:05.397544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789408100577744:2108] [node 5] ICP22 created new session: [2:7577789408100577785:2108]
2025-11-28T14:53:05.395783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789409669145341:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(37230415ns) {HandleHandshakeStatus:329(37228153ns) {DropIncomingHandshake:442(1662ns) DropOutgoingHandshake:457(2862ns) SwitchToState:179(605ns) ProcessPendingSessionEvents:495(30539210ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29158183ns max# 1975913ns}}}
2025-11-28T14:53:05.397304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789408751074008:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:05.396392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789409669145353:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:05.408185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410258517802:2123] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.396417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789409669145353:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:05.397735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789408100577785:2108] [node 5] ICS09 handshake done sender: [2:7577789408100577755:2113] self: [2:7577789408100577750:0] peer: [5:7577789410774600230:0] socket: 300 qp: -1
2025-11-28T14:53:05.405943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789410774600222:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:05.407552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:05.398195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789409669145353:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:05.403969Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789408751074076:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(30242618ns) {HandleHandshakeStatus:329(30240415ns) {DropIncomingHandshake:442(738ns) DropOutgoingHandshake:457(641ns) SwitchToState:179(407ns) ProcessPendingSessionEvents:495(24312014ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23845459ns max# 1747951ns}}}
2025-11-28T14:53:05.397753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789408100577746:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(34168835ns) {HandleHandshakeStatus:329(34167233ns) {DropIncomingHandshake:442(15350ns) DropOutgoingHandshake:457(623ns) SwitchToState:179(131ns) ProcessPendingSessionEvents:495(28963046ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28821117ns max# 2007696ns}}}
2025-11-28T14:53:05.401314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789409669145340:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.405627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789408751074075:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(29318403ns) {HandleHandshakeStatus:329(29315612ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(487ns) SwitchToState:179(997ns) ProcessPendingSessionEvents:495(19653327ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19539796ns max# 1968709ns}}}
2025-11-28T14:53:05.401397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:05.412533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410258517713:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.412168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789408751074010:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25713187ns) {HandleHandshakeStatus:329(25710284ns) {DropIncomingHandshake:442(765ns) DropOutgoingHandshake:457(3850ns) SwitchToState:179(426ns) ProcessPendingSessionEvents:495(14584717ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13886422ns max# 1232800ns}}}
2025-11-28T14:53:05.401594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789409669145340:2109] [node 6] ICP22 created new session: [3:7577789409669145378:2109]
2025-11-28T14:53:05.397773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789408100577785:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:05.406360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789410774600138:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.412608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:05.408152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789409043022085:2107] [node 4] ICP22 created new session: [1:7577789409043022125:2107]
2025-11-28T14:53:05.407305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789410774600213:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:05.402303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789409669145378:2109] [node 6] ICS09 handshake done sender: [3:7577789409669145353:2114] self: [3:7577789409669145352:0] peer: [6:7577789410258517810:0] socket: 312 qp: -1
2025-11-28T14:53:05.412699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789408751074111:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55333
2025-11-28T14:53:05.397951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:05.413529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410258517713:2050] [node 3] ICP22 created new session: [6:7577789410258517814:2050]
2025-11-28T14:53:05.408214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789409043022125:2107] [node 4] ICS09 handshake done sender: [1:7577789409043022094:2112] self: [1:7577789409043022088:0] peer: [4:7577789408751074110:0] socket: 358 qp: -1
2025-11-28T14:53:05.403932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789408100577681:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22314046ns) {HandleHandshakeStatus:329(22311386ns) {DropIncomingHandshake:442(1007ns) DropOutgoingHandshake:457(4554ns) SwitchToState:179(1456ns) ProcessPendingSessionEvents:495(17177366ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17085271ns max# 3897639ns}}}
2025-11-28T14:53:05.407901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789408100577682:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(27988701ns) {HandleHandshakeStatus:329(27986650ns) {DropIncomingHandshake:442(13758ns) DropOutgoingHandshake:457(327ns) SwitchToState:179(500ns) ProcessPendingSessionEvents:495(20485643ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19532583ns max# 2718324ns}}}
2025-11-28T14:53:05.403544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789409669145378:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:05.413599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410258517814:2050] [node 3] ICS09 handshake done sender: [6:7577789410258517802:2123] self: [6:7577789410258517810:0] peer: [3:7577789409669145352:0] socket: 364 qp: -1
2025-11-28T14:53:05.411983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789409043022125:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:05.414149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410258517814:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:05.403628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:05.407482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789410774600139:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.412347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:05.414339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:05.407781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:05.407806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:05.407825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789410774600139:2049] [node 2] ICP22 created new session: [5:7577789410774600237:2049]
2025-11-28T14:53:05.412388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789409043022021:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:05.408185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789410774600202:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(20424423ns) {HandleHandshakeStatus:329(20421797ns) {DropIncomingHandshake:442(1194ns) DropOutgoingHandshake:457(7385ns) SwitchToState:179(471ns) ProcessPendingSessionEvents:495(12288865ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11599041ns max# 1037880ns}}}
2025-11-28T14:53:05.408289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789410774600138:2048] [node 1] ICP22 created new session: [5:7577789410774600238:2048]
2025-11-28T14:53:05.418825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410258517813:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20313
2025-11-28T14:53:05.411961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789410774600203:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:05.412044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:05.412074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789410774600237:2049] [node 2] ICS09 handshake done sender: [5:7577789410774600222:2120] self: [5:7577789410774600230:0] peer: [2:7577789408100577750:0] socket: 362 qp: -1
2025-11-28T14:53:05.412408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789410774600203:2109] [node 6] ICP22 created new session: [5:7577789410774600239:2109]
2025-11-28T14:53:05.412427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789410774600238:2048] [node 1] ICS09 handshake done sender: [5:7577789410774600221:2119] self: [5:7577789410774600225:0] peer: [1:7577789409043022091:0] socket: 326 qp: -1
2025-11-28T14:53:05.412464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789410774600238:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:05.413129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789410774600237:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:05.413184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:05.413263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:05.414019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789410774600239:2109] [node 6] ICS09 handshake done sender: [5:7577789410774600213:2115] self: [5:7577789410774600208:0] peer: [6:7577789410258517811:0] socket: 365 qp: -1
2025-11-28T14:53:05.419466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789408751074008:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789408751074098:2123] held: no
2025-11-28T14:53:05.419433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410258517806:2124] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:05.414563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789410774600239:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:05.414608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:05.420643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410258517773:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.421374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:05.422155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410258517773:2106] [node 5] ICP22 created new session: [6:7577789410258517816:2106]
2025-11-28T14:53:05.422646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410258517816:2106] [node 5] ICS09 handshake done sender: [6:7577789410258517806:2124] self: [6:7577789410258517811:0] peer: [5:7577789410774600208:0] socket: 368 qp: -1
2025-11-28T14:53:05.422953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410258517816:2106] [node 5] ICS10 traffic start
2025-11-28T14:53:05.423015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:05.422511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789408751074098:2123] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:05.424240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789410258517773:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:05.423839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789408751074008:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:05.423907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:05.423974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789408751074008:2048] [node 1] ICP22 created new session: [4:7577789408751074112:2048]
2025-11-28T14:53:05.424023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789408751074112:2048] [node 1] ICS09 handshake done sender: [4:7577789408751074098:2123] self: [4:7577789408751074110:0] peer: [1:7577789409043022088:0] socket: 336 qp: -1
2025-11-28T14:53:05.424066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789408751074112:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:05.424098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789408751074075:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:05.424111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:05.424455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789409043022085:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16531517ns) {HandleHandshakeStatus:329(16529321ns) {DropIncomingHandshake:442(12738ns) DropOutgoingHandshake:457(381ns) SwitchToState:179(544ns) ProcessPendingSessionEvents:495(9442576ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8645611ns max# 3794555ns}}}
2025-11-28T14:53:05.482178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:05.482427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789408100577782:2048] [node 1] ICS07 socket disconnect 339 reason# EndOfStream
2025-11-28T14:53:05.482466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789408100577782:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.482593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:05.482645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789408100577782:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:05.483895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:05.484120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789409669145370:2048] [node 1] ICS01 socket: 338 reason# EndOfStream
2025-11-28T14:53:05.484174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789409669145276:2048] [node 1] ICP30 unregister session Session# [3:7577789409669145370:2048] VirtualId# [3:7577789409669145360:0]
2025-11-28T14:53:05.484210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789409669145370:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.484303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:05.484359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:05.485607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789408100577681:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25720) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:05.485673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789408100577782:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:05.486903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789408100577681:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25720) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:05.487051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789408100577782:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:05.487102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789408100577681:2048] [node 1] ICP30 unregister session Session# [2:7577789408100577782:2048] VirtualId# [2:7577789408100577766:0]
2025-11-28T14:53:05.487165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:05.487208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789408100577681:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25720) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:05.487229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25720) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:05.487348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:05.487562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789408751074112:2048] [node 1] ICS01 socket: 336 reason# EndOfStream
2025-11-28T14:53:05.487617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789408751074008:2048] [node 1] ICP30 unregister session Session# [4:7577789408751074112:2048] VirtualId# [4:7577789408751074110:0]
2025-11-28T14:53:05.487651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789408751074112:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.487753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:05.487816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:05.488379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:05.488663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789410774600238:2048] [node 1] ICS01 socket: 326 reason# EndOfStream
2025-11-28T14:53:05.488717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789410774600138:2048] [node 1] ICP30 unregister session Session# [5:7577789410774600238:2048] VirtualId# [5:7577789410774600225:0]
2025-11-28T14:53:05.488750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789410774600238:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.488828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:05.488879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:05.489130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:05.489339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410258517803:2048] [node 1] ICS01 socket: 325 reason# EndOfStream
2025-11-28T14:53:05.489397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410258517711:2048] [node 1] ICP30 unregister session Session# [6:7577789410258517803:2048] VirtualId# [6:7577789410258517795:0]
2025-11-28T14:53:05.489456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410258517803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.489532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:05.489585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:05.533701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:05.533875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789409669145374:2049] [node 2] ICS01 socket: 319 reason# EndOfStream
2025-11-28T14:53:05.533913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789409669145277:2049] [node 2] ICP30 unregister session Session# [3:7577789409669145374:2049] VirtualId# [3:7577789409669145365:0]
2025-11-28T14:53:05.533940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789409669145374:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.534015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:05.534061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:05.536799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:05.537034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789408751074108:2049] [node 2] ICS01 socket: 308 reason# EndOfStream
2025-11-28T14:53:05.537084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789408751074009:2049] [node 2] ICP30 unregister session Session# [4:7577789408751074108:2049] VirtualId# [4:7577789408751074094:0]
2025-11-28T14:53:05.537123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789408751074108:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.537243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:05.537305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:05.538056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:05.538211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789410774600237:2049] [node 2] ICS01 socket: 362 reason# EndOfStream
2025-11-28T14:53:05.538245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789410774600139:2049] [node 2] ICP30 unregister session Session# [5:7577789410774600237:2049] VirtualId# [5:7577789410774600230:0]
2025-11-28T14:53:05.538267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789410774600237:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.538331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:05.538387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:05.539143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:05.539367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410258517807:2049] [node 2] ICS01 socket: 315 reason# EndOfStream
2025-11-28T14:53:05.539405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410258517712:2049] [node 2] ICP30 unregister session Session# [6:7577789410258517807:2049] VirtualId# [6:7577789410258517794:0]
2025-11-28T14:53:05.539438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410258517807:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.539506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:05.539553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:05.621187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:05.621440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789408751074103:2050] [node 3] ICS07 socket disconnect 305 reason# EndOfStream
2025-11-28T14:53:05.621493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789408751074103:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.621650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:05.621732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789408751074103:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:05.622248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:05.622450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789410774600234:2050] [node 3] ICS01 socket: 348 reason# EndOfStream
2025-11-28T14:53:05.622500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789410774600140:2050] [node 3] ICP30 unregister session Session# [5:7577789410774600234:2050] VirtualId# [5:7577789410774600223:0]
2025-11-28T14:53:05.622536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789410774600234:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.622596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789408751074010:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:05.622635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:05.622662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789408751074103:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:05.622693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:05.622876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789408751074010:2050] [node 3] ICP30 unregister session Session# [4:7577789408751074103:2050] VirtualId# [4:7577789408751074092:0]
2025-11-28T14:53:05.623001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:05.623040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789408751074010:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:05.623060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:05.623405Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:05.623583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410258517814:2050] [node 3] ICS01 socket: 364 reason# EndOfStream
2025-11-28T14:53:05.623625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410258517713:2050] [node 3] ICP30 unregister session Session# [6:7577789410258517814:2050] VirtualId# [6:7577789410258517810:0]
2025-11-28T14:53:05.623657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410258517814:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.623731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:05.623778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:05.632940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789408751074010:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:05.720748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:05.720971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789410774600224:2108] [node 4] ICS07 socket disconnect 337 reason# EndOfStream
2025-11-28T14:53:05.721007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789410774600224:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.721101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:05.721160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789410774600224:2108] [node 4] ICS15 start handshake
2025-11-28T14:53:05.721861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:05.721902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789410774600202:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9688) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:05.721975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789410774600224:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:05.722087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410258517805:2104] [node 4] ICS01 socket: 334 reason# EndOfStream
2025-11-28T14:53:05.722023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789410774600202:2108] [node 4] ICP30 unregister session Session# [5:7577789410774600224:2108] VirtualId# [5:7577789410774600218:0]
2025-11-28T14:53:05.722151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:05.722134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410258517772:2104] [node 4] ICP30 unregister session Session# [6:7577789410258517805:2104] VirtualId# [6:7577789410258517796:0]
2025-11-28T14:53:05.722179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410258517805:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.722217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789410774600202:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9688) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:05.722238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9688) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:05.722258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:05.722308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:05.732387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789410774600202:2108] [node 4] ICP33 wake up from error state
2025-11-28T14:53:05.808064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:05.808295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789410258517816:2106] [node 5] ICS07 socket disconnect 368 reason# EndOfStream
2025-11-28T14:53:05.808326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410258517816:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:05.808453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:05.808495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789410258517816:2106] [node 5] ICS15 start handshake
2025-11-28T14:53:05.809541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789410258517773:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27715) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:05.809681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410258517816:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:05.809774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410258517773:2106] [node 5] ICP30 unregister session Session# [6:7577789410258517816:2106] VirtualId# [6:7577789410258517811:0]
2025-11-28T14:53:05.810191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:53:05.810237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789410258517773:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27715) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:05.810252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27715) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:05.919509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789408296307465:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:05.919545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789408296307466:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:05.941054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789407437951901:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:05.941112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789407437951902:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:05.963458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789407438130052:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:05.963502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789407438130053:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:05.986488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789407082536638:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:05.986544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789407082536639:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:05.986591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789407082536640:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:06.024672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789413353925576:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:06.024734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789413353925577:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:06.024783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789413353925578:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:06.128997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410961400773:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:06.262225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789412591274822:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:06.129130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410961400774:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:06.129987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410961400775:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:06.304682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789411732919263:2107] [node 5] ICP01 ready to work
2025-11-28T14:53:06.321907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789411377503978:2084] [node 5] ICP01 ready to work
2025-11-28T14:53:06.305148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789411732919264:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:06.366976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789412591274822:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(49577367ns) {Configure:104(49572781ns) {ProcessConfigured:135(6734187ns) {StartInitialHandshake:157(6729669ns) {DropHandshakes:470(5540ns) {DropIncomingHandshake:442(788ns) DropOutgoingHandshake:457(469ns)} GenerateSessionVirtualId:988(974ns) PrepareNewSessionHandshake:477(6271ns) {SwitchToState:179(3443ns)}}}}}
2025-11-28T14:53:06.351008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789411732919265:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:06.444861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789407438130053:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(180718083ns) {Configure:104(180711596ns) {ProcessConfigured:135(11030315ns) {StartInitialHandshake:157(11026562ns) {DropHandshakes:470(3270ns) {DropIncomingHandshake:442(841ns) DropOutgoingHandshake:457(812ns)} GenerateSessionVirtualId:988(720ns) PrepareNewSessionHandshake:477(6606ns) {SwitchToState:179(3115ns)}}}}}
2025-11-28T14:53:06.445637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789411733097413:2089] [node 4] ICP01 ready to work
2025-11-28T14:53:06.446238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789411733097414:2090] [node 6] ICP01 ready to work
2025-11-28T14:53:06.446789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789411733097415:2092] [node 5] ICP01 ready to work
2025-11-28T14:53:06.433584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789411377504003:2088] [node 6] ICP01 ready to work
2025-11-28T14:53:06.455685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789411732919264:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(39506172ns) {Configure:104(39500221ns) {ProcessConfigured:135(73174ns) {StartInitialHandshake:157(70731ns) {DropHandshakes:470(3713ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:477(2052ns) {SwitchToState:179(738ns)}}}}}
2025-11-28T14:53:06.470205Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789410961400773:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(155823277ns) {Configure:104(155811958ns) {ProcessConfigured:135(14848876ns) {StartInitialHandshake:157(14844720ns) {DropHandshakes:470(3114ns) {DropIncomingHandshake:442(951ns) DropOutgoingHandshake:457(508ns)} GenerateSessionVirtualId:988(667ns) PrepareNewSessionHandshake:477(2679ns) {SwitchToState:179(1297ns)}}}}}
2025-11-28T14:53:06.470611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410961400838:2061] [node 5] ICP01 ready to work
2025-11-28T14:53:06.471186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789410961400839:2063] [node 4] ICP01 ready to work
2025-11-28T14:53:06.468767Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789413353925576:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(163432252ns) {Configure:104(163426734ns) {ProcessConfigured:135(13691155ns) {StartInitialHandshake:157(13688640ns) {DropHandshakes:470(931ns) {DropIncomingHandshake:442(232ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(178ns) PrepareNewSessionHandshake:477(3924ns) {SwitchToState:179(934ns)}}}}}
2025-11-28T14:53:06.498445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789408296307465:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(103076833ns) {Configure:104(103067173ns) {ProcessConfigured:135(15515093ns) {StartInitialHandshake:157(15512477ns) {DropHandshakes:470(2633ns) {DropIncomingHandshake:442(717ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(976ns) PrepareNewSessionHandshake:477(2944ns) {SwitchToState:179(1511ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.469446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789413353925644:2075] [node 6] ICP01 ready to work
2025-11-28T14:53:06.500521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789413353925645:2079] [node 4] ICP01 ready to work
2025-11-28T14:53:06.476695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789411732919265:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18190037ns) {Configure:104(18185600ns) {ProcessConfigured:135(6536676ns) {StartInitialHandshake:157(6533757ns) {DropHandshakes:470(2691ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(411ns)} GenerateSessionVirtualId:988(823ns) PrepareNewSessionHandshake:477(3887ns) {SwitchToState:179(1785ns)}}}}}
2025-11-28T14:53:06.540716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789411733097414:2090] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(76594189ns) {Configure:104(76590105ns) {ProcessConfigured:135(18415671ns) {StartInitialHandshake:157(18412569ns) {DropHandshakes:470(2710ns) {DropIncomingHandshake:442(526ns) DropOutgoingHandshake:457(437ns)} GenerateSessionVirtualId:988(934ns) PrepareNewSessionHandshake:477(5265ns) {SwitchToState:179(3082ns)}}}}}
2025-11-28T14:53:06.532997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789412591274829:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:06.516803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789411732919266:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.552061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789411733097416:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46411
2025-11-28T14:53:06.533687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789412591274830:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.557666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789411732919277:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.559848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789413353925652:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24096
2025-11-28T14:53:06.560571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789413353925653:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24096
2025-11-28T14:53:06.562598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789413353925652:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.562403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789411732919266:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.563581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789413353925653:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.565500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789413353925577:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.565815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789411732919268:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.569065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789413353925658:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24096
2025-11-28T14:53:06.565850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789411732919268:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.566895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789411732919268:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:06.567214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789411732919263:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.567279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:06.567337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789411732919263:2107] [node 5] ICP22 created new session: [2:7577789411732919280:2107]
2025-11-28T14:53:06.567387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789411732919280:2107] [node 5] ICS09 handshake done sender: [2:7577789411732919268:2112] self: [2:7577789411732919267:0] peer: [5:7577789413353925657:0] socket: 316 qp: -1
2025-11-28T14:53:06.567422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789411732919280:2107] [node 5] ICS10 traffic start
2025-11-28T14:53:06.567455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:06.569712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789413353925653:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:06.569992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789413353925577:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.570069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:06.570119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789413353925577:2049] [node 2] ICP22 created new session: [5:7577789413353925659:2049]
2025-11-28T14:53:06.570170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789413353925659:2049] [node 2] ICS09 handshake done sender: [5:7577789413353925653:2116] self: [5:7577789413353925657:0] peer: [2:7577789411732919267:0] socket: 321 qp: -1
2025-11-28T14:53:06.570206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789413353925659:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:06.570243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.570865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789412591274831:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43391
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.571053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789412591274832:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43391
2025-11-28T14:53:06.571027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789411732919277:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.571534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789412591274839:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43391
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.571656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789412591274840:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43391
2025-11-28T14:53:06.573148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789412591274832:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.573348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789412591274839:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.573480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789411733097428:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46411
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.574331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789411732919283:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29245
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.574574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789411733097429:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46411
2025-11-28T14:53:06.574962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789411733097430:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46411
2025-11-28T14:53:06.576025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789411733097429:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.575913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789412591274840:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.576139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789412591274831:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.576547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789411733097430:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.576661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789407438130053:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.576966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789411732919272:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.577003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789411732919272:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.578071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789411377504010:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59385
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.578232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789411377504011:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59385
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.578472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789411733097433:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46411
2025-11-28T14:53:06.578540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789411732919272:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:06.578411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789411377504012:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59385
2025-11-28T14:53:06.578541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789411377504013:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59385
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.579663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789411733097416:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.579467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789411377504016:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59385
2025-11-28T14:53:06.579734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789411733097430:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:06.579653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789411377504010:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.579739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789407437951902:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.579810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:06.579937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789407437951902:2049] [node 3] ICP22 created new session: [2:7577789411732919286:2049]
2025-11-28T14:53:06.580059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789411377504013:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.580121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789411732919286:2049] [node 3] ICS09 handshake done sender: [2:7577789411732919272:2114] self: [2:7577789411732919271:0] peer: [3:7577789411733097432:0] socket: 344 qp: -1
2025-11-28T14:53:06.580283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789411377504012:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.580414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789411377504011:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:53:06.580060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789407438130053:2049] [node 2] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:53:06.580126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:06.580946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789413353925662:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24096
2025-11-28T14:53:06.580416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789411732919286:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:06.580468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:06.580301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789407438130053:2049] [node 2] ICP22 created new session: [3:7577789411733097434:2049]
2025-11-28T14:53:06.580896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789411733097434:2049] [node 2] ICS09 handshake done sender: [3:7577789411733097430:2118] self: [3:7577789411733097432:0] peer: [2:7577789411732919271:0] socket: 347 qp: -1
2025-11-28T14:53:06.580934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789411733097434:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:06.580971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:06.581418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789407082536640:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.581625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789407082536639:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.582653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789411732919270:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.582679Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789411732919270:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.584288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789411732919283:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.585193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789411732919288:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.585443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789407437951901:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.585650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400851:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
2025-11-28T14:53:06.585653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789411733097420:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.585685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789411733097420:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.585598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789411377504016:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.586314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789413353925662:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.586449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789413353925645:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.586443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789412591274828:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.586482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789412591274828:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.586695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789411732919270:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.586621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789407082536638:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.586840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789411377504023:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59385
2025-11-28T14:53:06.587260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789411733097428:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.587226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789411732919288:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.587275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789411732919264:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.587315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:06.587356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789411732919264:2108] [node 4] ICP22 created new session: [2:7577789411732919292:2108]
2025-11-28T14:53:06.587404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789411732919292:2108] [node 4] ICS09 handshake done sender: [2:7577789411732919270:2113] self: [2:7577789411732919269:0] peer: [4:7577789411377504022:0] socket: 317 qp: -1
2025-11-28T14:53:06.587496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789411732919292:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:06.587632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:06.587744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789412591274828:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:06.587690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789407438130052:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.588034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789411733097420:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:06.588870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789412591274834:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.588911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789412591274834:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.589572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789408296307465:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.589678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:06.589663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789411733097413:2089] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.589741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789408296307465:2048] [node 2] ICP22 created new session: [1:7577789412591274849:2048]
2025-11-28T14:53:06.589743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:06.589723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400852:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
2025-11-28T14:53:06.589859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789412591274849:2048] [node 2] ICS09 handshake done sender: [1:7577789412591274828:2111] self: [1:7577789412591274827:0] peer: [2:7577789411732919291:0] socket: 338 qp: -1
2025-11-28T14:53:06.589909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789411733097413:2089] [node 4] ICP22 created new session: [3:7577789411733097440:2089]
2025-11-28T14:53:06.589968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789412591274849:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:06.590004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:06.590040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789411733097440:2089] [node 4] ICS09 handshake done sender: [3:7577789411733097420:2113] self: [3:7577789411733097419:0] peer: [4:7577789411377504021:0] socket: 325 qp: -1
2025-11-28T14:53:06.590081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789411733097440:2089] [node 4] ICS10 traffic start
2025-11-28T14:53:06.590192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.590741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789413353925664:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24096
2025-11-28T14:53:06.591405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789412591274826:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.591440Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789412591274826:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.587845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789411377504005:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.587881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789411377504005:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.589636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789411377504011:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:06.590433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789407082536639:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.590531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:06.590579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789407082536639:2049] [node 2] ICP22 created new session: [4:7577789411377504027:2049]
2025-11-28T14:53:06.590621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789411377504027:2049] [node 2] ICS09 handshake done sender: [4:7577789411377504011:2117] self: [4:7577789411377504022:0] peer: [2:7577789411732919269:0] socket: 331 qp: -1
2025-11-28T14:53:06.590658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789411377504027:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:06.590710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.601637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410961400852:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.601755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789411732919264:2108] [node 4] ICP18 handshake (actor [2:7577789411732919288:2118]) is too old
2025-11-28T14:53:06.601751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789413353925665:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24096
2025-11-28T14:53:06.601815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410961400775:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.601884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789412591274834:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:06.602698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410961400851:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.603662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789412591274826:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.604638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789411733097422:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.605164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400854:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.604657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789408296307466:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.604675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789411733097422:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.605266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:06.605270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410961400774:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.605390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789408296307466:2049] [node 3] ICP22 created new session: [1:7577789412591274854:2049]
2025-11-28T14:53:06.605343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789411733097443:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46411
2025-11-28T14:53:06.605483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789412591274854:2049] [node 3] ICS09 handshake done sender: [1:7577789412591274834:2114] self: [1:7577789412591274833:0] peer: [3:7577789411733097439:0] socket: 339 qp: -1
2025-11-28T14:53:06.605520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789412591274854:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:06.605563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.606058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789411377504025:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59385
2025-11-28T14:53:06.606747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789413353925665:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.606311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789411733097428:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:06.606520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789407438130052:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.607050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789413353925578:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.606561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:06.606616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789407438130052:2048] [node 1] ICP22 created new session: [3:7577789411733097445:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.606661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789411733097445:2048] [node 1] ICS09 handshake done sender: [3:7577789411733097428:2116] self: [3:7577789411733097439:0] peer: [1:7577789412591274833:0] socket: 340 qp: -1
2025-11-28T14:53:06.606685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789411733097445:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:06.606829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:06.607513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789411732919294:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29245
2025-11-28T14:53:06.607617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400856:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
2025-11-28T14:53:06.607808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789411377504005:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:06.608274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789407437951901:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789411732919283:2117] held: no
2025-11-28T14:53:06.608649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789413353925662:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:06.608800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789412591274822:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.608872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:06.608894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789407082536640:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789411377504012:2118] held: no
2025-11-28T14:53:06.608922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789412591274822:2107] [node 4] ICP22 created new session: [1:7577789412591274856:2107]
2025-11-28T14:53:06.608974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789412591274856:2107] [node 4] ICS09 handshake done sender: [1:7577789412591274826:2110] self: [1:7577789412591274825:0] peer: [4:7577789411377504024:0] socket: 309 qp: -1
2025-11-28T14:53:06.609005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789412591274856:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:06.609043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:06.609330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789411732919274:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.609364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789411732919274:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.610384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789411377503978:2084] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.610478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:06.610521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789411377503978:2084] [node 5] ICP22 created new session: [4:7577789411377504030:2084]
2025-11-28T14:53:06.610558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789411377504030:2084] [node 5] ICS09 handshake done sender: [4:7577789411377504005:2113] self: [4:7577789411377504004:0] peer: [5:7577789413353925663:0] socket: 350 qp: -1
2025-11-28T14:53:06.610499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789413353925645:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.610586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:06.610586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789411377504030:2084] [node 5] ICS10 traffic start
2025-11-28T14:53:06.610615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:06.610636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789413353925645:2079] [node 4] ICP22 created new session: [5:7577789413353925667:2079]
2025-11-28T14:53:06.610705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789413353925667:2079] [node 4] ICS09 handshake done sender: [5:7577789413353925662:2118] self: [5:7577789413353925663:0] peer: [4:7577789411377504004:0] socket: 352 qp: -1
2025-11-28T14:53:06.610744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789413353925667:2079] [node 4] ICS10 traffic start
2025-11-28T14:53:06.610774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:06.611076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789411732919283:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:06.611243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789411733097422:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:06.612238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789407437951901:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.612293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:06.612494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789411377504012:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:06.612341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789407437951901:2048] [node 1] ICP22 created new session: [2:7577789411732919296:2048]
2025-11-28T14:53:06.612896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410961400852:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:06.612744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789411732919296:2048] [node 1] ICS09 handshake done sender: [2:7577789411732919283:2117] self: [2:7577789411732919291:0] peer: [1:7577789412591274827:0] socket: 343 qp: -1
2025-11-28T14:53:06.614533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410961400775:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.612781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789411732919296:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:06.612904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:06.614620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:06.614698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410961400775:2050] [node 3] ICP22 created new session: [6:7577789410961400857:2050]
2025-11-28T14:53:06.614754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410961400857:2050] [node 3] ICS09 handshake done sender: [6:7577789410961400852:2116] self: [6:7577789410961400853:0] peer: [3:7577789411733097421:0] socket: 359 qp: -1
2025-11-28T14:53:06.614786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410961400857:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:06.614825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:06.614888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789411733097424:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.615396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789407082536640:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.615458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:06.615499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789407082536640:2050] [node 3] ICP22 created new session: [4:7577789411377504034:2050]
2025-11-28T14:53:06.615544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789411377504034:2050] [node 3] ICS09 handshake done sender: [4:7577789411377504012:2118] self: [4:7577789411377504021:0] peer: [3:7577789411733097419:0] socket: 334 qp: -1
2025-11-28T14:53:06.615569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789411377504034:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:06.615594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:06.614924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789411733097424:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.617547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789411732919274:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:06.617846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789411733097414:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.618464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789411732919265:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.618597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:06.618670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789411732919265:2109] [node 6] ICP22 created new session: [2:7577789411732919298:2109]
2025-11-28T14:53:06.619080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789411732919298:2109] [node 6] ICS09 handshake done sender: [2:7577789411732919274:2115] self: [2:7577789411732919273:0] peer: [6:7577789410961400855:0] socket: 354 qp: -1
2025-11-28T14:53:06.617896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:06.618353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789411733097414:2090] [node 6] ICP22 created new session: [3:7577789411733097447:2090]
2025-11-28T14:53:06.618891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789411377504029:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59385
2025-11-28T14:53:06.619119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789411732919298:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:06.618731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789411733097447:2090] [node 6] ICS09 handshake done sender: [3:7577789411733097422:2114] self: [3:7577789411733097421:0] peer: [6:7577789410961400853:0] socket: 358 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.623476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400859:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
2025-11-28T14:53:06.619154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.625254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789413353925669:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24096
2025-11-28T14:53:06.626777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410961400851:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:06.618769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789411733097447:2090] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.629919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789411732919300:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29245
2025-11-28T14:53:06.628962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789411377504016:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:06.629053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410961400774:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.630157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:06.630215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410961400774:2049] [node 2] ICP22 created new session: [6:7577789410961400862:2049]
2025-11-28T14:53:06.631528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789407082536638:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.630284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410961400862:2049] [node 2] ICS09 handshake done sender: [6:7577789410961400851:2115] self: [6:7577789410961400855:0] peer: [2:7577789411732919273:0] socket: 355 qp: -1
2025-11-28T14:53:06.630327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410961400862:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:06.630367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.619225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:06.636054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789412591274859:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43391
2025-11-28T14:53:06.632389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:06.632458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789407082536638:2048] [node 1] ICP22 created new session: [4:7577789411377504036:2048]
2025-11-28T14:53:06.632510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789411377504036:2048] [node 1] ICS09 handshake done sender: [4:7577789411377504016:2121] self: [4:7577789411377504024:0] peer: [1:7577789412591274825:0] socket: 341 qp: -1
2025-11-28T14:53:06.636208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789411377504036:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:06.636254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:06.639166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789411733097424:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:06.639450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789411733097415:2092] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.639754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789413353925669:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.641013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400861:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
2025-11-28T14:53:06.639508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:06.640993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789413353925671:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24096
2025-11-28T14:53:06.641532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789413353925576:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.655466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789413353925665:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:06.655938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789413353925578:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.656765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410961400854:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.639561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789411733097415:2092] [node 5] ICP22 created new session: [3:7577789411733097450:2092]
2025-11-28T14:53:06.656031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:06.639612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789411733097450:2092] [node 5] ICS09 handshake done sender: [3:7577789411733097424:2115] self: [3:7577789411733097423:0] peer: [5:7577789413353925666:0] socket: 310 qp: -1
2025-11-28T14:53:06.639647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789411733097450:2092] [node 5] ICS10 traffic start
2025-11-28T14:53:06.656089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789413353925578:2050] [node 3] ICP22 created new session: [5:7577789413353925673:2050]
2025-11-28T14:53:06.639682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:06.656140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789413353925673:2050] [node 3] ICS09 handshake done sender: [5:7577789413353925665:2120] self: [5:7577789413353925666:0] peer: [3:7577789411733097423:0] socket: 365 qp: -1
2025-11-28T14:53:06.656180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789413353925673:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:06.656219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:06.662422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789412591274838:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.662724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410961400861:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.662468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789412591274838:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-28T14:53:06.663561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789411733097415:2092] [node 5] ICP35 event processing took too much time PendingConnection:279(22686740ns) {HandleHandshakeStatus:329(22679987ns) {DropIncomingHandshake:442(16270ns) DropOutgoingHandshake:457(634ns) SwitchToState:179(669ns) ProcessPendingSessionEvents:495(22331871ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13446676ns max# 5864271ns}}}
::ffff:127.1.0.0
2025-11-28T14:53:06.663715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400864:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
2025-11-28T14:53:06.663803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410961400773:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.663888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410961400838:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.666293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789412591274836:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.666339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789412591274836:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.667199Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789412591274822:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:06.667772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789412591274838:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:06.667854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789413353925651:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.667856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789410961400864:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.667892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789413353925651:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.670168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400867:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
2025-11-28T14:53:06.670167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789412591274836:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:06.670472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789412591274829:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.670654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:06.670729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789412591274829:2108] [node 5] ICP22 created new session: [1:7577789412591274862:2108]
2025-11-28T14:53:06.670762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789410961400839:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:06.670790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789412591274862:2108] [node 5] ICS09 handshake done sender: [1:7577789412591274838:2116] self: [1:7577789412591274837:0] peer: [5:7577789413353925672:0] socket: 336 qp: -1
2025-11-28T14:53:06.670834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789412591274862:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:06.670883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:06.670992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410961400854:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.671759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789413353925675:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24096
2025-11-28T14:53:06.671261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789411377504015:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:06.671298Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789411377504015:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:06.672023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789412591274830:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.672180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:06.672425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789413353925651:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:06.672380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789411377504015:2120] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:06.672522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789412591274830:2109] [node 6] ICP22 created new session: [1:7577789412591274864:2109]
2025-11-28T14:53:06.672581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789412591274864:2109] [node 6] ICS09 handshake done sender: [1:7577789412591274836:2115] self: [1:7577789412591274835:0] peer: [6:7577789410961400865:0] socket: 368 qp: -1
2025-11-28T14:53:06.672923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789413353925644:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.673012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789411377504003:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:06.672634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789412591274864:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:06.672741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:06.673105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:06.673119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:06.673158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789413353925644:2075] [node 6] ICP22 created new session: [5:7577789413353925677:2075]
2025-11-28T14:53:06.673174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789411377504003:2088] [node 6] ICP22 created new session: [4:7577789411377504039:2088]
2025-11-28T14:53:06.673228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789411377504039:2088] [node 6] ICS09 handshake done sender: [4:7577789411377504015:2120] self: [4:7577789411377504014:0] peer: [6:7577789410961400868:0] socket: 372 qp: -1
2025-11-28T14:53:06.673184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410961400773:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.673299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789411377504039:2088] [node 6] ICS10 traffic start
2025-11-28T14:53:06.673210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789413353925677:2075] [node 6] ICS09 handshake done sender: [5:7577789413353925651:2114] self: [5:7577789413353925650:0] peer: [6:7577789410961400866:0] socket: 351 qp: -1
2025-11-28T14:53:06.673239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:06.673337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:06.673292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410961400773:2048] [node 1] ICP22 created new session: [6:7577789410961400871:2048]
2025-11-28T14:53:06.673342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410961400871:2048] [node 1] ICS09 handshake done sender: [6:7577789410961400854:2117] self: [6:7577789410961400865:0] peer: [1:7577789412591274835:0] socket: 367 qp: -1
2025-11-28T14:53:06.673255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789413353925677:2075] [node 6] ICS10 traffic start
2025-11-28T14:53:06.673291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:06.673377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410961400871:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:06.673412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:06.674469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789413353925669:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:06.678651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789413353925576:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.678716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:06.678786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789413353925576:2048] [node 1] ICP22 created new session: [5:7577789413353925679:2048]
2025-11-28T14:53:06.678834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789413353925679:2048] [node 1] ICS09 handshake done sender: [5:7577789413353925669:2121] self: [5:7577789413353925672:0] peer: [1:7577789412591274837:0] socket: 329 qp: -1
2025-11-28T14:53:06.678877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789413353925679:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:06.678910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:06.679589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789413353925576:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.682962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400869:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:06.683480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789410961400870:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:246
2025-11-28T14:53:06.683954Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789410961400839:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63975) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789410961400864:2121] held: no
2025-11-28T14:53:06.684712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789410961400838:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8286) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789410961400861:2120] held: no
2025-11-28T14:53:06.685812Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410961400864:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:06.686363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789410961400861:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:06.686622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410961400839:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.686676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:06.686728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410961400839:2063] [node 4] ICP22 created new session: [6:7577789410961400873:2063]
2025-11-28T14:53:06.686915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410961400873:2063] [node 4] ICS09 handshake done sender: [6:7577789410961400864:2121] self: [6:7577789410961400868:0] peer: [4:7577789411377504014:0] socket: 376 qp: -1
2025-11-28T14:53:06.686964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410961400873:2063] [node 4] ICS10 traffic start
2025-11-28T14:53:06.687008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:06.688126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789410961400838:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:06.688175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:06.688220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789410961400838:2061] [node 5] ICP22 created new session: [6:7577789410961400875:2061]
2025-11-28T14:53:06.688266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789410961400875:2061] [node 5] ICS09 handshake done sender: [6:7577789410961400861:2120] self: [6:7577789410961400866:0] peer: [5:7577789413353925650:0] socket: 371 qp: -1
2025-11-28T14:53:06.688299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789410961400875:2061] [node 5] ICS10 traffic start
2025-11-28T14:53:06.688333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:06.714776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:06.715005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789411732919296:2048] [node 1] ICS01 socket: 343 reason# EndOfStream
2025-11-28T14:53:06.715052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789407437951901:2048] [node 1] ICP30 unregister session Session# [2:7577789411732919296:2048] VirtualId# [2:7577789411732919291:0]
2025-11-28T14:53:06.715088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789411732919296:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.715185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:06.715233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:06.716142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:06.716347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789411733097445:2048] [node 1] ICS07 socket disconnect 340 reason# EndOfStream
2025-11-28T14:53:06.716396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789411733097445:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.716473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:06.716527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789411733097445:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:06.724596Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:06.719917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:06.724595Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:06.720072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789411377504036:2048] [node 1] ICS07 socket disconnect 341 reason# EndOfStream
2025-11-28T14:53:06.725206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789411377504036:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.744041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789407438130052:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:06.744106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789411733097445:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:06.724824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789413353925679:2048] [node 1] ICS07 socket disconnect 329 reason# EndOfStream
2025-11-28T14:53:06.724855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789413353925679:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.724953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:06.725010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789413353925679:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:06.724827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789410961400871:2048] [node 1] ICS07 socket disconnect 367 reason# EndOfStream
2025-11-28T14:53:06.724874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410961400871:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.724964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:06.725013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789410961400871:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:06.727428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:06.727557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789411377504036:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:06.749395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789413353925576:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:06.749449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789413353925679:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:06.749490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789413353925576:2048] [node 1] ICP30 unregister session Session# [5:7577789413353925679:2048] VirtualId# [5:7577789413353925672:0]
2025-11-28T14:53:06.749581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:06.749612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789413353925576:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.749626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.750221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789407438130052:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:06.750286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789411733097445:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:06.750315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789407438130052:2048] [node 1] ICP30 unregister session Session# [3:7577789411733097445:2048] VirtualId# [3:7577789411733097439:0]
2025-11-28T14:53:06.750421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:06.750477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789407438130052:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.750493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.751353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789410961400773:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:06.751416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410961400871:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:06.751445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410961400773:2048] [node 1] ICP30 unregister session Session# [6:7577789410961400871:2048] VirtualId# [6:7577789410961400865:0]
2025-11-28T14:53:06.751529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:06.751563Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789410961400773:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.751576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.748582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789407082536638:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:06.749015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789411377504036:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:06.749067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789407082536638:2048] [node 1] ICP30 unregister session Session# [4:7577789411377504036:2048] VirtualId# [4:7577789411377504024:0]
2025-11-28T14:53:06.749126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:06.749164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789407082536638:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.749185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32681) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.759424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789407082536638:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:06.760333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789407438130052:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:06.760757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789413353925576:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:06.761372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789410961400773:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:06.796582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:06.796814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789411733097434:2049] [node 2] ICS07 socket disconnect 347 reason# EndOfStream
2025-11-28T14:53:06.796843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789411733097434:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.796928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:06.796966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789411733097434:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:06.800837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789407438130053:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:06.800876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789411733097434:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:06.802991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:06.803210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789413353925659:2049] [node 2] ICS01 socket: 321 reason# EndOfStream
2025-11-28T14:53:06.803661Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:06.803247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789413353925577:2049] [node 2] ICP30 unregister session Session# [5:7577789413353925659:2049] VirtualId# [5:7577789413353925657:0]
2025-11-28T14:53:06.807102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:06.807370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789410961400862:2049] [node 2] ICS07 socket disconnect 355 reason# EndOfStream
2025-11-28T14:53:06.807422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410961400862:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.807541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:06.807594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789410961400862:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:06.803280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789413353925659:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.808441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789411377504027:2049] [node 2] ICS07 socket disconnect 331 reason# EndOfStream
2025-11-28T14:53:06.803349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:06.808487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789411377504027:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.803391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:06.808574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:06.808618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789411377504027:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:06.813697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789407438130053:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:06.813761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789411733097434:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:06.813797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789407438130053:2049] [node 2] ICP30 unregister session Session# [3:7577789411733097434:2049] VirtualId# [3:7577789411733097432:0]
2025-11-28T14:53:06.813905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:06.813953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789407438130053:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.813970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.824194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789407082536639:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:06.824278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789411377504027:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:06.824315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789407082536639:2049] [node 2] ICP30 unregister session Session# [4:7577789411377504027:2049] VirtualId# [4:7577789411377504022:0]
2025-11-28T14:53:06.825019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:06.825087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789407082536639:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.825111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.824822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789407438130053:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:06.825456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789410961400774:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:06.825597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410961400862:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:06.825664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410961400774:2049] [node 2] ICP30 unregister session Session# [6:7577789410961400862:2049] VirtualId# [6:7577789410961400855:0]
2025-11-28T14:53:06.825750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:06.825810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789410961400774:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.825829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15730) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.835119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789407082536639:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:06.836131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789410961400774:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:06.850349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:06.850601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789411377504034:2050] [node 3] ICS01 socket: 334 reason# EndOfStream
2025-11-28T14:53:06.850658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789407082536640:2050] [node 3] ICP30 unregister session Session# [4:7577789411377504034:2050] VirtualId# [4:7577789411377504021:0]
2025-11-28T14:53:06.850697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789411377504034:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.850787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:06.850841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:06.851533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:06.851661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410961400857:2050] [node 3] ICS01 socket: 359 reason# EndOfStream
2025-11-28T14:53:06.851693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410961400775:2050] [node 3] ICP30 unregister session Session# [6:7577789410961400857:2050] VirtualId# [6:7577789410961400853:0]
2025-11-28T14:53:06.851711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410961400857:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.851765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:06.851791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:06.853145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:06.853386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789413353925673:2050] [node 3] ICS07 socket disconnect 365 reason# EndOfStream
2025-11-28T14:53:06.853429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789413353925673:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.853494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:06.853541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789413353925673:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:06.854260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789413353925578:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19381) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:06.854313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789413353925673:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:06.854347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789413353925578:2050] [node 3] ICP30 unregister session Session# [5:7577789413353925673:2050] VirtualId# [5:7577789413353925666:0]
2025-11-28T14:53:06.854387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:06.854416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789413353925578:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.854430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.863816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789413353925578:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:06.909203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:06.909371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789413353925667:2079] [node 4] ICS07 socket disconnect 352 reason# EndOfStream
2025-11-28T14:53:06.909408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789413353925667:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.909513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:06.909565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789413353925667:2079] [node 4] ICS15 start handshake
2025-11-28T14:53:06.910236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789413353925645:2079] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63975) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:06.910285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789413353925667:2079] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:06.910322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789413353925645:2079] [node 4] ICP30 unregister session Session# [5:7577789413353925667:2079] VirtualId# [5:7577789413353925663:0]
2025-11-28T14:53:06.910328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:06.910413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:06.910447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789413353925645:2079] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63975) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.910462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63975) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:06.910463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410961400873:2063] [node 4] ICS01 socket: 376 reason# EndOfStream
2025-11-28T14:53:06.910496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410961400839:2063] [node 4] ICP30 unregister session Session# [6:7577789410961400873:2063] VirtualId# [6:7577789410961400868:0]
2025-11-28T14:53:06.910520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410961400873:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.910575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:06.910618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:06.934171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:06.934417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789410961400875:2061] [node 5] ICS01 socket: 371 reason# EndOfStream
2025-11-28T14:53:06.934483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789410961400838:2061] [node 5] ICP30 unregister session Session# [6:7577789410961400875:2061] VirtualId# [6:7577789410961400866:0]
2025-11-28T14:53:06.934536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789410961400875:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:06.934625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:06.934679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:07.018276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789417577454277:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:07.018343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789417577454278:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:07.043650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789415921246676:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:07.043702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789415921246677:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:07.064411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789418811834886:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:07.064455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789418811834887:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:07.084346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789417873249983:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:07.084403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789417873249984:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:07.084436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789417873249985:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:07.107926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789417876122297:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:07.107975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789417876122298:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:07.108005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789417876122299:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:07.144126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789417663227158:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:07.144176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789417663227159:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:07.412890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789415921246739:2105] [node 5] ICP01 ready to work
2025-11-28T14:53:07.413457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789415921246740:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:07.409579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789417577454342:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:07.412582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789417577454343:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:07.419212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789417577454344:2110] [node 6] ICP01 ready to work
2025-11-28T14:53:07.409490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789418811834949:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:07.144212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789417663227160:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:07.412441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789417873250046:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:07.468408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789417663227186:2064] [node 4] ICP01 ready to work
2025-11-28T14:53:07.440506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789417876122337:2085] [node 6] ICP01 ready to work
2025-11-28T14:53:07.412787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789418811834950:2107] [node 6] ICP01 ready to work
2025-11-28T14:53:07.465743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789417873250047:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:07.464508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789415921246743:2106] [node 6] ICP01 ready to work
2025-11-28T14:53:07.460747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789418811834951:2109] [node 5] ICP01 ready to work
2025-11-28T14:53:07.494441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789415921246740:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29484919ns) {Configure:104(29479070ns) {ProcessConfigured:135(56465ns) {StartInitialHandshake:157(54148ns) {DropHandshakes:470(1471ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(694ns) PrepareNewSessionHandshake:477(903ns) {SwitchToState:179(423ns)}}}}}
2025-11-28T14:53:07.585856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789417876122361:2089] [node 4] ICP01 ready to work
2025-11-28T14:53:07.583218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789417577454342:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(65611039ns) {Configure:104(65602218ns) {ProcessConfigured:135(87076ns) {StartInitialHandshake:157(83013ns) {DropHandshakes:470(2840ns) {DropIncomingHandshake:442(690ns) DropOutgoingHandshake:457(407ns)} GenerateSessionVirtualId:988(1043ns) PrepareNewSessionHandshake:477(1744ns) {SwitchToState:179(709ns)}}}}}
2025-11-28T14:53:07.592455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789417663227207:2071] [node 5] ICP01 ready to work
2025-11-28T14:53:07.612917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789417876122298:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(282389880ns) {Configure:104(282384242ns) {ProcessConfigured:135(84702ns) {StartInitialHandshake:157(81466ns) {DropHandshakes:470(3458ns) {DropIncomingHandshake:442(916ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(898ns) PrepareNewSessionHandshake:477(2105ns) {SwitchToState:179(1155ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.656801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789418811834950:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(84228674ns) {Configure:104(84223770ns) {ProcessConfigured:135(15987145ns) {StartInitialHandshake:157(15982714ns) {DropHandshakes:470(3239ns) {DropIncomingHandshake:442(781ns) DropOutgoingHandshake:457(383ns)} GenerateSessionVirtualId:988(810ns) PrepareNewSessionHandshake:477(7065ns) {SwitchToState:179(3589ns)}}}}}
2025-11-28T14:53:07.660178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789418811834962:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50198
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.673062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789415921246676:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(243676960ns) {Configure:104(243672794ns) {ProcessConfigured:135(62770ns) {StartInitialHandshake:157(60983ns) {DropHandshakes:470(2259ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(379ns)} GenerateSessionVirtualId:988(631ns) PrepareNewSessionHandshake:477(1628ns) {SwitchToState:179(627ns)}}}}}
2025-11-28T14:53:07.673832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789415921246746:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41540
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.638215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789417577454356:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58454
2025-11-28T14:53:07.692701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789415921246746:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.687981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789417876122373:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5751
2025-11-28T14:53:07.688099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789417876122374:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5751
2025-11-28T14:53:07.681840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789417663227159:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(262838506ns) {Configure:104(262829701ns) {ProcessConfigured:135(73580ns) {StartInitialHandshake:157(70794ns) {DropHandshakes:470(3556ns) {DropIncomingHandshake:442(1051ns) DropOutgoingHandshake:457(709ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:477(1630ns) {SwitchToState:179(551ns)}}}}}
2025-11-28T14:53:07.702272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789415921246676:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.712518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789418811834962:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.721450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789417663227160:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36915150ns) {Configure:104(36911086ns) {ProcessConfigured:135(74592ns) {StartInitialHandshake:157(71305ns) {DropHandshakes:470(2756ns) {DropIncomingHandshake:442(762ns) DropOutgoingHandshake:457(398ns)} GenerateSessionVirtualId:988(653ns) PrepareNewSessionHandshake:477(1105ns) {SwitchToState:179(654ns)}}}}}
2025-11-28T14:53:07.731892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789417876122374:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.728440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789417577454356:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.733091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789417876122299:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(117019877ns) {Configure:104(117014844ns) {ProcessConfigured:135(29027405ns) {StartInitialHandshake:157(29025367ns) {DropHandshakes:470(2730ns) {DropIncomingHandshake:442(581ns) DropOutgoingHandshake:457(835ns)} GenerateSessionVirtualId:988(599ns) PrepareNewSessionHandshake:477(5947ns) {SwitchToState:179(4254ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.740107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227225:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
2025-11-28T14:53:07.740347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227226:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
2025-11-28T14:53:07.733215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789417577454346:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.735440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789417876122298:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.733255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789417577454346:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.741702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227227:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.747284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789417663227226:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.747465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789417663227225:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.742754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789415921246743:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(90666610ns) {Configure:104(90661467ns) {ProcessConfigured:135(2357292ns) {StartInitialHandshake:157(2354907ns) {DropHandshakes:470(2839ns) {DropIncomingHandshake:442(692ns) DropOutgoingHandshake:457(569ns)} GenerateSessionVirtualId:988(769ns) PrepareNewSessionHandshake:477(2596ns) {SwitchToState:179(1548ns)}}}}}
2025-11-28T14:53:07.743598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789418811834886:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.746870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789417577454346:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:07.746587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789417876122373:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.745555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789415921246748:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.749026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789418811834966:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50198
2025-11-28T14:53:07.754988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789417577454340:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.748795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789417876122299:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.759746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789417876122363:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.759779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789417876122363:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.745592Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789415921246748:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.749346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789418811834955:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.755027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789417577454340:2109] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.754740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789417663227158:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(90232780ns) {Configure:104(90225713ns) {ProcessConfigured:135(3846765ns) {StartInitialHandshake:157(3843608ns) {DropHandshakes:470(2816ns) {DropIncomingHandshake:442(559ns) DropOutgoingHandshake:457(597ns)} GenerateSessionVirtualId:988(706ns) PrepareNewSessionHandshake:477(3200ns) {SwitchToState:179(1375ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.762536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789415921246758:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41540
2025-11-28T14:53:07.761523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789417876122363:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:07.749371Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789418811834955:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.756168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789417577454340:2109] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:07.758542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789417663227227:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.750763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789418811834955:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.751085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789418811834951:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.759088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789417663227207:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.756238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789417577454277:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.764000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789417873249984:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20878932ns) {Configure:104(20871121ns) {ProcessConfigured:135(55905ns) {StartInitialHandshake:157(53343ns) {DropHandshakes:470(2961ns) {DropIncomingHandshake:442(688ns) DropOutgoingHandshake:457(1017ns)} GenerateSessionVirtualId:988(551ns) PrepareNewSessionHandshake:477(1870ns) {SwitchToState:179(1066ns)}}}}}
2025-11-28T14:53:07.766071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789415921246748:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:07.764471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789417876122379:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5751
2025-11-28T14:53:07.751128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:07.756290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.759174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789417663227158:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.751181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789418811834951:2109] [node 5] ICP22 created new session: [3:7577789418811834970:2109]
2025-11-28T14:53:07.771195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789415921246742:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.756483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789417577454278:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.771231Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789415921246742:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.759478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789417663227160:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.778813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789417876122337:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.778874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:07.778925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789417876122337:2085] [node 6] ICP22 created new session: [5:7577789417876122383:2085]
2025-11-28T14:53:07.751227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789418811834970:2109] [node 5] ICS09 handshake done sender: [3:7577789418811834955:2111] self: [3:7577789418811834954:0] peer: [5:7577789417876122378:0] socket: 323 qp: -1
2025-11-28T14:53:07.763691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227242:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
2025-11-28T14:53:07.756531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:07.771898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789415921246760:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41540
2025-11-28T14:53:07.779759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789417876122373:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:07.751261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789418811834970:2109] [node 5] ICS10 traffic start
2025-11-28T14:53:07.772051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789415921246757:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41540
2025-11-28T14:53:07.779798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789417663227226:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:07.756599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789417577454278:2049] [node 3] ICP22 created new session: [1:7577789417577454361:2049]
2025-11-28T14:53:07.751299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:07.780249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789417876122383:2085] [node 6] ICS09 handshake done sender: [5:7577789417876122363:2110] self: [5:7577789417876122362:0] peer: [6:7577789417663227239:0] socket: 316 qp: -1
2025-11-28T14:53:07.772173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789415921246746:2112] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:07.756647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789417577454361:2049] [node 3] ICS09 handshake done sender: [1:7577789417577454340:2109] self: [1:7577789417577454339:0] peer: [3:7577789418811834967:0] socket: 312 qp: -1
2025-11-28T14:53:07.756681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789417577454361:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:07.756717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:07.761193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789417577454277:2048] [node 2] ICP22 created new session: [1:7577789417577454360:2048]
2025-11-28T14:53:07.762028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789417577454360:2048] [node 2] ICS09 handshake done sender: [1:7577789417577454346:2111] self: [1:7577789417577454345:0] peer: [2:7577789415921246754:0] socket: 313 qp: -1
2025-11-28T14:53:07.763028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789417577454360:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:07.763577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:07.768409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789417577454348:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.768436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789417577454348:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.770900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789417577454348:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:07.780496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789417876122383:2085] [node 6] ICS10 traffic start
2025-11-28T14:53:07.755021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789418811834966:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.773854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789415921246739:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.786409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789417577454344:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.784483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789417873249985:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33967042ns) {Configure:104(33961286ns) {ProcessConfigured:135(1517227ns) {StartInitialHandshake:157(1514317ns) {DropHandshakes:470(1264ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:477(2166ns) {SwitchToState:179(707ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.780542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:07.765277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789418811834968:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50198
2025-11-28T14:53:07.773900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:07.790617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789417876122299:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.786463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:07.786523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789417577454344:2110] [node 6] ICP22 created new session: [1:7577789417577454367:2110]
2025-11-28T14:53:07.786586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789417577454367:2110] [node 6] ICS09 handshake done sender: [1:7577789417577454348:2112] self: [1:7577789417577454347:0] peer: [6:7577789417663227240:0] socket: 314 qp: -1
2025-11-28T14:53:07.786625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789417577454367:2110] [node 6] ICS10 traffic start
2025-11-28T14:53:07.786667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:07.765414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789418811834887:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.797062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789417663227207:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.773952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789415921246739:2105] [node 5] ICP22 created new session: [2:7577789415921246762:2105]
2025-11-28T14:53:07.790668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:07.765622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789418811834973:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50198
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.790752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789417876122299:2050] [node 3] ICP22 created new session: [5:7577789417876122385:2050]
2025-11-28T14:53:07.773999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789415921246762:2105] [node 5] ICS09 handshake done sender: [2:7577789415921246748:2113] self: [2:7577789415921246747:0] peer: [5:7577789417876122377:0] socket: 329 qp: -1
2025-11-28T14:53:07.790809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789417876122385:2050] [node 3] ICS09 handshake done sender: [5:7577789417876122373:2114] self: [5:7577789417876122378:0] peer: [3:7577789418811834954:0] socket: 326 qp: -1
2025-11-28T14:53:07.765776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789418811834959:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.805516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:07.790848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789417876122385:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:07.774039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789415921246762:2105] [node 5] ICS10 traffic start
2025-11-28T14:53:07.805897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789417663227207:2071] [node 5] ICP22 created new session: [6:7577789417663227245:2071]
2025-11-28T14:53:07.774081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:07.765822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789418811834959:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.790881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.805975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789417663227245:2071] [node 5] ICS09 handshake done sender: [6:7577789417663227226:2112] self: [6:7577789417663227239:0] peer: [5:7577789417876122362:0] socket: 331 qp: -1
2025-11-28T14:53:07.806024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789417663227245:2071] [node 5] ICS10 traffic start
2025-11-28T14:53:07.806084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:07.809964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227243:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
2025-11-28T14:53:07.774363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789415921246676:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.774399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:07.774443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789415921246676:2048] [node 1] ICP22 created new session: [2:7577789415921246764:2048]
2025-11-28T14:53:07.774493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789415921246764:2048] [node 1] ICS09 handshake done sender: [2:7577789415921246746:2112] self: [2:7577789415921246754:0] peer: [1:7577789417577454345:0] socket: 320 qp: -1
2025-11-28T14:53:07.774525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789415921246764:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:07.774555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:07.781786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789415921246760:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.782752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789415921246757:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.783910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789415921246742:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:07.805791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789415921246677:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.807994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:07.808510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789415921246677:2049] [node 3] ICP22 created new session: [2:7577789415921246770:2049]
2025-11-28T14:53:07.811453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789415921246770:2049] [node 3] ICS09 handshake done sender: [2:7577789415921246742:2110] self: [2:7577789415921246741:0] peer: [3:7577789418811834974:0] socket: 327 qp: -1
2025-11-28T14:53:07.812016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789415921246770:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:07.816733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:07.819438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789417873250049:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46392
2025-11-28T14:53:07.773234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789418811834962:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:07.793991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789417876122382:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5751
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.828355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227244:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.774104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789418811834968:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.819651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789417873250048:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46392
2025-11-28T14:53:07.830048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789417663227225:2111] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:07.808314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789417876122381:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5751
2025-11-28T14:53:07.774150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789418811834886:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.819776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789417873250050:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46392
2025-11-28T14:53:07.774195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:07.822836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789417876122374:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:07.830249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789417577454350:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.820042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789417873250055:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46392
2025-11-28T14:53:07.831236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789417663227158:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.774241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789418811834886:2048] [node 1] ICP22 created new session: [3:7577789418811834976:2048]
2025-11-28T14:53:07.830296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789417577454350:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.825425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789417876122382:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.841924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789417577454278:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(51501271ns) {HandleHandshakeStatus:329(51497834ns) {DropIncomingHandshake:442(935ns) DropOutgoingHandshake:457(544ns) SwitchToState:179(415ns) ProcessPendingSessionEvents:495(51124610ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 50200691ns max# 6111714ns}}}
2025-11-28T14:53:07.842205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789417577454355:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.842228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789417577454355:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.831318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227247:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.820147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789417873250056:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46392
2025-11-28T14:53:07.825470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789417876122298:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.778393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789418811834976:2048] [node 1] ICS09 handshake done sender: [3:7577789418811834962:2115] self: [3:7577789418811834967:0] peer: [1:7577789417577454339:0] socket: 321 qp: -1
2025-11-28T14:53:07.825517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:07.833034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:07.830915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789417873250056:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.779255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789418811834976:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:07.825561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789417876122298:2049] [node 2] ICP22 created new session: [5:7577789417876122389:2049]
2025-11-28T14:53:07.846519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789417577454369:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58454
2025-11-28T14:53:07.837259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789417663227158:2048] [node 1] ICP22 created new session: [6:7577789417663227249:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.831496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789417873250055:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.848234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789417577454355:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:07.849466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789417577454350:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:07.825612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789417876122389:2049] [node 2] ICS09 handshake done sender: [5:7577789417876122374:2115] self: [5:7577789417876122377:0] peer: [2:7577789415921246747:0] socket: 333 qp: -1
2025-11-28T14:53:07.860771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789417577454343:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.860949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789417577454342:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.861137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:07.861210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789417577454342:2107] [node 4] ICP22 created new session: [1:7577789417577454373:2107]
2025-11-28T14:53:07.861265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789417577454373:2107] [node 4] ICS09 handshake done sender: [1:7577789417577454355:2114] self: [1:7577789417577454354:0] peer: [4:7577789417873250066:0] socket: 359 qp: -1
2025-11-28T14:53:07.861308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789417577454373:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:07.861347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.779296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:07.780850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789418811834959:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:07.808451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789418811834950:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.810335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:07.811233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789418811834950:2107] [node 6] ICP22 created new session: [3:7577789418811834979:2107]
2025-11-28T14:53:07.838324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789417663227249:2048] [node 1] ICS09 handshake done sender: [6:7577789417663227225:2111] self: [6:7577789417663227240:0] peer: [1:7577789417577454347:0] socket: 330 qp: -1
2025-11-28T14:53:07.811292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789418811834979:2107] [node 6] ICS09 handshake done sender: [3:7577789418811834959:2113] self: [3:7577789418811834958:0] peer: [6:7577789417663227241:0] socket: 325 qp: -1
2025-11-28T14:53:07.831592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789417873250048:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.825652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789417876122389:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:07.868514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789415921246677:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(58994531ns) {HandleHandshakeStatus:329(58992314ns) {DropIncomingHandshake:442(18896ns) DropOutgoingHandshake:457(371ns) SwitchToState:179(998ns) ProcessPendingSessionEvents:495(43378724ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41900404ns max# 7975717ns}}}
2025-11-28T14:53:07.865302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.838372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789417663227249:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.811827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789418811834979:2107] [node 6] ICS10 traffic start
2025-11-28T14:53:07.837573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789417873249983:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.838448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:07.843416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789417663227227:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:07.847468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789417663227160:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.847560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:07.847614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789417663227160:2050] [node 3] ICP22 created new session: [6:7577789417663227251:2050]
2025-11-28T14:53:07.847667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789417663227251:2050] [node 3] ICS09 handshake done sender: [6:7577789417663227227:2113] self: [6:7577789417663227241:0] peer: [3:7577789418811834958:0] socket: 332 qp: -1
2025-11-28T14:53:07.847709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789417663227251:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:07.847975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:07.855899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789417663227160:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:07.877521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227253:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
2025-11-28T14:53:07.825687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:07.865689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789417577454343:2108] [node 5] ICP22 created new session: [1:7577789417577454372:2108]
2025-11-28T14:53:07.812179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:07.813131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789418811834950:2107] [node 6] ICP18 handshake (actor [3:7577789418811834968:2117]) is too old
2025-11-28T14:53:07.873968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789418811834978:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50198
2025-11-28T14:53:07.864824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789417873250069:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46392
2025-11-28T14:53:07.829075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789417876122388:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5751
2025-11-28T14:53:07.865758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789417577454372:2108] [node 5] ICS09 handshake done sender: [1:7577789417577454350:2113] self: [1:7577789417577454349:0] peer: [5:7577789417876122391:0] socket: 358 qp: -1
2025-11-28T14:53:07.829150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789417876122297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.883666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789418811834957:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.866520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789417873250058:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.883709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789418811834957:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.865798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789417577454372:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:07.847926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789417876122388:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.866560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789417873250058:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.865849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:07.902632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789415921246739:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(69697641ns) {HandleHandshakeStatus:329(69695371ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(648ns) SwitchToState:179(642ns) ProcessPendingSessionEvents:495(69325012ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 69185309ns max# 6184265ns}}}
2025-11-28T14:53:07.884694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789418811834887:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17570) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789418811834966:2116] held: no
2025-11-28T14:53:07.904098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789417663227253:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.867411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789417873250049:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.853530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789417876122392:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5751
2025-11-28T14:53:07.885493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789418811834966:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:07.908039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789417663227186:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.867511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789417873250056:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:07.856154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789417876122361:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.898253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789418811834981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50198
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.912914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789415921246745:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.912308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789417663227247:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.865399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789417876122382:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:07.870023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789417873249983:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.870095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:07.898554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789418811834887:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.870155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789417873249983:2048] [node 1] ICP22 created new session: [4:7577789417873250070:2048]
2025-11-28T14:53:07.870203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789417873250070:2048] [node 1] ICS09 handshake done sender: [4:7577789417873250056:2116] self: [4:7577789417873250066:0] peer: [1:7577789417577454354:0] socket: 371 qp: -1
2025-11-28T14:53:07.870234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789417873250070:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:07.870271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:07.871614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789417873249985:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.874601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789417873250058:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:07.879697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789417873250050:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.880738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789417873250046:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.881738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789417873249984:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.890505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:07.897146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789417873250046:2108] [node 5] ICP22 created new session: [4:7577789417873250076:2108]
2025-11-28T14:53:07.902369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789417873250076:2108] [node 5] ICS09 handshake done sender: [4:7577789417873250058:2117] self: [4:7577789417873250057:0] peer: [5:7577789417876122393:0] socket: 374 qp: -1
2025-11-28T14:53:07.904047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789417873250076:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:07.906053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:07.912014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789417873250065:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.912040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789417873250065:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.917984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789417873250046:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(36112922ns) {HandleHandshakeStatus:329(36111271ns) {DropIncomingHandshake:442(14527ns) DropOutgoingHandshake:457(238ns) SwitchToState:179(304ns) ProcessPendingSessionEvents:495(9684714ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9334879ns max# 3003424ns}}}
2025-11-28T14:53:07.918122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789417873250077:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46392
2025-11-28T14:53:07.916681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789417663227159:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:07.912958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789415921246745:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.922518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789417577454342:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18815516ns) {HandleHandshakeStatus:329(18813487ns) {DropIncomingHandshake:442(410ns) DropOutgoingHandshake:457(374ns) SwitchToState:179(719ns) ProcessPendingSessionEvents:495(18272275ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18195443ns max# 4845343ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.923618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789415921246772:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41540
2025-11-28T14:53:07.879079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789417876122297:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.924086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789417873250049:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:07.898611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:07.879134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.926468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789415921246773:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41540
2025-11-28T14:53:07.926689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789415921246756:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.926713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789415921246756:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:07.928586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789415921246739:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:07.879186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789417876122297:2048] [node 1] ICP22 created new session: [5:7577789417876122394:2048]
2025-11-28T14:53:07.898674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789418811834887:2049] [node 2] ICP22 created new session: [3:7577789418811834983:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.930048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789415921246745:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:07.898736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789418811834983:2049] [node 2] ICS09 handshake done sender: [3:7577789418811834966:2116] self: [3:7577789418811834974:0] peer: [2:7577789415921246741:0] socket: 335 qp: -1
2025-11-28T14:53:07.879241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789417876122394:2048] [node 1] ICS09 handshake done sender: [5:7577789417876122382:2119] self: [5:7577789417876122391:0] peer: [1:7577789417577454349:0] socket: 361 qp: -1
2025-11-28T14:53:07.930495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789415921246756:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:07.931476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227256:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
2025-11-28T14:53:07.930482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789417873249985:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5828) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789417873250049:2111] held: no
2025-11-28T14:53:07.898776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789418811834983:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:07.879283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789417876122394:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:07.879788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:07.901035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789417876122361:2089] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14517) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789417876122388:2120] held: no
2025-11-28T14:53:07.905481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789417876122396:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5751
2025-11-28T14:53:07.916301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789417876122388:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:07.921140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789417876122361:2089] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.921186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:07.922373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789417876122361:2089] [node 4] ICP22 created new session: [5:7577789417876122397:2089]
2025-11-28T14:53:07.922496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789417876122397:2089] [node 4] ICS09 handshake done sender: [5:7577789417876122388:2120] self: [5:7577789417876122393:0] peer: [4:7577789417873250057:0] socket: 376 qp: -1
2025-11-28T14:53:07.922554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789417876122397:2089] [node 4] ICS10 traffic start
2025-11-28T14:53:07.922730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:07.933314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789415921246743:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.898837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:07.934017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789417873249985:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.934143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:07.933703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789417663227247:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:07.933615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:07.908985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789418811834886:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(79844204ns) {HandleHandshakeStatus:329(79841737ns) {DropIncomingHandshake:442(683ns) DropOutgoingHandshake:457(5729ns) SwitchToState:179(517ns) ProcessPendingSessionEvents:495(72996599ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 71017658ns max# 6156389ns}}}
2025-11-28T14:53:07.934219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789417873249985:2050] [node 3] ICP22 created new session: [4:7577789417873250080:2050]
2025-11-28T14:53:07.933803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789415921246743:2106] [node 6] ICP22 created new session: [2:7577789415921246776:2106]
2025-11-28T14:53:07.910040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789418811834981:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-28T14:53:07.910234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789418811834957:2112] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-28T14:53:07.914195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789418811834949:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.914239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:07.914300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789418811834949:2108] [node 4] ICP22 created new session: [3:7577789418811834985:2108]
2025-11-28T14:53:07.914347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789418811834985:2108] [node 4] ICS09 handshake done sender: [3:7577789418811834957:2112] self: [3:7577789418811834956:0] peer: [4:7577789417873250073:0] socket: 324 qp: -1
2025-11-28T14:53:07.914386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789418811834985:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:07.914425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:07.915874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789418811834949:2108] [node 4] ICP18 handshake (actor [3:7577789418811834981:2120]) is too old
2025-11-28T14:53:07.934088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789415921246740:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.934139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:07.934195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789415921246740:2107] [node 4] ICP22 created new session: [2:7577789415921246777:2107]
2025-11-28T14:53:07.934500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789415921246777:2107] [node 4] ICS09 handshake done sender: [2:7577789415921246745:2111] self: [2:7577789415921246744:0] peer: [4:7577789417873250075:0] socket: 328 qp: -1
2025-11-28T14:53:07.934540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789415921246777:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:07.934584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:07.936604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789415921246776:2106] [node 6] ICS09 handshake done sender: [2:7577789415921246756:2115] self: [2:7577789415921246755:0] peer: [6:7577789417663227255:0] socket: 366 qp: -1
2025-11-28T14:53:07.936683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789415921246776:2106] [node 6] ICS10 traffic start
2025-11-28T14:53:07.936726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:07.934196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789417663227159:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.934250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:07.934556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789417663227159:2049] [node 2] ICP22 created new session: [6:7577789417663227258:2049]
2025-11-28T14:53:07.934621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789417663227258:2049] [node 2] ICS09 handshake done sender: [6:7577789417663227247:2121] self: [6:7577789417663227255:0] peer: [2:7577789415921246755:0] socket: 369 qp: -1
2025-11-28T14:53:07.934657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789417663227258:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:07.936878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:07.937764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789417663227257:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54304
2025-11-28T14:53:07.934709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789417873250080:2050] [node 3] ICS09 handshake done sender: [4:7577789417873250049:2111] self: [4:7577789417873250073:0] peer: [3:7577789418811834956:0] socket: 337 qp: -1
2025-11-28T14:53:07.934753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789417873250080:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:07.934816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:07.938762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789417577454343:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(73137652ns) {HandleHandshakeStatus:329(73132164ns) {DropIncomingHandshake:442(16515ns) DropOutgoingHandshake:457(577ns) SwitchToState:179(578ns) ProcessPendingSessionEvents:495(67882098ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 65826625ns max# 7680336ns}}}
2025-11-28T14:53:07.939909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789417873250082:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46392
2025-11-28T14:53:07.940382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789417873250065:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:07.945088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789417873250047:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:07.945188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:07.945247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789417873250047:2109] [node 6] ICP22 created new session: [4:7577789417873250083:2109]
2025-11-28T14:53:07.945296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789417873250083:2109] [node 6] ICS09 handshake done sender: [4:7577789417873250065:2119] self: [4:7577789417873250064:0] peer: [6:7577789417663227254:0] socket: 382 qp: -1
2025-11-28T14:53:07.945331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789417873250083:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:07.945391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:07.947723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789415921246773:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:07.948017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789415921246740:2107] [node 4] ICP18 handshake (actor [2:7577789415921246773:2120]) is too old
2025-11-28T14:53:07.949289Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789417873250050:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:07.953223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789417873249984:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.953300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:07.953968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789417663227253:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:07.953356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789417873249984:2049] [node 2] ICP22 created new session: [4:7577789417873250085:2049]
2025-11-28T14:53:07.953403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789417873250085:2049] [node 2] ICS09 handshake done sender: [4:7577789417873250050:2112] self: [4:7577789417873250075:0] peer: [2:7577789415921246744:0] socket: 339 qp: -1
2025-11-28T14:53:07.953446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789417873250085:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:07.953480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:07.957503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789417663227186:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:07.958488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:07.963894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789415921246743:2106] [node 6] ICP35 event processing took too much time PendingConnection:279(29633396ns) {HandleHandshakeStatus:329(29630220ns) {DropIncomingHandshake:442(16417ns) DropOutgoingHandshake:457(392ns) SwitchToState:179(591ns) ProcessPendingSessionEvents:495(22808067ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22702803ns max# 3301070ns}}}
2025-11-28T14:53:07.958956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789417663227186:2064] [node 4] ICP22 created new session: [6:7577789417663227260:2064]
2025-11-28T14:53:07.960576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789417663227260:2064] [node 4] ICS09 handshake done sender: [6:7577789417663227253:2122] self: [6:7577789417663227254:0] peer: [4:7577789417873250064:0] socket: 383 qp: -1
2025-11-28T14:53:07.960629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789417663227260:2064] [node 4] ICS10 traffic start
2025-11-28T14:53:07.960702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:07.963135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789417663227186:2064] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:08.015605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:08.015850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789415921246764:2048] [node 1] ICS07 socket disconnect 320 reason# EndOfStream
2025-11-28T14:53:08.015881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789415921246764:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.015977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:08.016023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789415921246764:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:08.022264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789415921246676:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:08.022310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789415921246764:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:08.017973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:08.018879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789418811834976:2048] [node 1] ICS07 socket disconnect 321 reason# EndOfStream
2025-11-28T14:53:08.018910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789418811834976:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.018983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:08.019033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789418811834976:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:08.020358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789418811834886:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:08.020432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789418811834976:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:08.020472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789418811834886:2048] [node 1] ICP30 unregister session Session# [3:7577789418811834976:2048] VirtualId# [3:7577789418811834967:0]
2025-11-28T14:53:08.020599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:08.020633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789418811834886:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.020648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.024769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789415921246676:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:08.024858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789415921246764:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:08.024892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789415921246676:2048] [node 1] ICP30 unregister session Session# [2:7577789415921246764:2048] VirtualId# [2:7577789415921246754:0]
2025-11-28T14:53:08.024944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:08.024978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789415921246676:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.024993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.036836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789418811834886:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:08.041417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:08.041621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789417876122394:2048] [node 1] ICS01 socket: 361 reason# EndOfStream
2025-11-28T14:53:08.041658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789417876122297:2048] [node 1] ICP30 unregister session Session# [5:7577789417876122394:2048] VirtualId# [5:7577789417876122391:0]
2025-11-28T14:53:08.041686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789417876122394:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.041745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:08.041785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:08.040496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:08.040935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789417873250070:2048] [node 1] ICS01 socket: 371 reason# EndOfStream
2025-11-28T14:53:08.040985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789417873249983:2048] [node 1] ICP30 unregister session Session# [4:7577789417873250070:2048] VirtualId# [4:7577789417873250066:0]
2025-11-28T14:53:08.041027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789417873250070:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.041188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:08.041258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:08.040785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:08.041009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789417663227249:2048] [node 1] ICS07 socket disconnect 330 reason# EndOfStream
2025-11-28T14:53:08.041048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789417663227249:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.041190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:08.041266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789417663227249:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:08.050828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789415921246676:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:08.057440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789417663227158:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:08.057496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789417663227249:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:08.057521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789417663227158:2048] [node 1] ICP30 unregister session Session# [6:7577789417663227249:2048] VirtualId# [6:7577789417663227240:0]
2025-11-28T14:53:08.057561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:08.057589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789417663227158:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.057605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22244) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.067096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789417663227158:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:08.135042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:08.135211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789418811834983:2049] [node 2] ICS01 socket: 335 reason# EndOfStream
2025-11-28T14:53:08.135239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789418811834887:2049] [node 2] ICP30 unregister session Session# [3:7577789418811834983:2049] VirtualId# [3:7577789418811834974:0]
2025-11-28T14:53:08.135262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789418811834983:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.135318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:08.135346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:08.138991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:08.139240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789417876122389:2049] [node 2] ICS01 socket: 333 reason# EndOfStream
2025-11-28T14:53:08.139298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789417876122298:2049] [node 2] ICP30 unregister session Session# [5:7577789417876122389:2049] VirtualId# [5:7577789417876122377:0]
2025-11-28T14:53:08.139338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789417876122389:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.140497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:08.139448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:08.139521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:08.140504Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:08.140767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789417663227258:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-11-28T14:53:08.140821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789417663227159:2049] [node 2] ICP30 unregister session Session# [6:7577789417663227258:2049] VirtualId# [6:7577789417663227255:0]
2025-11-28T14:53:08.140858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789417663227258:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.140972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:08.141027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:08.140738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789417873250085:2049] [node 2] ICS07 socket disconnect 339 reason# EndOfStream
2025-11-28T14:53:08.140776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789417873250085:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.140866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:08.140906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789417873250085:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:08.150596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789417873249984:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17570) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:08.150676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789417873250085:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:08.150713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789417873249984:2049] [node 2] ICP30 unregister session Session# [4:7577789417873250085:2049] VirtualId# [4:7577789417873250075:0]
2025-11-28T14:53:08.150828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:08.150862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789417873249984:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17570) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.150882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17570) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.161389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789417873249984:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:08.192496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:08.192758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789417873250080:2050] [node 3] ICS01 socket: 337 reason# EndOfStream
2025-11-28T14:53:08.192801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789417873249985:2050] [node 3] ICP30 unregister session Session# [4:7577789417873250080:2050] VirtualId# [4:7577789417873250073:0]
2025-11-28T14:53:08.192838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789417873250080:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.192937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:08.192993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:08.192441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:08.192695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789417663227251:2050] [node 3] ICS07 socket disconnect 332 reason# EndOfStream
2025-11-28T14:53:08.192724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789417663227251:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.192810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:08.192851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789417663227251:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:08.195157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789417663227160:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5828) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:08.195215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789417663227251:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:08.195243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789417663227160:2050] [node 3] ICP30 unregister session Session# [6:7577789417663227251:2050] VirtualId# [6:7577789417663227241:0]
2025-11-28T14:53:08.195274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:08.195304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789417663227160:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5828) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.195316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5828) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.193204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:08.193386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789417876122385:2050] [node 3] ICS07 socket disconnect 326 reason# EndOfStream
2025-11-28T14:53:08.193410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789417876122385:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.193486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:08.193517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789417876122385:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:08.194452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789417876122299:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5828) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:08.194506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789417876122385:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:08.194542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789417876122299:2050] [node 3] ICP30 unregister session Session# [5:7577789417876122385:2050] VirtualId# [5:7577789417876122378:0]
2025-11-28T14:53:08.194582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:08.194615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789417876122299:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5828) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.194632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5828) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.204466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789417876122299:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:08.204672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789417663227160:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:08.256567Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:08.256793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789417876122397:2089] [node 4] ICS01 socket: 376 reason# EndOfStream
2025-11-28T14:53:08.256843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789417876122361:2089] [node 4] ICP30 unregister session Session# [5:7577789417876122397:2089] VirtualId# [5:7577789417876122393:0]
2025-11-28T14:53:08.256879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789417876122397:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.256972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:08.257017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:08.257114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:08.257303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789417663227260:2064] [node 4] ICS07 socket disconnect 383 reason# EndOfStream
2025-11-28T14:53:08.257335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789417663227260:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.257389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:08.257427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789417663227260:2064] [node 4] ICS15 start handshake
2025-11-28T14:53:08.258716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789417663227186:2064] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14517) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:08.258773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789417663227260:2064] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:08.258805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789417663227186:2064] [node 4] ICP30 unregister session Session# [6:7577789417663227260:2064] VirtualId# [6:7577789417663227254:0]
2025-11-28T14:53:08.258898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:08.258928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789417663227186:2064] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14517) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.258942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14517) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:08.272469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789417663227186:2064] [node 4] ICP33 wake up from error state
2025-11-28T14:53:08.344567Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:08.344816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789417663227245:2071] [node 5] ICS07 socket disconnect 331 reason# EndOfStream
2025-11-28T14:53:08.344846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789417663227245:2071] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:08.344957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:08.345002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789417663227245:2071] [node 5] ICS15 start handshake
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:08.460542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789422739571740:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:08.460614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789422739571741:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:08.477213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789423031307707:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:08.477394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789423031307708:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:08.508786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789422774867135:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:08.508856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789422774867136:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:08.535598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789422877138941:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:08.535649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789422877138942:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:08.535682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789422877138943:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:08.568721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789419879059099:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:08.568792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789419879059100:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:08.568829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789419879059101:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:08.579854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789421096797088:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:08.579898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789421096797089:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:08.579932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789421096797090:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:08.636657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789421096797144:2102] [node 5] ICP01 ready to work
2025-11-28T14:53:08.661583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789421096797145:2100] [node 4] ICP01 ready to work
2025-11-28T14:53:08.776256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789422774867198:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:08.777339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789422739571804:2109] [node 5] ICP01 ready to work
2025-11-28T14:53:08.786002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789423031307771:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:08.786859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789419879059165:2109] [node 4] ICP01 ready to work
2025-11-28T14:53:08.786937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789422774867200:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:08.787571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789422739571805:2110] [node 6] ICP01 ready to work
2025-11-28T14:53:08.787510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789422877139007:2109] [node 5] ICP01 ready to work
2025-11-28T14:53:08.788615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789419879059166:2110] [node 6] ICP01 ready to work
2025-11-28T14:53:08.787579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789423031307773:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:08.798571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789421096797090:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(152067546ns) {Configure:104(152061861ns) {ProcessConfigured:135(73614ns) {StartInitialHandshake:157(70718ns) {DropHandshakes:470(2722ns) {DropIncomingHandshake:442(671ns) DropOutgoingHandshake:457(603ns)} GenerateSessionVirtualId:988(918ns) PrepareNewSessionHandshake:477(1375ns) {SwitchToState:179(631ns)}}}}}
2025-11-28T14:53:08.788242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789422774867201:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:08.788439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789422739571806:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:08.789910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789422877139008:2111] [node 6] ICP01 ready to work
2025-11-28T14:53:08.788470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789423031307772:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:08.791394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789423031307708:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(152572151ns) {Configure:104(152566211ns) {ProcessConfigured:135(52051ns) {StartInitialHandshake:157(49403ns) {DropHandshakes:470(5299ns) {DropIncomingHandshake:442(813ns) DropOutgoingHandshake:457(518ns)} GenerateSessionVirtualId:988(717ns) PrepareNewSessionHandshake:477(924ns) {SwitchToState:179(442ns)}}}}}
2025-11-28T14:53:08.804669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789421096797088:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(90535083ns) {Configure:104(90530052ns) {ProcessConfigured:135(3516042ns) {StartInitialHandshake:157(3513289ns) {DropHandshakes:470(2455ns) {DropIncomingHandshake:442(597ns) DropOutgoingHandshake:457(639ns)} GenerateSessionVirtualId:988(749ns) PrepareNewSessionHandshake:477(3298ns) {SwitchToState:179(1558ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.866191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789419879059165:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17090350ns) {Configure:104(17086014ns) {ProcessConfigured:135(69315ns) {StartInitialHandshake:157(67493ns) {DropHandshakes:470(1875ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:477(1439ns) {SwitchToState:179(549ns)}}}}}
2025-11-28T14:53:08.872285Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789422877138941:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(60898294ns) {Configure:104(60895180ns) {ProcessConfigured:135(7455335ns) {StartInitialHandshake:157(7453085ns) {DropHandshakes:470(2358ns) {DropIncomingHandshake:442(542ns) DropOutgoingHandshake:457(435ns)} GenerateSessionVirtualId:988(1197ns) PrepareNewSessionHandshake:477(2077ns) {SwitchToState:179(850ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.872802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789422877139018:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41299
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.867903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789423031307707:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(60604264ns) {Configure:104(60599246ns) {ProcessConfigured:135(14305336ns) {StartInitialHandshake:157(14303062ns) {DropHandshakes:470(3173978ns) {DropIncomingHandshake:442(2979ns) DropOutgoingHandshake:457(478ns)} GenerateSessionVirtualId:988(2539ns) PrepareNewSessionHandshake:477(2083ns) {SwitchToState:179(771ns)}}}}}
2025-11-28T14:53:08.873416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789422774867199:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7466
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.897092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789419879059171:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3953
2025-11-28T14:53:08.890172Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789421096797089:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(87369937ns) {Configure:104(87365707ns) {ProcessConfigured:135(13361697ns) {StartInitialHandshake:157(13359987ns) {DropHandshakes:470(1377ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(308ns) PrepareNewSessionHandshake:477(3617ns) {SwitchToState:179(1549ns)}}}}}
2025-11-28T14:53:08.892062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789422774867203:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7466
2025-11-28T14:53:08.912800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789422774867202:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7466
2025-11-28T14:53:08.913231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789422774867199:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:08.890970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789421096797144:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(244408248ns) {Configure:104(244398577ns) {ProcessConfigured:135(44514ns) {StartInitialHandshake:157(42557ns) {DropHandshakes:470(1982ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(488ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:477(1436ns) {SwitchToState:179(778ns)}}}}}
2025-11-28T14:53:08.898372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789422877138942:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52372061ns) {Configure:104(52369278ns) {ProcessConfigured:135(30062015ns) {StartInitialHandshake:157(30059135ns) {DropHandshakes:470(7069ns) {DropIncomingHandshake:442(1785ns) DropOutgoingHandshake:457(844ns)} GenerateSessionVirtualId:988(3340ns) PrepareNewSessionHandshake:477(2517ns) {SwitchToState:179(477ns)}}}}}
2025-11-28T14:53:08.891619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789421096797164:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
2025-11-28T14:53:08.904956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789421096797164:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:08.911272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789421096797089:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.936997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789423031307779:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.932204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789419879059171:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.937042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789423031307779:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:08.931195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789421096797169:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.937500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789422774867198:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(84665320ns) {Configure:104(84661061ns) {ProcessConfigured:135(8643912ns) {StartInitialHandshake:157(8640890ns) {DropHandshakes:470(2417ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(446ns)} GenerateSessionVirtualId:988(1214ns) PrepareNewSessionHandshake:477(5242ns) {SwitchToState:179(3210ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.950189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789421096797169:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:08.961681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789421096797170:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
2025-11-28T14:53:08.942946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789422877139018:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:08.932387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789419879059100:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:08.951254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789423031307781:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:08.951297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789423031307781:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:08.962026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789423031307779:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:08.975488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789423031307787:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38508
2025-11-28T14:53:08.952464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789422774867203:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:08.953251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789422877139021:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41299
2025-11-28T14:53:08.961327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789422877139021:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:08.967343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789422739571741:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(177221363ns) {Configure:104(177215448ns) {ProcessConfigured:135(9117744ns) {StartInitialHandshake:157(9113965ns) {DropHandshakes:470(2530ns) {DropIncomingHandshake:442(635ns) DropOutgoingHandshake:457(352ns)} GenerateSessionVirtualId:988(394ns) PrepareNewSessionHandshake:477(3423ns) {SwitchToState:179(2196ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.994516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789422877139010:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:08.994552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789422877139010:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.995969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789422877139010:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:08.953376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789422774867202:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:08.953925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789422774867136:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:08.956600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789422774867212:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7466
2025-11-28T14:53:09.000427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789423031307775:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.000499Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789423031307775:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:08.960495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789419879059179:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3953
2025-11-28T14:53:08.973026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789421096797144:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.012650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789423031307781:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:09.013002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789419879059171:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:08.975617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789421096797167:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.022330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789422739571804:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(137435165ns) {Configure:104(137429932ns) {ProcessConfigured:135(12499518ns) {StartInitialHandshake:157(12495844ns) {DropHandshakes:470(7229933ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(1542ns)} GenerateSessionVirtualId:988(2160ns) PrepareNewSessionHandshake:477(1949ns) {SwitchToState:179(183ns)}}}}}
2025-11-28T14:53:09.022445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789422739571809:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19211
2025-11-28T14:53:09.022581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789422739571810:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19211
2025-11-28T14:53:09.022683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789422739571811:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19211
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.013875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789423031307787:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.003477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789422877139008:2111] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.014123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789419879059100:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:08.989103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789421096797170:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.009657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789422774867212:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.003546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:09.003599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789422877139008:2111] [node 6] ICP22 created new session: [4:7577789427172106321:2111]
2025-11-28T14:53:09.003653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789427172106321:2111] [node 6] ICS09 handshake done sender: [4:7577789422877139010:2112] self: [4:7577789422877139009:0] peer: [6:7577789421096797174:0] socket: 334 qp: -1
2025-11-28T14:53:09.003688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789427172106321:2111] [node 6] ICS10 traffic start
2025-11-28T14:53:09.003740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:08.993831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789421096797167:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.020757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789423031307775:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.024987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789419879059173:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.026309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789423031307772:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.026368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:09.026418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789423031307772:2108] [node 5] ICP22 created new session: [2:7577789427326275088:2108]
2025-11-28T14:53:09.026478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789427326275088:2108] [node 5] ICS09 handshake done sender: [2:7577789423031307779:2113] self: [2:7577789423031307778:0] peer: [5:7577789419879059178:0] socket: 321 qp: -1
2025-11-28T14:53:09.026507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789427326275088:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:09.026543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:09.027930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789427326275087:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38508
2025-11-28T14:53:09.029652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789423031307773:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.029704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:09.029747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789423031307773:2109] [node 6] ICP22 created new session: [2:7577789427326275093:2109]
2025-11-28T14:53:09.029800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789427326275093:2109] [node 6] ICS09 handshake done sender: [2:7577789423031307781:2114] self: [2:7577789423031307780:0] peer: [6:7577789421096797168:0] socket: 322 qp: -1
2025-11-28T14:53:09.029831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789427326275093:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:09.029864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:09.030811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789423031307708:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.030877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:09.030921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789423031307708:2049] [node 3] ICP22 created new session: [2:7577789427326275095:2049]
2025-11-28T14:53:09.030960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789427326275095:2049] [node 3] ICS09 handshake done sender: [2:7577789423031307775:2111] self: [2:7577789423031307774:0] peer: [3:7577789422774867220:0] socket: 320 qp: -1
2025-11-28T14:53:09.030988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789427326275095:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:09.031019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:09.034680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789427326275092:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38508
2025-11-28T14:53:09.013315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789427069834517:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7466
2025-11-28T14:53:09.071312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789427326275087:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.025027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789419879059173:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.994173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789421096797145:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.071974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789422739571809:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.072654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789422739571810:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.072947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789427034539116:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19211
2025-11-28T14:53:09.073195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789427034539118:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19211
2025-11-28T14:53:09.074549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789427034539116:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.016574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789422774867207:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.079700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789427172106324:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41299
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.028562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.105324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789427172106326:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41299
2025-11-28T14:53:09.080851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789422739571740:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(176702200ns) {Configure:104(176691699ns) {ProcessConfigured:135(8512843ns) {StartInitialHandshake:157(8508756ns) {DropHandshakes:470(2532ns) {DropIncomingHandshake:442(636ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(1186ns) PrepareNewSessionHandshake:477(1751ns) {SwitchToState:179(529ns)}}}}}
2025-11-28T14:53:08.994254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789421096797173:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
2025-11-28T14:53:09.016617Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789422774867207:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.024947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789422774867207:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:09.037587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789419879059173:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:09.092406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789427034539118:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.101522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789422739571811:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:08.995797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789421096797175:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
2025-11-28T14:53:09.124131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789427172106324:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.124235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789427326275092:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.044995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789419879059100:2049] [node 2] ICP22 created new session: [5:7577789424174026477:2049]
2025-11-28T14:53:09.000654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789421096797090:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.089881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789422774867136:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789422774867202:2111] held: no
2025-11-28T14:53:09.125787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789427172106325:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41299
2025-11-28T14:53:09.001630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789421096797167:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:09.150476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789422739571813:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.048853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789424174026477:2049] [node 2] ICS09 handshake done sender: [5:7577789419879059171:2113] self: [5:7577789419879059178:0] peer: [2:7577789423031307778:0] socket: 330 qp: -1
2025-11-28T14:53:09.052957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789424174026477:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:09.057451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:09.063113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789419879059166:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.094363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:09.113397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789419879059166:2110] [node 6] ICP22 created new session: [5:7577789424174026480:2110]
2025-11-28T14:53:09.122274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789419879059100:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(107176739ns) {HandleHandshakeStatus:329(107174362ns) {DropIncomingHandshake:442(739ns) DropOutgoingHandshake:457(16387ns) SwitchToState:179(76ns) ProcessPendingSessionEvents:495(19663676ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19593940ns max# 19135183ns}}}
2025-11-28T14:53:09.128414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789424174026481:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3953
2025-11-28T14:53:09.132430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789424174026480:2110] [node 6] ICS09 handshake done sender: [5:7577789419879059173:2114] self: [5:7577789419879059172:0] peer: [6:7577789421096797171:0] socket: 342 qp: -1
2025-11-28T14:53:09.132494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789424174026480:2110] [node 6] ICS10 traffic start
2025-11-28T14:53:09.144566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789424174026481:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.144910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789419879059101:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.145422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:09.149866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789423031307773:2109] [node 6] ICP18 handshake (actor [2:7577789427326275087:2117]) is too old
2025-11-28T14:53:09.090626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789422774867201:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.144476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789427172106326:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.005168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789421096797145:2100] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.150513Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789422739571813:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:09.090723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.192476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789423031307777:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.146428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789422877138941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.090773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789422774867201:2109] [node 6] ICP22 created new session: [3:7577789427069834521:2109]
2025-11-28T14:53:09.192563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789423031307777:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:09.005218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:09.152809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789422739571808:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.160633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789427172106325:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.090828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789427069834521:2109] [node 6] ICS09 handshake done sender: [3:7577789422774867207:2114] self: [3:7577789422774867206:0] peer: [6:7577789421096797172:0] socket: 345 qp: -1
2025-11-28T14:53:09.152846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789422739571808:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:09.181424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789422877138943:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.005260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789421096797145:2100] [node 4] ICP22 created new session: [6:7577789425391764472:2100]
2025-11-28T14:53:09.223524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789423031307777:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:09.198797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789422739571808:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:09.235027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789422739571813:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:09.090866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789427069834521:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.005303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789425391764472:2100] [node 4] ICS09 handshake done sender: [6:7577789421096797167:2116] self: [6:7577789421096797174:0] peer: [4:7577789422877139009:0] socket: 338 qp: -1
2025-11-28T14:53:09.090899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:09.227880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789423031307771:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.184112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789422877138942:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.005329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789425391764472:2100] [node 4] ICS10 traffic start
2025-11-28T14:53:09.237721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789422739571741:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.227976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:09.092017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789422774867202:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:09.005359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:09.198938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789422877138943:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(36206490ns) {IncomingHandshake:261(28999009ns) {DropIncomingHandshake:442(958ns) PrepareNewSessionHandshake:477(16600659ns) {SwitchToState:179(4307ns)} GenerateSessionVirtualId:988(717ns) IssueIncomingHandshakeReply:193(66256ns) {SwitchToState:179(172ns)}}}
2025-11-28T14:53:09.241181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789422739571806:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.099639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789422774867136:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.228046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789423031307771:2107] [node 4] ICP22 created new session: [2:7577789427326275100:2107]
2025-11-28T14:53:09.247641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789424174026484:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3953
2025-11-28T14:53:09.241259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:09.241301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789422739571806:2107] [node 4] ICP22 created new session: [1:7577789427034539129:2107]
2025-11-28T14:53:09.241347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789427034539129:2107] [node 4] ICS09 handshake done sender: [1:7577789422739571813:2115] self: [1:7577789422739571812:0] peer: [4:7577789427172106327:0] socket: 364 qp: -1
2025-11-28T14:53:09.241376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789427034539129:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:09.241433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:09.242557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789422739571815:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.242581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789422739571815:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:09.243646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:09.254765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789422739571741:2049] [node 3] ICP22 created new session: [1:7577789427034539128:2049]
2025-11-28T14:53:09.259475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789427034539128:2049] [node 3] ICS09 handshake done sender: [1:7577789422739571808:2111] self: [1:7577789422739571807:0] peer: [3:7577789427069834527:0] socket: 363 qp: -1
2025-11-28T14:53:09.269321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789427034539128:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:09.028626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789421096797164:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:09.283558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.228098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789427326275100:2107] [node 4] ICS09 handshake done sender: [2:7577789423031307777:2112] self: [2:7577789423031307776:0] peer: [4:7577789427172106329:0] socket: 370 qp: -1
2025-11-28T14:53:09.273965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789419879059101:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10781) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789424174026481:2117] held: no
2025-11-28T14:53:09.099719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:09.293067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789422739571815:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:09.036203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789425391764475:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
2025-11-28T14:53:09.205858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789427172106330:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41299
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.274766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789424174026481:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:09.060033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789425391764476:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
2025-11-28T14:53:09.060578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789421096797089:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.060675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:09.060729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789421096797089:2049] [node 2] ICP22 created new session: [6:7577789425391764477:2049]
2025-11-28T14:53:09.060781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789425391764477:2049] [node 2] ICS09 handshake done sender: [6:7577789421096797164:2114] self: [6:7577789421096797168:0] peer: [2:7577789423031307780:0] socket: 332 qp: -1
2025-11-28T14:53:09.060817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789425391764477:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:09.060862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:09.089647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789421096797170:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:09.108774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789421096797169:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:09.138472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789421096797089:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(56198288ns) {HandleHandshakeStatus:329(56184895ns) {DropIncomingHandshake:442(726ns) DropOutgoingHandshake:457(5606ns) SwitchToState:179(847ns) ProcessPendingSessionEvents:495(55730502ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 55629419ns max# 11320885ns}}}
2025-11-28T14:53:09.138540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789421096797090:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.138632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:09.138685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789421096797090:2050] [node 3] ICP22 created new session: [6:7577789425391764480:2050]
2025-11-28T14:53:09.138740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789425391764480:2050] [node 3] ICS09 handshake done sender: [6:7577789421096797170:2118] self: [6:7577789421096797172:0] peer: [3:7577789422774867206:0] socket: 348 qp: -1
2025-11-28T14:53:09.138775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789425391764480:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:09.138830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:09.144799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789421096797144:2102] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.144845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:09.144893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789421096797144:2102] [node 5] ICP22 created new session: [6:7577789425391764482:2102]
2025-11-28T14:53:09.144937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789425391764482:2102] [node 5] ICS09 handshake done sender: [6:7577789421096797169:2117] self: [6:7577789421096797171:0] peer: [5:7577789419879059172:0] socket: 343 qp: -1
2025-11-28T14:53:09.144972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789425391764482:2102] [node 5] ICS10 traffic start
2025-11-28T14:53:09.145005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:09.199843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789421096797090:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:09.232793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789425391764479:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
2025-11-28T14:53:09.237580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789425391764479:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.238293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789421096797088:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.291648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789425391764486:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39727
2025-11-28T14:53:09.293632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789425391764479:2123] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:09.293818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789421096797088:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.293865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:09.293910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789421096797088:2048] [node 1] ICP22 created new session: [6:7577789425391764487:2048]
2025-11-28T14:53:09.293953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789425391764487:2048] [node 1] ICS09 handshake done sender: [6:7577789425391764479:2123] self: [6:7577789425391764485:0] peer: [1:7577789422739571814:0] socket: 344 qp: -1
2025-11-28T14:53:09.293986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789425391764487:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:09.294021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:09.299139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789419879059101:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.099777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789422774867136:2049] [node 2] ICP22 created new session: [3:7577789427069834523:2049]
2025-11-28T14:53:09.223493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789427172106326:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.228139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789427326275100:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.099842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789427069834523:2049] [node 2] ICS09 handshake done sender: [3:7577789422774867202:2111] self: [3:7577789422774867220:0] peer: [2:7577789423031307774:0] socket: 325 qp: -1
2025-11-28T14:53:09.237620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789422877138943:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.099881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789427069834523:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:09.228179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:09.239024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789427172106331:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41299
2025-11-28T14:53:09.305053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789424174026485:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3953
2025-11-28T14:53:09.308967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789427326275102:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38508
2025-11-28T14:53:09.099920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:09.239435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:09.239672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789422877138943:2050] [node 3] ICP22 created new session: [4:7577789427172106333:2050]
2025-11-28T14:53:09.324527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789422739571805:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.305161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789424174026486:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3953
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.100638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789427069834520:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7466
2025-11-28T14:53:09.308651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:09.244464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789427172106333:2050] [node 3] ICS09 handshake done sender: [4:7577789427172106326:2120] self: [4:7577789427172106328:0] peer: [3:7577789422774867210:0] socket: 379 qp: -1
2025-11-28T14:53:09.308729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789419879059101:2050] [node 3] ICP22 created new session: [5:7577789424174026487:2050]
2025-11-28T14:53:09.133608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789427069834520:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.328930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:09.244504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789427172106333:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:09.244600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:09.245701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789422877138942:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789427172106325:2119] held: no
2025-11-28T14:53:09.285846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789427172106325:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:09.286317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789422877138941:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2891) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789427172106324:2118] held: no
2025-11-28T14:53:09.286946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789422877138942:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.286986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:09.287038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789422877138942:2049] [node 2] ICP22 created new session: [4:7577789427172106336:2049]
2025-11-28T14:53:09.287085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789427172106336:2049] [node 2] ICS09 handshake done sender: [4:7577789427172106325:2119] self: [4:7577789427172106329:0] peer: [2:7577789423031307776:0] socket: 375 qp: -1
2025-11-28T14:53:09.287123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789427172106336:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:09.287158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:09.304737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789427172106332:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41299
2025-11-28T14:53:09.320412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789427172106324:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:09.328806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789422877138941:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.333006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:09.333064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789422877138941:2048] [node 1] ICP22 created new session: [4:7577789427172106338:2048]
2025-11-28T14:53:09.333117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789427172106338:2048] [node 1] ICS09 handshake done sender: [4:7577789427172106324:2118] self: [4:7577789427172106327:0] peer: [1:7577789422739571812:0] socket: 374 qp: -1
2025-11-28T14:53:09.333157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789427172106338:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:09.333208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:09.338679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789422877139013:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.338707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789422877139013:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:09.342597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789422877139013:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:09.342818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789422877139007:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.342873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:09.349266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789422877139007:2109] [node 5] ICP22 created new session: [4:7577789427172106341:2109]
2025-11-28T14:53:09.357526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789423031307771:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(72602191ns) {HandleHandshakeStatus:329(72599079ns) {DropIncomingHandshake:442(15616ns) DropOutgoingHandshake:457(925ns) SwitchToState:179(725ns) ProcessPendingSessionEvents:495(72126474ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 72009574ns max# 14173200ns}}}
2025-11-28T14:53:09.329002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789422739571805:2110] [node 6] ICP22 created new session: [1:7577789427034539135:2110]
2025-11-28T14:53:09.308786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789424174026487:2050] [node 3] ICS09 handshake done sender: [5:7577789424174026481:2117] self: [5:7577789424174026482:0] peer: [3:7577789422774867216:0] socket: 378 qp: -1
2025-11-28T14:53:09.133736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789422774867135:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.329055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789427034539135:2110] [node 6] ICS09 handshake done sender: [1:7577789422739571815:2116] self: [1:7577789422739571814:0] peer: [6:7577789425391764485:0] socket: 349 qp: -1
2025-11-28T14:53:09.329092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789427034539135:2110] [node 6] ICS10 traffic start
2025-11-28T14:53:09.329141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:09.353877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789422739571741:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(115166991ns) {HandleHandshakeStatus:329(115164672ns) {DropIncomingHandshake:442(12661ns) DropOutgoingHandshake:457(247ns) SwitchToState:179(310ns) ProcessPendingSessionEvents:495(51813899ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 51728354ns max# 11111552ns}}}
2025-11-28T14:53:09.198532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789422774867217:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.198590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789422774867217:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:09.203916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789422774867211:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.203949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789422774867211:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:09.205281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789427069834528:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7466
2025-11-28T14:53:09.217431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789422774867211:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:09.219681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789422774867217:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:09.236591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789422774867200:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.236689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:09.236735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789422774867200:2107] [node 4] ICP22 created new session: [3:7577789427069834531:2107]
2025-11-28T14:53:09.236783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789427069834531:2107] [node 4] ICS09 handshake done sender: [3:7577789422774867211:2116] self: [3:7577789422774867210:0] peer: [4:7577789427172106328:0] socket: 328 qp: -1
2025-11-28T14:53:09.236822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789427069834531:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:09.236871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:09.238573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789422774867198:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.239300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:09.239811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789422774867198:2108] [node 5] ICP22 created new session: [3:7577789427069834533:2108]
2025-11-28T14:53:09.239860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789427069834533:2108] [node 5] ICS09 handshake done sender: [3:7577789422774867217:2118] self: [3:7577789422774867216:0] peer: [5:7577789424174026482:0] socket: 377 qp: -1
2025-11-28T14:53:09.239893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789427069834533:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:09.239926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:09.240961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789427069834520:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:09.258782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789422774867135:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.258880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:09.258932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789422774867135:2048] [node 1] ICP22 created new session: [3:7577789427069834535:2048]
2025-11-28T14:53:09.258981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789427069834535:2048] [node 1] ICS09 handshake done sender: [3:7577789427069834520:2120] self: [3:7577789427069834527:0] peer: [1:7577789422739571807:0] socket: 360 qp: -1
2025-11-28T14:53:09.259025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789427069834535:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:09.259076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:09.308832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789424174026487:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:09.369082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789427172106341:2109] [node 5] ICS09 handshake done sender: [4:7577789422877139013:2113] self: [4:7577789422877139012:0] peer: [5:7577789424174026489:0] socket: 318 qp: -1
2025-11-28T14:53:09.308896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:09.369178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789427172106341:2109] [node 5] ICS10 traffic start
2025-11-28T14:53:09.389125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789427326275102:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.321731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789424174026486:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.391219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789422774867135:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(131465211ns) {HandleHandshakeStatus:329(131462334ns) {DropIncomingHandshake:442(910ns) DropOutgoingHandshake:457(15495ns) SwitchToState:179(399ns) ProcessPendingSessionEvents:495(106425409ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 106252284ns max# 10360624ns}}}
2025-11-28T14:53:09.335406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789419879059165:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.391346Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789422774867135:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:09.336831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789419879059166:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(272666933ns) {HandleHandshakeStatus:329(272664534ns) {DropIncomingHandshake:442(233ns) DropOutgoingHandshake:457(389ns) SwitchToState:179(256ns) ProcessPendingSessionEvents:495(152557704ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 152384642ns max# 16420496ns}}}
2025-11-28T14:53:09.340750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789424174026490:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3953
2025-11-28T14:53:09.356022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789424174026486:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:09.369237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:09.356586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789424174026485:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.387580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789419879059099:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.389165Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789419879059165:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21409) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789424174026486:2120] held: no
2025-11-28T14:53:09.389772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789423031307707:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:09.392763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789427034539123:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.392805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789427034539123:2120] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.397337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789427326275104:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38508
2025-11-28T14:53:09.397741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789427326275102:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:09.393587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789427034539113:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:09.397780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789419879059165:2109] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:09.393608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789427034539113:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:09.401344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789423031307707:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.402270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:09.396632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789427034539123:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:09.398675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789422739571740:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.412964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:09.402331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789419879059165:2109] [node 4] ICP22 created new session: [5:7577789424174026492:2109]
2025-11-28T14:53:09.425856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789422877139007:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(82094569ns) {HandleHandshakeStatus:329(82091066ns) {DropIncomingHandshake:442(12393ns) DropOutgoingHandshake:457(400ns) SwitchToState:179(652ns) ProcessPendingSessionEvents:495(40925929ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40824251ns max# 11733309ns}}}
2025-11-28T14:53:09.398753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:09.410646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789424174026493:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3953
2025-11-28T14:53:09.416978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789419879059099:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2891) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789424174026485:2119] held: no
2025-11-28T14:53:09.418012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789424174026485:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:09.418603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789424174026492:2109] [node 4] ICS09 handshake done sender: [5:7577789424174026486:2120] self: [5:7577789424174026489:0] peer: [4:7577789422877139012:0] socket: 326 qp: -1
2025-11-28T14:53:09.428547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789424174026492:2109] [node 4] ICS10 traffic start
2025-11-28T14:53:09.436572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:09.439020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789419879059099:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:09.445201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:09.445280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789419879059099:2048] [node 1] ICP22 created new session: [5:7577789424174026495:2048]
2025-11-28T14:53:09.445346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789424174026495:2048] [node 1] ICS09 handshake done sender: [5:7577789424174026485:2119] self: [5:7577789424174026491:0] peer: [1:7577789427034539112:0] socket: 331 qp: -1
2025-11-28T14:53:09.445398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789424174026495:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:09.445439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:09.460827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789419879059165:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(63040301ns) {HandleHandshakeStatus:329(63037742ns) {DropIncomingHandshake:442(753ns) DropOutgoingHandshake:457(329ns) SwitchToState:179(590ns) ProcessPendingSessionEvents:495(2441ns)}}
2025-11-28T14:53:09.398802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789422739571740:2048] [node 2] ICP22 created new session: [1:7577789427034539138:2048]
2025-11-28T14:53:09.424573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789423031307707:2048] [node 1] ICP22 created new session: [2:7577789427326275105:2048]
2025-11-28T14:53:09.398849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789427034539138:2048] [node 2] ICS09 handshake done sender: [1:7577789427034539123:2120] self: [1:7577789427034539119:0] peer: [2:7577789427326275103:0] socket: 335 qp: -1
2025-11-28T14:53:09.398880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789427034539138:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:09.398921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:09.402128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789427034539113:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:09.408721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789422739571804:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:09.408829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:09.408891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789422739571804:2109] [node 5] ICP22 created new session: [1:7577789427034539141:2109]
2025-11-28T14:53:09.408941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789427034539141:2109] [node 5] ICS09 handshake done sender: [1:7577789427034539113:2117] self: [1:7577789427034539112:0] peer: [5:7577789424174026491:0] socket: 319 qp: -1
2025-11-28T14:53:09.408999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789427034539141:2109] [node 5] ICS10 traffic start
2025-11-28T14:53:09.409039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:09.436877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789427326275105:2048] [node 1] ICS09 handshake done sender: [2:7577789427326275102:2119] self: [2:7577789427326275103:0] peer: [1:7577789427034539119:0] socket: 353 qp: -1
2025-11-28T14:53:09.436939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789427326275105:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:09.437011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:09.438740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789423031307707:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(36728085ns) {HandleHandshakeStatus:329(36725745ns) {DropIncomingHandshake:442(1180ns) DropOutgoingHandshake:457(5066ns) SwitchToState:179(1026ns) ProcessPendingSessionEvents:495(607778ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 546869ns max# 69827ns}}}
2025-11-28T14:53:09.438881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789423031307707:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:09.518167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:09.518422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789427326275105:2048] [node 1] ICS07 socket disconnect 353 reason# EndOfStream
2025-11-28T14:53:09.518456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789427326275105:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.518537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:09.518581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789427326275105:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:09.521300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:09.521474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789427069834535:2048] [node 1] ICS01 socket: 360 reason# EndOfStream
2025-11-28T14:53:09.521505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789422774867135:2048] [node 1] ICP30 unregister session Session# [3:7577789427069834535:2048] VirtualId# [3:7577789427069834527:0]
2025-11-28T14:53:09.521530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789427069834535:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.521573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:09.521602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:09.523626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789423031307707:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2891) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:09.523667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789427326275105:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:09.524309Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789423031307707:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2891) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:09.524343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789427326275105:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:09.524392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789423031307707:2048] [node 1] ICP30 unregister session Session# [2:7577789427326275105:2048] VirtualId# [2:7577789427326275103:0]
2025-11-28T14:53:09.524473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:09.524502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789423031307707:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2891) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.524515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2891) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.527063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:09.527285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789427172106338:2048] [node 1] ICS01 socket: 374 reason# EndOfStream
2025-11-28T14:53:09.527331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789422877138941:2048] [node 1] ICP30 unregister session Session# [4:7577789427172106338:2048] VirtualId# [4:7577789427172106327:0]
2025-11-28T14:53:09.527411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789427172106338:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.527504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:09.527549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:09.533515Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:09.536556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789425391764487:2048] [node 1] ICS07 socket disconnect 344 reason# EndOfStream
2025-11-28T14:53:09.536729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:09.536616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789425391764487:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.536777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:09.536842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789425391764487:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:09.542546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789421096797088:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2891) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:09.542643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789424174026495:2048] [node 1] ICS01 socket: 331 reason# EndOfStream
2025-11-28T14:53:09.543116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789425391764487:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:09.544934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789421096797088:2048] [node 1] ICP30 unregister session Session# [6:7577789425391764487:2048] VirtualId# [6:7577789425391764485:0]
2025-11-28T14:53:09.545235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:09.545278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789421096797088:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2891) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.545300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2891) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.544509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789419879059099:2048] [node 1] ICP30 unregister session Session# [5:7577789424174026495:2048] VirtualId# [5:7577789424174026491:0]
2025-11-28T14:53:09.544562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789424174026495:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.544679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:09.544730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:09.556926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789421096797088:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:09.602806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:09.603054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789427069834523:2049] [node 2] ICS01 socket: 325 reason# EndOfStream
2025-11-28T14:53:09.603113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789422774867136:2049] [node 2] ICP30 unregister session Session# [3:7577789427069834523:2049] VirtualId# [3:7577789422774867220:0]
2025-11-28T14:53:09.603147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789427069834523:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.603269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:09.603332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:09.608594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:09.608836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789427172106336:2049] [node 2] ICS01 socket: 375 reason# EndOfStream
2025-11-28T14:53:09.608888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789422877138942:2049] [node 2] ICP30 unregister session Session# [4:7577789427172106336:2049] VirtualId# [4:7577789427172106329:0]
2025-11-28T14:53:09.608940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789427172106336:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.609042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:09.609094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:09.608603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:09.608976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789424174026477:2049] [node 2] ICS07 socket disconnect 330 reason# EndOfStream
2025-11-28T14:53:09.609032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789424174026477:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.609194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:09.609279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789424174026477:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:09.608757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:09.609086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789425391764477:2049] [node 2] ICS07 socket disconnect 332 reason# EndOfStream
2025-11-28T14:53:09.620851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789419879059100:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27798) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:09.609131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789425391764477:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.609265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:09.609324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789425391764477:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:09.624991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789424174026477:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:09.625051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789419879059100:2049] [node 2] ICP30 unregister session Session# [5:7577789424174026477:2049] VirtualId# [5:7577789419879059178:0]
2025-11-28T14:53:09.625574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:09.625616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789419879059100:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.625632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.633736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789421096797089:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27798) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:09.633798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789425391764477:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:09.633836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789421096797089:2049] [node 2] ICP30 unregister session Session# [6:7577789425391764477:2049] VirtualId# [6:7577789421096797168:0]
2025-11-28T14:53:09.633955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:09.633995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789421096797089:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.634010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.641063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789419879059100:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:09.644613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789421096797089:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:09.690494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:09.690766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789427172106333:2050] [node 3] ICS01 socket: 379 reason# EndOfStream
2025-11-28T14:53:09.690825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789422877138943:2050] [node 3] ICP30 unregister session Session# [4:7577789427172106333:2050] VirtualId# [4:7577789427172106328:0]
2025-11-28T14:53:09.690862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789427172106333:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.690948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:09.690989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:09.693041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:09.693216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789424174026487:2050] [node 3] ICS01 socket: 378 reason# EndOfStream
2025-11-28T14:53:09.693258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789419879059101:2050] [node 3] ICP30 unregister session Session# [5:7577789424174026487:2050] VirtualId# [5:7577789424174026482:0]
2025-11-28T14:53:09.693290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789424174026487:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.693367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:09.693416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:09.693512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:09.693654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789425391764480:2050] [node 3] ICS07 socket disconnect 348 reason# EndOfStream
2025-11-28T14:53:09.693677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789425391764480:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.693715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:09.693741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789425391764480:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:09.694753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789421096797090:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10781) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:09.694790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789425391764480:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:09.694817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789421096797090:2050] [node 3] ICP30 unregister session Session# [6:7577789425391764480:2050] VirtualId# [6:7577789421096797172:0]
2025-11-28T14:53:09.694846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:09.694867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789421096797090:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10781) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.694880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10781) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:09.705394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789421096797090:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:09.799730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:09.799891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789424174026492:2109] [node 4] ICS01 socket: 326 reason# EndOfStream
2025-11-28T14:53:09.799934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789419879059165:2109] [node 4] ICP30 unregister session Session# [5:7577789424174026492:2109] VirtualId# [5:7577789424174026489:0]
2025-11-28T14:53:09.799965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789424174026492:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.800058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:09.800090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:09.801011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:09.801249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789425391764472:2100] [node 4] ICS01 socket: 338 reason# EndOfStream
2025-11-28T14:53:09.801288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789421096797145:2100] [node 4] ICP30 unregister session Session# [6:7577789425391764472:2100] VirtualId# [6:7577789421096797174:0]
2025-11-28T14:53:09.801326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789425391764472:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.801444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:09.801498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:09.856566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:09.856816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789425391764482:2102] [node 5] ICS01 socket: 343 reason# EndOfStream
2025-11-28T14:53:09.856862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789421096797144:2102] [node 5] ICP30 unregister session Session# [6:7577789425391764482:2102] VirtualId# [6:7577789421096797171:0]
2025-11-28T14:53:09.856901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789425391764482:2102] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:09.857041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:09.857092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:09.988212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789427621124881:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:09.988271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789427621124882:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:10.014267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789430542738627:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:10.014316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789430542738628:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:10.043688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789430066951320:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:10.043738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789430066951321:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:10.071959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789429435405535:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:10.072013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789429435405536:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:10.072047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789429435405537:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:10.104664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789429577295487:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:10.104727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789429577295488:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:10.104763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789429577295489:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:10.134645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789430906186762:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:10.134695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789430906186763:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:10.134726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789430906186764:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:10.420509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789430906186823:2101] [node 4] ICP01 ready to work
2025-11-28T14:53:10.417223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789430906186821:2096] [node 5] ICP01 ready to work
2025-11-28T14:53:10.526806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789431916092240:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:10.564414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789431916092241:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:10.532931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789429577295488:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20924874ns) {Configure:104(20919088ns) {ProcessConfigured:135(107431ns) {StartInitialHandshake:157(85922ns) {DropHandshakes:470(3574ns) {DropIncomingHandshake:442(813ns) DropOutgoingHandshake:457(694ns)} GenerateSessionVirtualId:988(1566ns) PrepareNewSessionHandshake:477(2706ns) {SwitchToState:179(1124ns)}}}}}
2025-11-28T14:53:10.568006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789430542738690:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:10.580543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789430066951383:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:10.582309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789429435405535:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(131940251ns) {Configure:104(131933761ns) {ProcessConfigured:135(68521ns) {StartInitialHandshake:157(66049ns) {DropHandshakes:470(2824ns) {DropIncomingHandshake:442(654ns) DropOutgoingHandshake:457(756ns)} GenerateSessionVirtualId:988(594ns) PrepareNewSessionHandshake:477(1537ns) {SwitchToState:179(651ns)}}}}}
2025-11-28T14:53:10.583424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789430542738692:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:10.585156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789431916092242:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:10.593487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789429577295557:2110] [node 6] ICP01 ready to work
2025-11-28T14:53:10.592020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789429577295556:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:10.591068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789430066951384:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:10.588133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789429435405610:2108] [node 6] ICP01 ready to work
2025-11-28T14:53:10.588753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789429435405607:2109] [node 5] ICP01 ready to work
2025-11-28T14:53:10.586896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789430542738691:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:10.615411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789430906186763:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(42998729ns) {Configure:104(42993435ns) {ProcessConfigured:135(88768ns) {StartInitialHandshake:157(86035ns) {DropHandshakes:470(3182ns) {DropIncomingHandshake:442(757ns) DropOutgoingHandshake:457(508ns)} GenerateSessionVirtualId:988(2025ns) PrepareNewSessionHandshake:477(2256ns) {SwitchToState:179(846ns)}}}}}
2025-11-28T14:53:10.605863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789430066951385:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.652674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789430542738693:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12342
2025-11-28T14:53:10.650824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789431916092243:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35189
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.650904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789431916092244:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35189
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.733324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789430906186764:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(146107650ns) {Configure:104(146102613ns) {ProcessConfigured:135(69217ns) {StartInitialHandshake:157(66861ns) {DropHandshakes:470(2132ns) {DropIncomingHandshake:442(461ns) DropOutgoingHandshake:457(387ns)} GenerateSessionVirtualId:988(617ns) PrepareNewSessionHandshake:477(2037ns) {SwitchToState:179(977ns)}}}}}
2025-11-28T14:53:10.735479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789430906186833:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.677362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789427621124881:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(63640606ns) {Configure:104(63633966ns) {ProcessConfigured:135(81988ns) {StartInitialHandshake:157(79472ns) {DropHandshakes:470(2770ns) {DropIncomingHandshake:442(621ns) DropOutgoingHandshake:457(526ns)} GenerateSessionVirtualId:988(1490ns) PrepareNewSessionHandshake:477(1562ns) {SwitchToState:179(597ns)}}}}}
2025-11-28T14:53:10.722749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789429577295567:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43351
2025-11-28T14:53:10.732601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789429577295565:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43351
2025-11-28T14:53:10.732759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789429577295566:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43351
2025-11-28T14:53:10.749939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789430066951321:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58028305ns) {Configure:104(58022833ns) {ProcessConfigured:135(20444600ns) {StartInitialHandshake:157(20441267ns) {DropHandshakes:470(2376ns) {DropIncomingHandshake:442(495ns) DropOutgoingHandshake:457(476ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:477(3964ns) {SwitchToState:179(2115ns)}}}}}
2025-11-28T14:53:10.750269Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789430066951384:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24123594ns) {Configure:104(24118116ns) {ProcessConfigured:135(18498069ns) {StartInitialHandshake:157(18495307ns) {DropHandshakes:470(984ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(112ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:477(5157ns) {SwitchToState:179(3251ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.739628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789431916092241:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(57349484ns) {Configure:104(57342678ns) {ProcessConfigured:135(29446525ns) {StartInitialHandshake:157(29444235ns) {DropHandshakes:470(2625ns) {DropIncomingHandshake:442(779ns) DropOutgoingHandshake:457(423ns)} GenerateSessionVirtualId:988(926ns) PrepareNewSessionHandshake:477(3078ns) {SwitchToState:179(1665ns)}}}}}
2025-11-28T14:53:10.749228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789430906186833:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.753370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789429577295567:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.756832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789429577295488:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:10.757022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789429577295566:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.753308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789430066951386:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57862
2025-11-28T14:53:10.756465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789429435405613:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30322
2025-11-28T14:53:10.769210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789429577295487:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.764708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789430542738628:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(157624014ns) {Configure:104(157618354ns) {ProcessConfigured:135(22783654ns) {StartInitialHandshake:157(22782167ns) {DropHandshakes:470(1376ns) {DropIncomingHandshake:442(448ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(440ns) PrepareNewSessionHandshake:477(2501ns) {SwitchToState:179(1254ns)}}}}}
2025-11-28T14:53:10.768928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789430542738627:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(170434549ns) {Configure:104(170423585ns) {ProcessConfigured:135(64975ns) {StartInitialHandshake:157(61483ns) {DropHandshakes:470(2224ns) {DropIncomingHandshake:442(463ns) DropOutgoingHandshake:457(478ns)} GenerateSessionVirtualId:988(1360ns) PrepareNewSessionHandshake:477(1247ns) {SwitchToState:179(407ns)}}}}}
2025-11-28T14:53:10.771363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789430542738708:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12342
2025-11-28T14:53:10.777083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789429435405537:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(184343403ns) {Configure:104(184338731ns) {ProcessConfigured:135(13645245ns) {StartInitialHandshake:157(13642577ns) {DropHandshakes:470(2017ns) {DropIncomingHandshake:442(407ns) DropOutgoingHandshake:457(460ns)} GenerateSessionVirtualId:988(702ns) PrepareNewSessionHandshake:477(2062ns) {SwitchToState:179(77ns)}}}}}
2025-11-28T14:53:10.760903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789430066951390:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57862
2025-11-28T14:53:10.771086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789429577295570:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43351
2025-11-28T14:53:10.764840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789430906186762:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(157721972ns) {Configure:104(157717915ns) {ProcessConfigured:135(39533114ns) {StartInitialHandshake:157(34849702ns) {DropHandshakes:470(2772ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(1384ns) PrepareNewSessionHandshake:477(3616ns) {SwitchToState:179(1699ns)}}}}}
2025-11-28T14:53:10.761210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789431916092244:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.792862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789429577295561:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.792907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789429577295561:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:10.794066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789429577295572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43351
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.788007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789429435405536:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(160649657ns) {Configure:104(160644364ns) {ProcessConfigured:135(50552689ns) {StartInitialHandshake:157(50550731ns) {DropHandshakes:470(11202201ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(2169ns)} GenerateSessionVirtualId:988(1415ns) PrepareNewSessionHandshake:477(4590ns) {SwitchToState:179(1919ns)}}}}}
2025-11-28T14:53:10.795979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789429435405613:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.796257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789429435405621:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30322
2025-11-28T14:53:10.796343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789429435405536:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.780616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789431916092246:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.771740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789430066951390:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.786708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789430906186821:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:10.787219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789430542738697:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.805044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789429577295566:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.772224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789430066951320:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:10.809536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789430906186834:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
2025-11-28T14:53:10.787281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789430542738697:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:10.808810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789430542738708:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.811223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789430542738711:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12342
2025-11-28T14:53:10.821715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789430542738695:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.821764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789430542738695:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:10.821843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789430542738704:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12342
2025-11-28T14:53:10.832619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789430542738697:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:10.841599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789430542738693:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.841188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789429435405623:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30322
2025-11-28T14:53:10.811688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789429577295561:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:10.817799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789429577295573:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43351
2025-11-28T14:53:10.849207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789429577295487:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:10.849778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789429577295567:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:10.780651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789431916092246:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:10.852682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789429435405621:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.852860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789429435405537:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:10.850966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789430542738695:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:10.772763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789430066951396:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57862
2025-11-28T14:53:10.781859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789430066951386:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.810783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789430066951396:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.820900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789430066951402:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57862
2025-11-28T14:53:10.854434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789430066951383:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(72441286ns) {Configure:104(72430742ns) {ProcessConfigured:135(71218ns) {StartInitialHandshake:157(67291ns) {DropHandshakes:470(2459ns) {DropIncomingHandshake:442(457ns) DropOutgoingHandshake:457(587ns)} GenerateSessionVirtualId:988(787ns) PrepareNewSessionHandshake:477(1397ns) {SwitchToState:179(785ns)}}}}}
2025-11-28T14:53:10.854629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789430066951393:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.854654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789430066951393:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:10.856177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789430066951390:2112] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:10.856705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789430066951393:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:10.787589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789431916092246:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:10.810014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789430906186843:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
2025-11-28T14:53:10.852344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:10.857260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789429577295570:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.860454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789429577295487:2048] [node 1] ICP22 created new session: [5:7577789429577295574:2048]
2025-11-28T14:53:10.865373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789429577295488:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:10.865464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:10.865516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789429577295488:2049] [node 2] ICP22 created new session: [5:7577789429577295577:2049]
2025-11-28T14:53:10.865562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789429577295577:2049] [node 2] ICS09 handshake done sender: [5:7577789429577295567:2117] self: [5:7577789429577295568:0] peer: [2:7577789430542738696:0] socket: 347 qp: -1
2025-11-28T14:53:10.865609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789429577295577:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:10.865645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:10.867884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789429577295565:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.867971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789429577295556:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:10.877253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789429577295557:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:10.880470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789429577295574:2048] [node 1] ICS09 handshake done sender: [5:7577789429577295566:2116] self: [5:7577789429577295569:0] peer: [1:7577789431916092245:0] socket: 346 qp: -1
2025-11-28T14:53:10.804467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789431916092248:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.856911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789430542738711:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.864430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789430906186834:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.900830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789429435405615:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.900875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789429435405615:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:10.902351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789429435405536:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789429435405613:2120] held: no
2025-11-28T14:53:10.903721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789429435405613:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:10.880583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789430066951320:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:10.880693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.804518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789431916092248:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:10.885169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789429577295574:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:10.878247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789430906186833:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:10.904560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789429435405536:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:10.857508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789430542738692:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.880757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789430066951320:2048] [node 1] ICP22 created new session: [3:7577789430066951407:2048]
2025-11-28T14:53:10.861762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789430542738690:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:10.808925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789431916092260:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35189
2025-11-28T14:53:10.885239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:10.879281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789430906186763:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:10.880815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789430066951407:2048] [node 1] ICS09 handshake done sender: [3:7577789430066951390:2112] self: [3:7577789430066951400:0] peer: [1:7577789431916092247:0] socket: 340 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.909978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789429435405626:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30322
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.896843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:10.851469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789431916092243:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.910082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789429435405627:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30322
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.861833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:10.916437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789430906186821:2096] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:10.880845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789430066951407:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:10.851574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789431916092248:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:10.917076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789429577295582:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43351
2025-11-28T14:53:10.920375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789430906186846:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
2025-11-28T14:53:10.923875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789429435405615:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:10.861891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789430542738690:2107] [node 4] ICP22 created new session: [2:7577789430542738714:2107]
2025-11-28T14:53:10.880898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:10.940435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:10.917137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789429577295583:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43351
2025-11-28T14:53:10.920439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789430906186847:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
2025-11-28T14:53:10.852462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789427621124882:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:10.861942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789430542738714:2107] [node 4] ICS09 handshake done sender: [2:7577789430542738695:2111] self: [2:7577789430542738694:0] peer: [4:7577789429435405622:0] socket: 339 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.889758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789430066951384:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:10.889823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:10.889886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789430066951384:2107] [node 4] ICP22 created new session: [3:7577789430066951409:2107]
2025-11-28T14:53:10.889942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789430066951409:2107] [node 4] ICS09 handshake done sender: [3:7577789430066951393:2114] self: [3:7577789430066951392:0] peer: [4:7577789429435405624:0] socket: 361 qp: -1
2025-11-28T14:53:10.889984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789430066951409:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:10.890025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:10.937842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789430066951413:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57862
2025-11-28T14:53:10.937913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789430066951414:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57862
2025-11-28T14:53:10.992854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789430066951413:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.020541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789430066951321:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:11.032470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789430066951414:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.036554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789430066951384:2107] [node 4] ICP18 handshake (actor [3:7577789430066951414:2120]) is too old
2025-11-28T14:53:11.043525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789430066951395:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.043560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789430066951395:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:11.044231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789430066951404:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.044255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789430066951404:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.948489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789429435405626:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.924496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789429577295557:2110] [node 6] ICP22 created new session: [5:7577789429577295581:2110]
2025-11-28T14:53:10.920496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789430906186848:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
2025-11-28T14:53:10.944524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:10.944613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789430906186821:2096] [node 5] ICP22 created new session: [6:7577789430906186849:2096]
2025-11-28T14:53:10.944673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789430906186849:2096] [node 5] ICS09 handshake done sender: [6:7577789430906186833:2112] self: [6:7577789430906186842:0] peer: [5:7577789429577295560:0] socket: 338 qp: -1
2025-11-28T14:53:10.944732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789430906186849:2096] [node 5] ICS10 traffic start
2025-11-28T14:53:10.944786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:10.968813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789430906186834:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:10.970777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789430906186763:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:10.970867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:10.970930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789430906186763:2049] [node 2] ICP22 created new session: [6:7577789430906186851:2049]
2025-11-28T14:53:10.970983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789430906186851:2049] [node 2] ICS09 handshake done sender: [6:7577789430906186834:2113] self: [6:7577789430906186844:0] peer: [2:7577789430542738698:0] socket: 342 qp: -1
2025-11-28T14:53:10.971027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789430906186851:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:10.971064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:10.980539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789430906186846:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.984878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789430906186847:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.029921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789430906186762:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:11.029992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789430906186764:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:11.057057Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789430906186821:2096] [node 5] ICP35 event processing took too much time PendingConnection:279(40403502ns) {HandleHandshakeStatus:329(40401229ns) {DropIncomingHandshake:442(5309ns) DropOutgoingHandshake:457(22813ns) SwitchToState:179(953ns) ProcessPendingSessionEvents:495(620953ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 566304ns max# 63441ns}}}
2025-11-28T14:53:10.855983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:10.861983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789430542738714:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:10.934724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789429577295487:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(84473858ns) {HandleHandshakeStatus:329(84470684ns) {DropIncomingHandshake:442(1023ns) DropOutgoingHandshake:457(6314ns) SwitchToState:179(2074ns) ProcessPendingSessionEvents:495(848368ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 786090ns max# 125230ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:11.104656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789430066951404:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:11.104732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789430066951395:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:11.119117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789430066951321:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789430066951413:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:11.121176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789434361918714:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57862
2025-11-28T14:53:10.951470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789429577295581:2110] [node 6] ICS09 handshake done sender: [5:7577789429577295561:2113] self: [5:7577789429577295560:0] peer: [6:7577789430906186842:0] socket: 330 qp: -1
2025-11-28T14:53:10.955053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789429435405607:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:10.955117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:10.955187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789429435405607:2109] [node 5] ICP22 created new session: [4:7577789429435405631:2109]
2025-11-28T14:53:10.955259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789429435405631:2109] [node 5] ICS09 handshake done sender: [4:7577789429435405615:2121] self: [4:7577789429435405614:0] peer: [5:7577789429577295579:0] socket: 355 qp: -1
2025-11-28T14:53:10.955303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789429435405631:2109] [node 5] ICS10 traffic start
2025-11-28T14:53:10.955375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:10.962677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789429435405536:2049] [node 2] ICP22 created new session: [4:7577789429435405630:2049]
2025-11-28T14:53:10.985209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789429435405630:2049] [node 2] ICS09 handshake done sender: [4:7577789429435405613:2120] self: [4:7577789429435405622:0] peer: [2:7577789430542738694:0] socket: 343 qp: -1
2025-11-28T14:53:10.990683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789429435405627:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.991039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789429435405607:2109] [node 5] ICP18 handshake (actor [4:7577789429435405626:2127]) is too old
2025-11-28T14:53:10.991726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789429435405633:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30322
2025-11-28T14:53:10.991975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789429435405535:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:10.992795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789429435405630:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:11.002392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789429435405625:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30322
2025-11-28T14:53:11.019056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:11.048827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789429435405535:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30089) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789429435405627:2128] held: no
2025-11-28T14:53:11.049598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789429435405633:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.049992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789429435405621:2124] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:11.065336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789429435405536:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(157059441ns) {HandleHandshakeStatus:329(157056546ns) {DropIncomingHandshake:442(722ns) DropOutgoingHandshake:457(320ns) SwitchToState:179(611ns) ProcessPendingSessionEvents:495(2177ns)}}
2025-11-28T14:53:11.076137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789433730372932:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30322
2025-11-28T14:53:11.083518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789429435405610:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(241713946ns) {Configure:104(241706473ns) {ProcessConfigured:135(44211931ns) {StartInitialHandshake:157(44207716ns) {DropHandshakes:470(15104719ns) {DropIncomingHandshake:442(356ns) DropOutgoingHandshake:457(3196ns)} GenerateSessionVirtualId:988(3714ns) PrepareNewSessionHandshake:477(5322ns) {SwitchToState:179(2705ns)}}}}}
2025-11-28T14:53:11.083584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789429435405537:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:11.083710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:11.083771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789429435405537:2050] [node 3] ICP22 created new session: [4:7577789433730372935:2050]
2025-11-28T14:53:11.083848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789433730372935:2050] [node 3] ICS09 handshake done sender: [4:7577789429435405621:2124] self: [4:7577789429435405624:0] peer: [3:7577789430066951392:0] socket: 366 qp: -1
2025-11-28T14:53:11.083897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789433730372935:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:11.083940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:11.107414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789429435405627:2128] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:11.125913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789429435405535:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:11.125976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:11.126031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789429435405535:2048] [node 1] ICP22 created new session: [4:7577789433730372939:2048]
2025-11-28T14:53:11.126080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789433730372939:2048] [node 1] ICS09 handshake done sender: [4:7577789429435405627:2128] self: [4:7577789429435405634:0] peer: [1:7577789431916092255:0] socket: 380 qp: -1
2025-11-28T14:53:11.126118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789433730372939:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:11.126158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.956678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789429577295581:2110] [node 6] ICS10 traffic start
2025-11-28T14:53:10.862020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:11.123243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789435201154152:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
2025-11-28T14:53:10.856015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789431916092260:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.862496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789430542738704:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.862389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789431916092242:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:11.146194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789435201154154:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
2025-11-28T14:53:11.161694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789430906186847:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:11.161915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789435201154153:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
2025-11-28T14:53:10.974741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789429577295570:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:11.156647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789430066951385:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:11.156763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:11.156823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789430066951385:2109] [node 6] ICP22 created new session: [3:7577789434361918715:2109]
2025-11-28T14:53:11.156882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789434361918715:2109] [node 6] ICS09 handshake done sender: [3:7577789430066951395:2115] self: [3:7577789430066951394:0] peer: [6:7577789435201154150:0] socket: 360 qp: -1
2025-11-28T14:53:11.156930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789434361918715:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:11.156973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:11.158140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789430066951413:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:11.158551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789430066951383:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:11.176731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789430066951321:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:11.176825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:11.176887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789430066951321:2049] [node 2] ICP22 created new session: [3:7577789434361918717:2049]
2025-11-28T14:53:11.176937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789434361918717:2049] [node 2] ICS09 handshake done sender: [3:7577789430066951413:2119] self: [3:7577789434361918711:0] peer: [2:7577789430542738703:0] socket: 390 qp: -1
2025-11-28T14:53:11.176976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789434361918717:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:11.177037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:10.864439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789427621124882:2049] [node 3] ICP22 created new session: [1:7577789431916092263:2049]
2025-11-28T14:53:10.868773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:10.872838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789431916092263:2049] [node 3] ICS09 handshake done sender: [1:7577789431916092248:2113] self: [1:7577789431916092247:0] peer: [3:7577789430066951400:0] socket: 335 qp: -1
2025-11-28T14:53:10.872898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789431916092263:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:10.872957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:10.874657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789427621124882:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(21305152ns) {HandleHandshakeStatus:329(21302237ns) {DropIncomingHandshake:442(698ns) DropOutgoingHandshake:457(486ns) SwitchToState:179(714ns) ProcessPendingSessionEvents:495(598299ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 526975ns max# 64920ns}}}
2025-11-28T14:53:10.874804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789427621124882:2049] [node 3] ICP18 handshake (actor [1:7577789431916092260:2117]) is too old
2025-11-28T14:53:10.875733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789431916092242:2108] [node 5] ICP22 created new session: [1:7577789431916092264:2108]
2025-11-28T14:53:10.904756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789431916092264:2108] [node 5] ICS09 handshake done sender: [1:7577789431916092246:2112] self: [1:7577789431916092245:0] peer: [5:7577789429577295569:0] socket: 334 qp: -1
2025-11-28T14:53:10.912634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789431916092264:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:10.920958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:10.939910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789431916092269:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35189
2025-11-28T14:53:10.951557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789431916092253:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.885076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:11.181166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:11.181247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789430066951383:2108] [node 5] ICP22 created new session: [3:7577789434361918719:2108]
2025-11-28T14:53:10.975391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:10.904594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789430542738692:2108] [node 5] ICP22 created new session: [2:7577789430542738718:2108]
2025-11-28T14:53:11.181309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789434361918719:2108] [node 5] ICS09 handshake done sender: [3:7577789430066951404:2118] self: [3:7577789430066951403:0] peer: [5:7577789433872262881:0] socket: 363 qp: -1
2025-11-28T14:53:11.181351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789434361918719:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:11.181395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:10.951600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789431916092253:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:11.188884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789435201154154:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.201828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789433730372934:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.987668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789429577295582:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.909039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789430542738699:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:10.909059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789430542738699:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:10.909446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789430542738627:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:10.912618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789430542738718:2108] [node 5] ICS09 handshake done sender: [2:7577789430542738697:2112] self: [2:7577789430542738696:0] peer: [5:7577789429577295568:0] socket: 336 qp: -1
2025-11-28T14:53:10.928206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789430542738699:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:10.928278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789430542738718:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:10.928332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:10.972620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789430542738691:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:10.972708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:10.972763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789430542738691:2109] [node 6] ICP22 created new session: [2:7577789430542738724:2109]
2025-11-28T14:53:10.972817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789430542738724:2109] [node 6] ICS09 handshake done sender: [2:7577789430542738699:2113] self: [2:7577789430542738698:0] peer: [6:7577789430906186844:0] socket: 337 qp: -1
2025-11-28T14:53:10.972860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789430542738724:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:10.972912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:10.982253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789430542738692:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(124319790ns) {HandleHandshakeStatus:329(124316148ns) {DropIncomingHandshake:442(901ns) DropOutgoingHandshake:457(680ns) SwitchToState:179(574ns) ProcessPendingSessionEvents:495(582528ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 518516ns max# 66867ns}}}
2025-11-28T14:53:10.982396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789430542738723:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12342
2025-11-28T14:53:11.036119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789430542738705:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.036166Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789430542738705:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:11.050882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789430542738705:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:11.051120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789430542738628:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:11.051211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:11.051263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789430542738628:2049] [node 3] ICP22 created new session: [2:7577789434837706023:2049]
2025-11-28T14:53:11.051316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789434837706023:2049] [node 3] ICS09 handshake done sender: [2:7577789430542738705:2115] self: [2:7577789430542738703:0] peer: [3:7577789434361918711:0] socket: 382 qp: -1
2025-11-28T14:53:11.051354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789434837706023:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:11.051393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:11.097727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789430542738690:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(234987251ns) {HandleHandshakeStatus:329(234984327ns) {DropIncomingHandshake:442(13964ns) DropOutgoingHandshake:457(452ns) SwitchToState:179(377ns) ProcessPendingSessionEvents:495(234552629ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 211999355ns max# 31670544ns}}}
2025-11-28T14:53:11.098782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789434837706025:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12342
2025-11-28T14:53:11.109666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789430542738704:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:11.139029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789430542738627:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:11.139119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:11.139176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789430542738627:2048] [node 1] ICP22 created new session: [2:7577789434837706026:2048]
2025-11-28T14:53:11.139239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789434837706026:2048] [node 1] ICS09 handshake done sender: [2:7577789430542738704:2114] self: [2:7577789430542738720:0] peer: [1:7577789431916092252:0] socket: 351 qp: -1
2025-11-28T14:53:11.139281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789434837706026:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:11.139319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:11.008828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789431916092256:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.201873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789433730372934:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:11.203562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789433730372934:2131] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:11.203808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789429435405610:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:11.203869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:11.203928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789429435405610:2108] [node 6] ICP22 created new session: [4:7577789433730372942:2108]
2025-11-28T14:53:11.203979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789433730372942:2108] [node 6] ICS09 handshake done sender: [4:7577789433730372934:2131] self: [4:7577789433730372933:0] peer: [6:7577789435201154159:0] socket: 324 qp: -1
2025-11-28T14:53:11.204014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789433730372942:2108] [node 6] ICS10 traffic start
2025-11-28T14:53:11.204051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:11.018943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789429577295489:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:11.188990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789430906186764:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1762) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789430906186847:2119] held: no
2025-11-28T14:53:11.008860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789431916092256:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:11.009784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789431916092256:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:11.010008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789431916092240:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:11.010103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:11.010160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789431916092240:2107] [node 4] ICP22 created new session: [1:7577789436211059567:2107]
2025-11-28T14:53:11.010208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789436211059567:2107] [node 4] ICS09 handshake done sender: [1:7577789431916092256:2116] self: [1:7577789431916092255:0] peer: [4:7577789429435405634:0] socket: 331 qp: -1
2025-11-28T14:53:11.010248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789436211059567:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:11.010300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:11.020605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789431916092269:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.060776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789431916092254:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.060818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789431916092254:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:11.080678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789431916092253:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:11.099445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789431916092254:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:11.110402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789427621124881:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:11.120589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789436211059570:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35189
2025-11-28T14:53:11.132691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789431916092241:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:11.132778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:11.132840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789431916092241:2109] [node 6] ICP22 created new session: [1:7577789436211059573:2109]
2025-11-28T14:53:11.132901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789436211059573:2109] [node 6] ICS09 handshake done sender: [1:7577789431916092254:2115] self: [1:7577789431916092251:0] peer: [6:7577789435201154149:0] socket: 377 qp: -1
2025-11-28T14:53:11.132944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789436211059573:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:11.132998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:11.146807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789436211059570:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:11.159736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:11.163654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789427621124881:2048] [node 2] ICP22 created new session: [1:7577789436211059575:2048]
2025-11-28T14:53:11.172588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789436211059575:2048] [node 2] ICS09 handshake done sender: [1:7577789431916092253:2114] self: [1:7577789431916092252:0] peer: [2:7577789430542738720:0] socket: 350 qp: -1
2025-11-28T14:53:11.178837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789431916092241:2109] [node 6] ICP18 handshake (actor [1:7577789436211059570:2119]) is too old
2025-11-28T14:53:11.184495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789436211059575:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:11.184609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:11.217652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789427621124881:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(106341520ns) {HandleHandshakeStatus:329(106338063ns) {DropIncomingHandshake:442(18826ns) DropOutgoingHandshake:457(689ns) SwitchToState:179(1308ns) ProcessPendingSessionEvents:495(31983865ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31577747ns max# 10717311ns}}}
2025-11-28T14:53:11.189612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789430906186764:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:11.189693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:11.189747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789430906186764:2050] [node 3] ICP22 created new session: [6:7577789435201154155:2050]
2025-11-28T14:53:11.189802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789435201154155:2050] [node 3] ICS09 handshake done sender: [6:7577789430906186847:2119] self: [6:7577789435201154150:0] peer: [3:7577789430066951394:0] socket: 388 qp: -1
2025-11-28T14:53:11.189841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789435201154155:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:11.189880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:11.192354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789430906186846:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:11.193118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789430906186762:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:11.193186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:11.193234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789430906186762:2048] [node 1] ICP22 created new session: [6:7577789435201154157:2048]
2025-11-28T14:53:11.033387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789429577295489:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(26000880ns) {IncomingHandshake:261(25999167ns) {DropIncomingHandshake:442(1930ns) PrepareNewSessionHandshake:477(3723ns) {SwitchToState:179(2371ns)} GenerateSessionVirtualId:988(1033ns) IssueIncomingHandshakeReply:193(64570ns) {SwitchToState:179(296ns)}}}
2025-11-28T14:53:11.193282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789435201154157:2048] [node 1] ICS09 handshake done sender: [6:7577789430906186846:2118] self: [6:7577789435201154149:0] peer: [1:7577789431916092251:0] socket: 381 qp: -1
2025-11-28T14:53:11.193320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789435201154157:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:11.033479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789429577295556:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:11.193357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:11.038789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:11.194365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789430906186823:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:11.059422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789429577295557:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(195946360ns) {HandleHandshakeStatus:329(195942932ns) {DropIncomingHandshake:442(1265ns) DropOutgoingHandshake:457(1051ns) SwitchToState:179(772ns) ProcessPendingSessionEvents:495(19520569ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19445962ns max# 19014979ns}}}
2025-11-28T14:53:11.213414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789435201154160:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23557
2025-11-28T14:53:11.059534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789429577295557:2110] [node 6] ICP18 handshake (actor [5:7577789429577295565:2115]) is too old
2025-11-28T14:53:11.213904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789435201154154:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:11.060761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789429577295556:2108] [node 4] ICP22 created new session: [5:7577789433872262882:2108]
2025-11-28T14:53:11.214127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789430906186823:2101] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:11.093515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789433872262882:2108] [node 4] ICS09 handshake done sender: [5:7577789429577295570:2118] self: [5:7577789429577295579:0] peer: [4:7577789429435405614:0] socket: 357 qp: -1
2025-11-28T14:53:11.099621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789433872262882:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:11.214201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:11.099664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:11.214553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789430906186823:2101] [node 4] ICP22 created new session: [6:7577789435201154161:2101]
2025-11-28T14:53:11.109134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789433872262883:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43351
2025-11-28T14:53:11.228905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789435201154161:2101] [node 4] ICS09 handshake done sender: [6:7577789435201154154:2123] self: [6:7577789435201154159:0] peer: [4:7577789433730372933:0] socket: 367 qp: -1
2025-11-28T14:53:11.149578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789429577295582:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:11.240489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789435201154161:2101] [node 4] ICS10 traffic start
2025-11-28T14:53:11.149801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789429577295489:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:11.240572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:11.149885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:11.254359Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789430906186823:2101] [node 4] ICP35 event processing took too much time PendingConnection:279(39298770ns) {HandleHandshakeStatus:329(39293881ns) {DropIncomingHandshake:442(1302ns) DropOutgoingHandshake:457(9885ns) SwitchToState:179(621ns) ProcessPendingSessionEvents:495(574201ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 514404ns max# 57350ns}}}
2025-11-28T14:53:11.149946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789429577295489:2050] [node 3] ICP22 created new session: [5:7577789433872262885:2050]
2025-11-28T14:53:11.149997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789433872262885:2050] [node 3] ICS09 handshake done sender: [5:7577789429577295582:2121] self: [5:7577789433872262881:0] peer: [3:7577789430066951403:0] socket: 386 qp: -1
2025-11-28T14:53:11.150038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789433872262885:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:11.150080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:11.251584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789429577295556:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(217078157ns) {HandleHandshakeStatus:329(217075304ns) {DropIncomingHandshake:442(8235ns) DropOutgoingHandshake:457(26936ns) SwitchToState:179(968ns) ProcessPendingSessionEvents:495(103885153ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 103754864ns max# 11207165ns}}}
2025-11-28T14:53:11.251674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789429577295556:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:11.257909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789431916092242:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(399596055ns) {HandleHandshakeStatus:329(399592538ns) {DropIncomingHandshake:442(18980ns) DropOutgoingHandshake:457(303ns) SwitchToState:179(893ns) ProcessPendingSessionEvents:495(298700015ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 298453568ns max# 27584215ns}}}
2025-11-28T14:53:11.258784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789429435405537:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(174333830ns) {HandleHandshakeStatus:329(174330601ns) {DropIncomingHandshake:442(1699ns) DropOutgoingHandshake:457(5565ns) SwitchToState:179(314ns) ProcessPendingSessionEvents:495(173698661ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 164592758ns max# 15844940ns}}}
2025-11-28T14:53:11.258961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789429435405537:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:11.262322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789429435405610:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(52404762ns) {HandleHandshakeStatus:329(52402432ns) {DropIncomingHandshake:442(14661ns) DropOutgoingHandshake:457(515ns) SwitchToState:179(666ns) ProcessPendingSessionEvents:495(52057479ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 51962339ns max# 11235258ns}}}
2025-11-28T14:53:11.264154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789430066951383:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(107653728ns) {HandleHandshakeStatus:329(107650655ns) {DropIncomingHandshake:442(22355ns) DropOutgoingHandshake:457(483ns) SwitchToState:179(681ns) ProcessPendingSessionEvents:495(78999881ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 78827922ns max# 23036023ns}}}
2025-11-28T14:53:11.334833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:11.335079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789434837706026:2048] [node 1] ICS01 socket: 351 reason# EndOfStream
2025-11-28T14:53:11.335121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789430542738627:2048] [node 1] ICP30 unregister session Session# [2:7577789434837706026:2048] VirtualId# [2:7577789430542738720:0]
2025-11-28T14:53:11.335157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789434837706026:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.335250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:11.335304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:11.340553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:11.340817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789430066951407:2048] [node 1] ICS07 socket disconnect 340 reason# EndOfStream
2025-11-28T14:53:11.340850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789430066951407:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.340957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:11.341001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789430066951407:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:11.342072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789430066951320:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30089) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:11.342129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789430066951407:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:11.342165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789430066951320:2048] [node 1] ICP30 unregister session Session# [3:7577789430066951407:2048] VirtualId# [3:7577789430066951400:0]
2025-11-28T14:53:11.342205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:11.342239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789430066951320:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.342254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.349699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:11.350025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789429577295574:2048] [node 1] ICS01 socket: 346 reason# EndOfStream
2025-11-28T14:53:11.348556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:11.350119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789429577295487:2048] [node 1] ICP30 unregister session Session# [5:7577789429577295574:2048] VirtualId# [5:7577789429577295569:0]
2025-11-28T14:53:11.350173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789429577295574:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.349293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:11.350323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:11.349608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789435201154157:2048] [node 1] ICS07 socket disconnect 381 reason# EndOfStream
2025-11-28T14:53:11.350423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:11.349659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789435201154157:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.349775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:11.348811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789433730372939:2048] [node 1] ICS01 socket: 380 reason# EndOfStream
2025-11-28T14:53:11.349855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789435201154157:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:11.348858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789429435405535:2048] [node 1] ICP30 unregister session Session# [4:7577789433730372939:2048] VirtualId# [4:7577789429435405634:0]
2025-11-28T14:53:11.348894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789433730372939:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.349075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:11.349182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:11.356750Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789430906186762:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30089) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:11.360464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789430066951320:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:11.356830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789435201154157:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:11.356869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789430906186762:2048] [node 1] ICP30 unregister session Session# [6:7577789435201154157:2048] VirtualId# [6:7577789435201154149:0]
2025-11-28T14:53:11.356928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:11.356965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789430906186762:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.356984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30089) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.392922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789430906186762:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:11.464634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:11.464878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789434361918717:2049] [node 2] ICS01 socket: 390 reason# EndOfStream
2025-11-28T14:53:11.464926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789430066951321:2049] [node 2] ICP30 unregister session Session# [3:7577789434361918717:2049] VirtualId# [3:7577789434361918711:0]
2025-11-28T14:53:11.464962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789434361918717:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.465078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:11.465122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:11.466039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:11.466225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789429435405630:2049] [node 2] ICS01 socket: 343 reason# EndOfStream
2025-11-28T14:53:11.466263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789429435405536:2049] [node 2] ICP30 unregister session Session# [4:7577789429435405630:2049] VirtualId# [4:7577789429435405622:0]
2025-11-28T14:53:11.466289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789429435405630:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.465688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:11.466359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:11.465878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789430906186851:2049] [node 2] ICS07 socket disconnect 342 reason# EndOfStream
2025-11-28T14:53:11.466398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:11.465905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789430906186851:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.465971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:11.466002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789430906186851:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:11.467062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789430906186763:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13872) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:11.467114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789430906186851:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:11.467151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789430906186763:2049] [node 2] ICP30 unregister session Session# [6:7577789430906186851:2049] VirtualId# [6:7577789430906186844:0]
2025-11-28T14:53:11.467250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:11.467284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789430906186763:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.467310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.468469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:11.468680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789429577295577:2049] [node 2] ICS01 socket: 347 reason# EndOfStream
2025-11-28T14:53:11.468786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789429577295488:2049] [node 2] ICP30 unregister session Session# [5:7577789429577295577:2049] VirtualId# [5:7577789429577295568:0]
2025-11-28T14:53:11.468822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789429577295577:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.468887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:11.468931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:11.474380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789430906186763:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:11.580768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:11.580594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:11.580952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789433872262885:2050] [node 3] ICS01 socket: 386 reason# EndOfStream
2025-11-28T14:53:11.580998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789429577295489:2050] [node 3] ICP30 unregister session Session# [5:7577789433872262885:2050] VirtualId# [5:7577789433872262881:0]
2025-11-28T14:53:11.581029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789433872262885:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.581102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:11.581145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:11.581524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789433730372935:2050] [node 3] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-28T14:53:11.581555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789433730372935:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.581609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:11.581654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789433730372935:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:11.582499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789429435405537:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1762) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:11.582551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789433730372935:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:11.582583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789429435405537:2050] [node 3] ICP30 unregister session Session# [4:7577789433730372935:2050] VirtualId# [4:7577789429435405624:0]
2025-11-28T14:53:11.582679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:11.582707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789429435405537:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.582723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.580686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:11.580905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789435201154155:2050] [node 3] ICS01 socket: 388 reason# EndOfStream
2025-11-28T14:53:11.580951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789430906186764:2050] [node 3] ICP30 unregister session Session# [6:7577789435201154155:2050] VirtualId# [6:7577789435201154150:0]
2025-11-28T14:53:11.580984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789435201154155:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.581066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:11.581112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:11.637010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:11.637237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789435201154161:2101] [node 4] ICS01 socket: 367 reason# EndOfStream
2025-11-28T14:53:11.637298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789430906186823:2101] [node 4] ICP30 unregister session Session# [6:7577789435201154161:2101] VirtualId# [6:7577789435201154159:0]
2025-11-28T14:53:11.637335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789435201154161:2101] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.637427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:11.637466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:11.641548Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:11.641805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789433872262882:2108] [node 4] ICS07 socket disconnect 357 reason# EndOfStream
2025-11-28T14:53:11.641843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789433872262882:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:11.641947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:11.641990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789433872262882:2108] [node 4] ICS15 start handshake
2025-11-28T14:53:11.644483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789429577295556:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29302) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:11.648611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789433872262882:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:11.648674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789429577295556:2108] [node 4] ICP30 unregister session Session# [5:7577789433872262882:2108] VirtualId# [5:7577789429577295579:0]
2025-11-28T14:53:11.648797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:11.648835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789429577295556:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29302) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.648854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29302) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:11.665915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789429577295556:2108] [node 4] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:11.892539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789436382428447:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:11.892590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789436382428448:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:11.917589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789434049958697:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:11.917641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789434049958698:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:11.948818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789432946739020:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:11.948863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789432946739021:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:11.974746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789434961414200:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:11.974792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789434961414201:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:11.974824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789434961414202:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:12.000962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789432323831625:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:12.001002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789432323831626:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:12.001033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789432323831627:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:12.343743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789436618798943:2068] [node 6] ICP01 ready to work
2025-11-28T14:53:12.184502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789437933093410:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:12.345125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789440677395806:2109] [node 4] ICP01 ready to work
2025-11-28T14:53:12.364431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789439256381552:2100] [node 6] ICP01 ready to work
2025-11-28T14:53:12.184641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789437933093411:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:12.510146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789436618798943:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(166032352ns) {Configure:104(166025785ns) {ProcessConfigured:135(51210856ns) {StartInitialHandshake:157(51207179ns) {DropHandshakes:470(3228ns) {DropIncomingHandshake:442(867ns) DropOutgoingHandshake:457(627ns)} GenerateSessionVirtualId:988(1421ns) PrepareNewSessionHandshake:477(5123ns) {SwitchToState:179(2990ns)}}}}}
2025-11-28T14:53:12.460636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789437241706379:2107] [node 5] ICP01 ready to work
2025-11-28T14:53:12.504412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789440677395807:2107] [node 5] ICP01 ready to work
2025-11-28T14:53:12.540582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789436618798969:2073] [node 4] ICP01 ready to work
2025-11-28T14:53:12.204571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789437933093412:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:12.460419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789438344926056:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:12.515273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789437241706380:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.588797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789434961414200:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27871654ns) {Configure:104(27865258ns) {ProcessConfigured:135(74065ns) {StartInitialHandshake:157(70870ns) {DropHandshakes:470(2383ns) {DropIncomingHandshake:442(705ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(814ns) PrepareNewSessionHandshake:477(1984ns) {SwitchToState:179(628ns)}}}}}
2025-11-28T14:53:12.589218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789439256381560:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48201
2025-11-28T14:53:12.500550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789438344926057:2109] [node 5] ICP01 ready to work
2025-11-28T14:53:12.520914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789437933093444:2074] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.552695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789440677395808:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.504439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789438344926058:2107] [node 6] ICP01 ready to work
2025-11-28T14:53:12.540059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789437241706381:2108] [node 4] ICP01 ready to work
2025-11-28T14:53:12.557872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789437933093410:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(228265088ns) {Configure:104(228260045ns) {ProcessConfigured:135(17578693ns) {StartInitialHandshake:157(17574859ns) {DropHandshakes:470(2707ns) {DropIncomingHandshake:442(659ns) DropOutgoingHandshake:457(529ns)} GenerateSessionVirtualId:988(767ns) PrepareNewSessionHandshake:477(4957ns) {SwitchToState:179(2961ns)}}}}}
2025-11-28T14:53:12.654604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789434049958698:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(124152371ns) {Configure:104(124144559ns) {ProcessConfigured:135(67766ns) {StartInitialHandshake:157(65221ns) {DropHandshakes:470(2362ns) {DropIncomingHandshake:442(581ns) DropOutgoingHandshake:457(441ns)} GenerateSessionVirtualId:988(1007ns) PrepareNewSessionHandshake:477(1590ns) {SwitchToState:179(659ns)}}}}}
2025-11-28T14:53:12.604460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789439256381561:2104] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.619185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789437241706382:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62540
2025-11-28T14:53:12.606658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789437933093411:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(143692115ns) {Configure:104(143680677ns) {ProcessConfigured:135(80770ns) {StartInitialHandshake:157(76986ns) {DropHandshakes:470(2216ns) {DropIncomingHandshake:442(586ns) DropOutgoingHandshake:457(446ns)} GenerateSessionVirtualId:988(882ns) PrepareNewSessionHandshake:477(1437ns) {SwitchToState:179(579ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.637091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789439256381569:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48201
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.683402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789440677395811:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.661550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789434961414201:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(118698572ns) {Configure:104(118692984ns) {ProcessConfigured:135(29677397ns) {StartInitialHandshake:157(29673756ns) {DropHandshakes:470(2468ns) {DropIncomingHandshake:442(920ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:477(3815ns) {SwitchToState:179(1500ns)}}}}}
2025-11-28T14:53:12.664691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789437933093483:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.700431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789437241706382:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.694029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789434049958697:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(189306833ns) {Configure:104(177378921ns) {ProcessConfigured:135(82634ns) {StartInitialHandshake:157(78764ns) {DropHandshakes:470(2775ns) {DropIncomingHandshake:442(1111ns) DropOutgoingHandshake:457(395ns)} GenerateSessionVirtualId:988(874ns) PrepareNewSessionHandshake:477(1743ns) {SwitchToState:179(803ns)}}}}}
2025-11-28T14:53:12.725963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789436618798992:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25921
2025-11-28T14:53:12.683481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789440677395812:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
2025-11-28T14:53:12.688456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789437933093483:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.696889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789439256381569:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.696662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789440677395812:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.704175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789432946739020:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(338802601ns) {Configure:104(338791872ns) {ProcessConfigured:135(64548ns) {StartInitialHandshake:157(60800ns) {DropHandshakes:470(2479ns) {DropIncomingHandshake:442(637ns) DropOutgoingHandshake:457(338ns)} GenerateSessionVirtualId:988(1099ns) PrepareNewSessionHandshake:477(1602ns) {SwitchToState:179(569ns)}}}}}
2025-11-28T14:53:12.750883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789438344926057:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18038947ns) {Configure:104(18031226ns) {ProcessConfigured:135(71470ns) {StartInitialHandshake:157(68156ns) {DropHandshakes:470(2273ns) {DropIncomingHandshake:442(727ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(1304ns) PrepareNewSessionHandshake:477(1995ns) {SwitchToState:179(602ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.709283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789437933093485:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.731208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789439256381575:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48201
2025-11-28T14:53:12.836430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789436618798992:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.836990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789432323831626:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(354849721ns) {Configure:104(354841701ns) {ProcessConfigured:135(84122ns) {StartInitialHandshake:157(81191ns) {DropHandshakes:470(8584ns) {DropIncomingHandshake:442(911ns) DropOutgoingHandshake:457(299ns)} GenerateSessionVirtualId:988(872ns) PrepareNewSessionHandshake:477(1641ns) {SwitchToState:179(692ns)}}}}}
2025-11-28T14:53:12.801186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789438344926075:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5636
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.717174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789440677395806:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(371710364ns) {Configure:104(371707197ns) {ProcessConfigured:135(39872039ns) {StartInitialHandshake:157(39869886ns) {DropHandshakes:470(1582ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(612ns) PrepareNewSessionHandshake:477(2501ns) {SwitchToState:179(130ns)}}}}}
2025-11-28T14:53:12.728642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789440677395811:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.770307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789440677395824:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
2025-11-28T14:53:12.810188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789436382428448:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(165442279ns) {Configure:104(165434636ns) {ProcessConfigured:135(80747ns) {StartInitialHandshake:157(77448ns) {DropHandshakes:470(2511ns) {DropIncomingHandshake:442(481ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(794ns) PrepareNewSessionHandshake:477(1453ns) {SwitchToState:179(697ns)}}}}}
2025-11-28T14:53:12.816165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789436382428447:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(356034344ns) {Configure:104(356025292ns) {ProcessConfigured:135(59656005ns) {StartInitialHandshake:157(59653099ns) {DropHandshakes:470(2247ns) {DropIncomingHandshake:442(672ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:477(4987ns) {SwitchToState:179(2458ns)}}}}}
2025-11-28T14:53:12.820695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789440677395825:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
2025-11-28T14:53:12.832591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789440677395824:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.715817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789437933093484:2086] [node 5] ICP01 ready to work
2025-11-28T14:53:12.756769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789439256381552:2100] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(170286714ns) {Configure:104(170280100ns) {ProcessConfigured:135(69714006ns) {StartInitialHandshake:157(69710301ns) {DropHandshakes:470(1809ns) {DropIncomingHandshake:442(544ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(980ns) PrepareNewSessionHandshake:477(5765ns) {SwitchToState:179(2706ns)}}}}}
2025-11-28T14:53:12.868683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789440677395825:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.869199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789437241706381:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(192733857ns) {Configure:104(192729077ns) {ProcessConfigured:135(71883ns) {StartInitialHandshake:157(69603ns) {DropHandshakes:470(3419ns) {DropIncomingHandshake:442(648ns) DropOutgoingHandshake:457(568ns)} GenerateSessionVirtualId:988(946ns) PrepareNewSessionHandshake:477(1649ns) {SwitchToState:179(966ns)}}}}}
2025-11-28T14:53:12.804409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789438344926067:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5636
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.836419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789437933093412:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(486386007ns) {Configure:104(486380207ns) {ProcessConfigured:135(12434907ns) {StartInitialHandshake:157(12431579ns) {DropHandshakes:470(14622ns) {DropIncomingHandshake:442(12407ns) DropOutgoingHandshake:457(394ns)} GenerateSessionVirtualId:988(1133ns) PrepareNewSessionHandshake:477(3738ns) {SwitchToState:179(1725ns)}}}}}
2025-11-28T14:53:12.772953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789439256381560:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.896718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789437241706391:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62540
2025-11-28T14:53:12.903609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789437241706391:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.903804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789432946739021:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:12.928798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789437241706387:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.848702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789438344926067:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.840726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789437933093487:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.795357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789439256381561:2104] [node 5] ICP35 event processing took too much time PendingConnection:279(27993983ns) {IncomingHandshake:261(51338ns) {DropIncomingHandshake:442(468ns) PrepareNewSessionHandshake:477(2855ns) {SwitchToState:179(2098ns)} GenerateSessionVirtualId:988(959ns) IssueIncomingHandshakeReply:193(7390ns)}}
2025-11-28T14:53:12.928856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789437241706387:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:12.852984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789438344926068:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5636
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.916815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789432323831627:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(356323873ns) {Configure:104(356315691ns) {ProcessConfigured:135(19997626ns) {StartInitialHandshake:157(19994248ns) {DropHandshakes:470(2395ns) {DropIncomingHandshake:442(428ns) DropOutgoingHandshake:457(309ns)} GenerateSessionVirtualId:988(845ns) PrepareNewSessionHandshake:477(22404ns) {SwitchToState:179(16166ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.930864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789440677395817:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.912540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789437933093487:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.930906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789440677395817:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:12.960590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789440677395817:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:12.962056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789440677395806:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:12.962161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:12.962217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789440677395806:2109] [node 4] ICP22 created new session: [1:7577789440677395831:2109]
2025-11-28T14:53:12.962268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789440677395831:2109] [node 4] ICS09 handshake done sender: [1:7577789440677395817:2114] self: [1:7577789440677395810:0] peer: [4:7577789439256381579:0] socket: 359 qp: -1
2025-11-28T14:53:12.962308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789440677395831:2109] [node 4] ICS10 traffic start
2025-11-28T14:53:12.962366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:12.988415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789440677395816:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.988473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789440677395816:2113] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.929824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789439256381575:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.929986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789434961414200:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:12.944969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789439256381580:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48201
2025-11-28T14:53:12.961760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789439256381575:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:12.962705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789434961414200:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:12.962772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:12.962824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789434961414200:2048] [node 1] ICP22 created new session: [4:7577789439256381581:2048]
2025-11-28T14:53:12.962872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789439256381581:2048] [node 1] ICS09 handshake done sender: [4:7577789439256381575:2116] self: [4:7577789439256381579:0] peer: [1:7577789440677395810:0] socket: 362 qp: -1
2025-11-28T14:53:12.962910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789439256381581:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:12.962945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:12.992766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789439256381584:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48201
2025-11-28T14:53:12.917189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789432323831627:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:12.932305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789437241706387:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:12.856695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789438344926068:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.912727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789437933093411:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:12.940820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789436618799001:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25921
2025-11-28T14:53:13.040725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789440677395816:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:12.864494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789438344926069:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5636
2025-11-28T14:53:12.888178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789438344926077:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5636
2025-11-28T14:53:12.896446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789438344926069:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.904179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789438344926060:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.904210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789438344926060:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:12.928467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789438344926077:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.932710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789438344926075:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.941495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789434049958697:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:12.948843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789438344926060:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:12.949044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789438344926065:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:12.949094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789438344926065:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.058931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789439256381584:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.059119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789434961414202:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.065202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789442639893380:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5636
2025-11-28T14:53:12.961210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789437241706398:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62540
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.957350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789437933093485:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.041075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789436382428447:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.041151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:13.041216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789436382428447:2048] [node 2] ICP22 created new session: [1:7577789444972363131:2048]
2025-11-28T14:53:13.041268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789444972363131:2048] [node 2] ICS09 handshake done sender: [1:7577789440677395816:2113] self: [1:7577789440677395809:0] peer: [2:7577789438344926080:0] socket: 358 qp: -1
2025-11-28T14:53:13.041351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789444972363131:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:13.041388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:13.072415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789440677395814:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.072481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789440677395814:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.078373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789440677395806:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(115208974ns) {HandleHandshakeStatus:329(115206124ns) {DropIncomingHandshake:442(24831ns) DropOutgoingHandshake:457(393ns) SwitchToState:179(445ns) ProcessPendingSessionEvents:495(114752405ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 97537993ns max# 26620975ns}}}
2025-11-28T14:53:13.088607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789440677395819:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.088651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789440677395819:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.090180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789440677395819:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:13.090285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789440677395814:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:12.956710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789436618798992:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:13.000429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789437241706379:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.020478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789437241706391:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:13.057624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789432946739021:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.057712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:13.057769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789432946739021:2049] [node 2] ICP22 created new session: [3:7577789441536673696:2049]
2025-11-28T14:53:13.057851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789441536673696:2049] [node 2] ICS09 handshake done sender: [3:7577789437241706391:2114] self: [3:7577789437241706396:0] peer: [2:7577789438344926059:0] socket: 349 qp: -1
2025-11-28T14:53:13.057884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789441536673696:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:13.057948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:13.068608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:13.081428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789437241706394:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.081460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789437241706394:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.083578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789437241706394:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:13.092574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789437241706379:2107] [node 5] ICP22 created new session: [3:7577789441536673698:2107]
2025-11-28T14:53:13.098207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789441536673700:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62540
2025-11-28T14:53:13.117362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789441536673700:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.117430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789437241706381:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.117505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:13.117564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789437241706381:2108] [node 4] ICP22 created new session: [3:7577789441536673701:2108]
2025-11-28T14:53:13.117616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789441536673701:2108] [node 4] ICS09 handshake done sender: [3:7577789437241706394:2115] self: [3:7577789437241706393:0] peer: [4:7577789443551348881:0] socket: 378 qp: -1
2025-11-28T14:53:13.117652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789441536673701:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:13.117707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:13.068637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789438344926065:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:12.976716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789436618799002:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25921
2025-11-28T14:53:12.985306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789437933093410:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:12.984658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789432323831625:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(315952062ns) {Configure:104(315945187ns) {ProcessConfigured:135(7924985ns) {StartInitialHandshake:157(7922035ns) {DropHandshakes:470(2238ns) {DropIncomingHandshake:442(657ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(1078ns) PrepareNewSessionHandshake:477(2942ns) {SwitchToState:179(1444ns)}}}}}
2025-11-28T14:53:13.024461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789432323831627:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.032676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789436618799002:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.048959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789432323831625:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:13.064822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789440913766299:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25921
2025-11-28T14:53:13.104479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:13.104558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789432323831627:2050] [node 3] ICP22 created new session: [5:7577789440913766302:2050]
2025-11-28T14:53:13.124949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789440913766302:2050] [node 3] ICS09 handshake done sender: [5:7577789436618798992:2112] self: [5:7577789436618798999:0] peer: [3:7577789437241706386:0] socket: 361 qp: -1
2025-11-28T14:53:13.132612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789440913766302:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.105188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789440677395808:2108] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.029109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789442228060787:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.124638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789434961414202:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19700) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789439256381584:2119] held: no
2025-11-28T14:53:13.111812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789440677395807:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.111862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:13.140491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789440677395807:2107] [node 5] ICP22 created new session: [1:7577789444972363136:2107]
2025-11-28T14:53:13.143079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789444972363137:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
2025-11-28T14:53:13.150129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789440677395827:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.150154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789440677395827:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.162643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789444972363136:2107] [node 5] ICS09 handshake done sender: [1:7577789440677395819:2115] self: [1:7577789440677395818:0] peer: [5:7577789440913766300:0] socket: 380 qp: -1
2025-11-28T14:53:13.162704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789444972363136:2107] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.132428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789441536673698:2107] [node 5] ICS09 handshake done sender: [3:7577789437241706387:2112] self: [3:7577789437241706386:0] peer: [5:7577789436618798999:0] socket: 360 qp: -1
2025-11-28T14:53:13.139920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789440913766304:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25921
2025-11-28T14:53:13.090518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789442228060788:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.172736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:13.104473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789434049958697:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789438344926075:2118] held: no
2025-11-28T14:53:13.140448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789441536673698:2107] [node 5] ICS10 traffic start
2025-11-28T14:53:13.148452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:13.156451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789432946739020:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:13.160132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789441536673705:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62540
2025-11-28T14:53:13.172025Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789432946739020:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27969839ns) {IncomingHandshake:261(15952424ns) {DropIncomingHandshake:442(295ns) PrepareNewSessionHandshake:477(1764ns) {SwitchToState:179(931ns)} GenerateSessionVirtualId:988(445ns) IssueIncomingHandshakeReply:193(10281386ns) {SwitchToState:179(3479ns)}}}
2025-11-28T14:53:13.185268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789440677395808:2108] [node 6] ICP22 created new session: [1:7577789444972363139:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.148806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:13.104586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789434049958698:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.172470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789443551348882:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48201
2025-11-28T14:53:13.097118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789437933093487:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:13.192183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.173968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789439256381574:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.160568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789436618798970:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.109282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789437933093411:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.192438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789444972363137:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.192614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789440677395827:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:13.194508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789436382428448:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.194571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:13.194621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789436382428448:2049] [node 3] ICP22 created new session: [1:7577789444972363141:2049]
2025-11-28T14:53:13.200652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789444972363141:2049] [node 3] ICS09 handshake done sender: [1:7577789440677395827:2118] self: [1:7577789440677395826:0] peer: [3:7577789441536673704:0] socket: 350 qp: -1
2025-11-28T14:53:13.200703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789444972363141:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:13.200784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:13.221167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789444972363139:2108] [node 6] ICS09 handshake done sender: [1:7577789440677395814:2112] self: [1:7577789440677395813:0] peer: [6:7577789437933093490:0] socket: 351 qp: -1
2025-11-28T14:53:13.225725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789444972363139:2108] [node 6] ICS10 traffic start
2025-11-28T14:53:13.238227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789436382428448:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(42788446ns) {HandleHandshakeStatus:329(42785403ns) {DropIncomingHandshake:442(16117ns) DropOutgoingHandshake:457(265ns) SwitchToState:179(507ns) ProcessPendingSessionEvents:495(596434ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 529608ns max# 77726ns}}}
2025-11-28T14:53:13.104775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789438344926058:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.220736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789441536673700:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.104835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.160598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789436618798970:2095] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.231090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789437241706380:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(306404648ns) {Configure:104(306395421ns) {ProcessConfigured:135(8931767ns) {StartInitialHandshake:157(8928336ns) {DropHandshakes:470(2669ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(649ns) PrepareNewSessionHandshake:477(2939ns) {SwitchToState:179(1222ns)}}}}}
2025-11-28T14:53:13.173992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789439256381574:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.109357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.166992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789440913766299:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.261589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:13.286171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789440677395807:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(173441188ns) {HandleHandshakeStatus:329(173437699ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(879ns) SwitchToState:179(444ns) ProcessPendingSessionEvents:495(9438719ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9371456ns max# 8932323ns}}}
2025-11-28T14:53:13.286351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789440677395807:2107] [node 5] ICP18 handshake (actor [1:7577789444972363137:2119]) is too old
2025-11-28T14:53:13.174614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789443551348883:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48201
2025-11-28T14:53:13.231200Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789432946739020:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789441536673700:2117] held: no
2025-11-28T14:53:13.105511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789438344926058:2107] [node 6] ICP22 created new session: [2:7577789442639893381:2107]
2025-11-28T14:53:13.109408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789437933093411:2049] [node 2] ICP22 created new session: [6:7577789442228060789:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.116477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:13.192518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789439256381574:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:13.172997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789440913766304:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.239469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789441536673706:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62540
2025-11-28T14:53:13.327418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789440677395808:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(221306149ns) {HandleHandshakeStatus:329(221303460ns) {DropIncomingHandshake:442(20064450ns) DropOutgoingHandshake:457(499ns) SwitchToState:179(705ns) ProcessPendingSessionEvents:495(542688ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 488155ns max# 63719ns}}}
2025-11-28T14:53:13.109459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789442228060789:2049] [node 2] ICS09 handshake done sender: [6:7577789437933093487:2115] self: [6:7577789437933093489:0] peer: [2:7577789438344926064:0] socket: 368 qp: -1
2025-11-28T14:53:13.128650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789434049958698:2049] [node 3] ICP22 created new session: [2:7577789442639893382:2049]
2025-11-28T14:53:13.132578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789442639893381:2107] [node 6] ICS09 handshake done sender: [2:7577789438344926065:2112] self: [2:7577789438344926064:0] peer: [6:7577789437933093489:0] socket: 365 qp: -1
2025-11-28T14:53:13.140153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789442639893381:2107] [node 6] ICS10 traffic start
2025-11-28T14:53:13.164462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789442639893382:2049] [node 3] ICS09 handshake done sender: [2:7577789438344926060:2110] self: [2:7577789438344926059:0] peer: [3:7577789437241706396:0] socket: 345 qp: -1
2025-11-28T14:53:13.168456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:13.168540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789442639893382:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:13.200437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:13.200663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789438344926071:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.200690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789438344926071:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.201599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789438344926071:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:13.201920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789438344926057:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.201993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:13.202053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789438344926057:2109] [node 5] ICP22 created new session: [2:7577789442639893388:2109]
2025-11-28T14:53:13.202097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789442639893388:2109] [node 5] ICS09 handshake done sender: [2:7577789438344926071:2116] self: [2:7577789438344926070:0] peer: [5:7577789440913766309:0] socket: 352 qp: -1
2025-11-28T14:53:13.202131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789442639893388:2109] [node 5] ICS10 traffic start
2025-11-28T14:53:13.202176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:13.296019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789438344926058:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(190300102ns) {HandleHandshakeStatus:329(190297672ns) {DropIncomingHandshake:442(15018ns) DropOutgoingHandshake:457(322ns) SwitchToState:179(488ns) ProcessPendingSessionEvents:495(19609177ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19546048ns max# 19138442ns}}}
2025-11-28T14:53:13.318191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789434049958698:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(216602601ns) {HandleHandshakeStatus:329(216594597ns) {DropIncomingHandshake:442(21686ns) DropOutgoingHandshake:457(141ns) SwitchToState:179(541ns) ProcessPendingSessionEvents:495(518642ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 463591ns max# 55814ns}}}
2025-11-28T14:53:13.320479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789438344926074:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.320519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789438344926074:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.194176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789439256381561:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.173154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789436618798969:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:13.173390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789432323831626:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:13.239618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789432946739020:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.109501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789442228060789:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:13.109544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:13.111523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789437933093485:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:13.137248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789437933093410:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.137325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:13.137384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789437933093410:2048] [node 1] ICP22 created new session: [6:7577789442228060791:2048]
2025-11-28T14:53:13.137438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789442228060791:2048] [node 1] ICS09 handshake done sender: [6:7577789437933093485:2114] self: [6:7577789437933093490:0] peer: [1:7577789440677395813:0] socket: 356 qp: -1
2025-11-28T14:53:13.137473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789442228060791:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:13.137508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:13.158308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789437933093484:2086] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:13.163069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789437933093484:2086] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(46616643ns) {Configure:104(46615413ns) {ProcessConfigured:135(190797ns) {StartInitialHandshake:157(81308ns) {DropHandshakes:470(2901ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(455ns)} GenerateSessionVirtualId:988(7886ns) PrepareNewSessionHandshake:477(4606ns) {SwitchToState:179(562ns)}} IncomingHandshake:261(77315ns) {DropIncomingHandshake:442(114ns) PrepareNewSessionHandshake:477(208ns) {SwitchToState:179(104ns)} GenerateSessionVirtualId:988(166ns) IssueIncomingHandshakeReply:193(54957ns) {SwitchToState:179(66ns)}}}}}
2025-11-28T14:53:13.166492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789442228060797:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
2025-11-28T14:53:13.192923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789442228060799:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
2025-11-28T14:53:13.193719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789437933093483:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:13.195987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789442228060799:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.196116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789437933093444:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:13.196750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789437933093484:2086] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.224635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:13.224708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789437933093484:2086] [node 5] ICP22 created new session: [6:7577789442228060801:2086]
2025-11-28T14:53:13.224767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789442228060801:2086] [node 5] ICS09 handshake done sender: [6:7577789437933093483:2113] self: [6:7577789442228060795:0] peer: [5:7577789436618798961:0] socket: 340 qp: -1
2025-11-28T14:53:13.228636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789442228060801:2086] [node 5] ICS10 traffic start
2025-11-28T14:53:13.228714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:13.230256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789437933093484:2086] [node 5] ICP35 event processing took too much time PendingConnection:279(32632260ns) {HandleHandshakeStatus:329(32629827ns) {DropIncomingHandshake:442(825ns) DropOutgoingHandshake:457(3561ns) SwitchToState:179(144ns) ProcessPendingSessionEvents:495(513855ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 460368ns max# 51256ns}}}
2025-11-28T14:53:13.259526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789442228060803:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
2025-11-28T14:53:13.278670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789442228060803:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.278939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789437933093412:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:13.337671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789437933093444:2074] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789442228060799:2121] held: no
2025-11-28T14:53:13.338839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789442228060806:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
2025-11-28T14:53:13.340670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789442228060804:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58976
2025-11-28T14:53:13.353772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789442228060803:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:13.175330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789440913766310:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25921
2025-11-28T14:53:13.369462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789438344926074:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:13.370021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789438344926056:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.370129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:13.370192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789438344926056:2108] [node 4] ICP22 created new session: [2:7577789442639893391:2108]
2025-11-28T14:53:13.370249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789442639893391:2108] [node 4] ICS09 handshake done sender: [2:7577789438344926074:2117] self: [2:7577789438344926073:0] peer: [4:7577789443551348891:0] socket: 354 qp: -1
2025-11-28T14:53:13.370289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789442639893391:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:13.370331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:13.194271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:13.192399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789436618798970:2095] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:13.192517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789440913766306:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25921
2025-11-28T14:53:13.256649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789436618798943:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.256699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:13.259975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789432323831626:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1046) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789440913766304:2120] held: no
2025-11-28T14:53:13.268603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789436618798969:2073] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789440913766299:2118] held: no
2025-11-28T14:53:13.269706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789440913766299:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:13.276643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789436618798943:2068] [node 6] ICP22 created new session: [5:7577789440913766312:2068]
2025-11-28T14:53:13.276713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789440913766312:2068] [node 6] ICS09 handshake done sender: [5:7577789436618798970:2095] self: [5:7577789436618798961:0] peer: [6:7577789442228060795:0] socket: 330 qp: -1
2025-11-28T14:53:13.276758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789440913766312:2068] [node 6] ICS10 traffic start
2025-11-28T14:53:13.276801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:13.278403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789436618798943:2068] [node 6] ICP35 event processing took too much time PendingConnection:279(20856699ns) {HandleHandshakeStatus:329(20853703ns) {DropIncomingHandshake:442(1024ns) DropOutgoingHandshake:457(936ns) SwitchToState:179(494ns) ProcessPendingSessionEvents:495(531423ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 472401ns max# 40445ns}}}
2025-11-28T14:53:13.278741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789440913766303:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25921
2025-11-28T14:53:13.293205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789436618798969:2073] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.293275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:13.293325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789436618798969:2073] [node 4] ICP22 created new session: [5:7577789440913766314:2073]
2025-11-28T14:53:13.293388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789440913766314:2073] [node 4] ICS09 handshake done sender: [5:7577789440913766299:2118] self: [5:7577789440913766308:0] peer: [4:7577789439256381573:0] socket: 335 qp: -1
2025-11-28T14:53:13.293421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789440913766314:2073] [node 4] ICS10 traffic start
2025-11-28T14:53:13.293455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:13.293779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789436618798943:2068] [node 6] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:13.301601Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789432323831627:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(300192752ns) {HandleHandshakeStatus:329(276194664ns) {DropIncomingHandshake:442(9298ns) DropOutgoingHandshake:457(33665ns) SwitchToState:179(965ns) ProcessPendingSessionEvents:495(151571052ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 151431968ns max# 29246092ns}}}
2025-11-28T14:53:13.313106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789440913766311:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25921
2025-11-28T14:53:13.319674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789436618799002:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:13.328675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789432323831625:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789436618799002:2117] held: no
2025-11-28T14:53:13.339114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789440913766304:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:13.352121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789432323831626:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.352212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:13.352269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789432323831626:2049] [node 2] ICP22 created new session: [5:7577789440913766316:2049]
2025-11-28T14:53:13.352318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789440913766316:2049] [node 2] ICS09 handshake done sender: [5:7577789440913766304:2120] self: [5:7577789440913766309:0] peer: [2:7577789438344926070:0] socket: 338 qp: -1
2025-11-28T14:53:13.352354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789440913766316:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:13.352417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:13.377333Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789432323831625:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(40321286ns) {HandleHandshakeStatus:395(40313414ns) {LogHandshakeFail:472(14238ns) {UpdateErrorStateLog:545(5874ns)} DropOutgoingHandshake:457(555ns) ProcessPendingSessionEvents:495(40228778ns) {unordered DropSessionEvent:512 num# 100 sum# 13074205ns max# 7976121ns}}}
2025-11-28T14:53:13.377400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789432323831625:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.377447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:13.377496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789432323831625:2048] [node 1] ICP22 created new session: [5:7577789440913766318:2048]
2025-11-28T14:53:13.377549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789440913766318:2048] [node 1] ICS09 handshake done sender: [5:7577789436618799002:2117] self: [5:7577789440913766300:0] peer: [1:7577789440677395818:0] socket: 381 qp: -1
2025-11-28T14:53:13.377583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789440913766318:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:13.377649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:13.239661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:13.372727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789437933093412:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.194330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789439256381561:2104] [node 5] ICP22 created new session: [4:7577789443551348886:2104]
2025-11-28T14:53:13.378548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789442228060799:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:13.396558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789437933093444:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.400512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:13.408878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789437933093412:2050] [node 3] ICP22 created new session: [6:7577789442228060807:2050]
2025-11-28T14:53:13.408950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789442228060807:2050] [node 3] ICS09 handshake done sender: [6:7577789442228060803:2122] self: [6:7577789442228060805:0] peer: [3:7577789441536673707:0] socket: 371 qp: -1
2025-11-28T14:53:13.409003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789442228060807:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:13.409081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:13.410839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789437933093412:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(44062391ns) {HandleHandshakeStatus:329(37230509ns) {DropIncomingHandshake:442(1184ns) DropOutgoingHandshake:457(7055ns) SwitchToState:179(509ns) ProcessPendingSessionEvents:495(671808ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 609063ns max# 64848ns}}}
2025-11-28T14:53:13.194377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789443551348886:2104] [node 5] ICS09 handshake done sender: [4:7577789439256381574:2115] self: [4:7577789439256381573:0] peer: [5:7577789440913766308:0] socket: 382 qp: -1
2025-11-28T14:53:13.194410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789443551348886:2104] [node 5] ICS10 traffic start
2025-11-28T14:53:13.194453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:13.241627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789439256381576:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.241668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789439256381576:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.257594Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789434961414202:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(132112460ns) {HandleHandshakeStatus:395(132109398ns) {LogHandshakeFail:472(16034ns) {UpdateErrorStateLog:545(5924ns)} DropOutgoingHandshake:457(445ns) ProcessPendingSessionEvents:495(132012288ns) {unordered DropSessionEvent:512 num# 100 sum# 38887542ns max# 15977692ns}}}
2025-11-28T14:53:13.259085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789439256381584:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:13.260768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789434961414202:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.260835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:13.239716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789432946739020:2048] [node 1] ICP22 created new session: [3:7577789441536673709:2048]
2025-11-28T14:53:13.239763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789441536673709:2048] [node 1] ICS09 handshake done sender: [3:7577789441536673700:2117] self: [3:7577789441536673704:0] peer: [1:7577789440677395826:0] socket: 386 qp: -1
2025-11-28T14:53:13.239795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789441536673709:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:13.239829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:13.273149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789441536673706:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.274817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789437241706379:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(301381943ns) {HandleHandshakeStatus:329(301378662ns) {DropIncomingHandshake:442(1643ns) DropOutgoingHandshake:457(1182ns) SwitchToState:179(747ns) ProcessPendingSessionEvents:495(25140487ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25085349ns max# 24646836ns}}}
2025-11-28T14:53:13.304582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789441536673708:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.304631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789441536673708:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:13.333339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789441536673708:2120] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:13.376519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789437241706380:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.406127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:13.260883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789434961414202:2050] [node 3] ICP22 created new session: [4:7577789443551348889:2050]
2025-11-28T14:53:13.261718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789443551348883:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:13.264753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789443551348889:2050] [node 3] ICS09 handshake done sender: [4:7577789439256381584:2119] self: [4:7577789443551348881:0] peer: [3:7577789437241706393:0] socket: 379 qp: -1
2025-11-28T14:53:13.264791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789443551348889:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:13.264838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:13.272523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789439256381576:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:13.292856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789434961414201:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:13.305004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789439256381552:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:13.308790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:13.319981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789439256381552:2100] [node 6] ICP22 created new session: [4:7577789443551348892:2100]
2025-11-28T14:53:13.333454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789443551348892:2100] [node 6] ICS09 handshake done sender: [4:7577789439256381576:2117] self: [4:7577789439256381572:0] peer: [6:7577789442228060800:0] socket: 398 qp: -1
2025-11-28T14:53:13.333507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789443551348892:2100] [node 6] ICS10 traffic start
2025-11-28T14:53:13.333568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:13.392543Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789434961414201:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1046) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789443551348883:2121] held: no
2025-11-28T14:53:13.397185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789443551348894:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48201
2025-11-28T14:53:13.404648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789443551348883:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:13.415772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789434961414201:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.415870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:13.415927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789434961414201:2049] [node 2] ICP22 created new session: [4:7577789443551348895:2049]
2025-11-28T14:53:13.415974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789443551348895:2049] [node 2] ICS09 handshake done sender: [4:7577789443551348883:2121] self: [4:7577789443551348891:0] peer: [2:7577789438344926073:0] socket: 396 qp: -1
2025-11-28T14:53:13.416017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789443551348895:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:13.416077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:13.421992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:13.422884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789437241706380:2109] [node 6] ICP22 created new session: [3:7577789441536673714:2109]
2025-11-28T14:53:13.422947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789441536673714:2109] [node 6] ICS09 handshake done sender: [3:7577789441536673708:2120] self: [3:7577789441536673707:0] peer: [6:7577789442228060805:0] socket: 343 qp: -1
2025-11-28T14:53:13.422985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789441536673714:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:13.423046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:13.422079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789437933093444:2074] [node 4] ICP22 created new session: [6:7577789442228060809:2074]
2025-11-28T14:53:13.422160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789442228060809:2074] [node 4] ICS09 handshake done sender: [6:7577789442228060799:2121] self: [6:7577789442228060800:0] peer: [4:7577789439256381572:0] socket: 400 qp: -1
2025-11-28T14:53:13.422208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789442228060809:2074] [node 4] ICS10 traffic start
2025-11-28T14:53:13.422290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:13.422548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789437933093444:2074] [node 4] ICP35 event processing took too much time PendingConnection:279(25944555ns) {HandleHandshakeStatus:329(25940523ns) {DropIncomingHandshake:442(1696ns) DropOutgoingHandshake:457(366ns) SwitchToState:179(1183ns) ProcessPendingSessionEvents:495(1309ns)}}
2025-11-28T14:53:13.451469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789439256381561:2104] [node 5] ICP35 event processing took too much time PendingConnection:279(256176930ns) {HandleHandshakeStatus:329(256173978ns) {DropIncomingHandshake:442(14932ns) DropOutgoingHandshake:457(468ns) SwitchToState:179(656ns) ProcessPendingSessionEvents:495(255703698ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 255497618ns max# 19846404ns}}}
2025-11-28T14:53:13.455445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789439256381552:2100] [node 6] ICP35 event processing took too much time PendingConnection:279(100521652ns) {HandleHandshakeStatus:329(100519099ns) {DropIncomingHandshake:442(36789ns) DropOutgoingHandshake:457(614ns) SwitchToState:179(705ns) ProcessPendingSessionEvents:495(7640037ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7550670ns max# 6929318ns}}}
2025-11-28T14:53:13.459838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789434049958697:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(212144576ns) {HandleHandshakeStatus:395(188022737ns) {LogHandshakeFail:472(18868ns) {UpdateErrorStateLog:545(6202ns)} DropOutgoingHandshake:457(3239ns) ProcessPendingSessionEvents:495(168106475ns) {unordered DropSessionEvent:512 num# 100 sum# 168012721ns max# 20028657ns}}}
2025-11-28T14:53:13.459977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789438344926075:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:13.460484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789434049958697:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:13.465320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:13.465400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789434049958697:2048] [node 1] ICP22 created new session: [2:7577789442639893393:2048]
2025-11-28T14:53:13.465449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789442639893393:2048] [node 1] ICS09 handshake done sender: [2:7577789438344926075:2118] self: [2:7577789438344926080:0] peer: [1:7577789440677395809:0] socket: 364 qp: -1
2025-11-28T14:53:13.465488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789442639893393:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:13.465561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:13.458190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789437241706380:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(77156934ns) {HandleHandshakeStatus:329(77153303ns) {DropIncomingHandshake:442(4572045ns) DropOutgoingHandshake:457(1676ns) SwitchToState:179(541ns) ProcessPendingSessionEvents:495(30476595ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30389127ns max# 8635592ns}}}
2025-11-28T14:53:13.545794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:13.546003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789441536673709:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-28T14:53:13.546043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789432946739020:2048] [node 1] ICP30 unregister session Session# [3:7577789441536673709:2048] VirtualId# [3:7577789441536673704:0]
2025-11-28T14:53:13.546073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789441536673709:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.546147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:13.546181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:13.544891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:13.545174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789442639893393:2048] [node 1] ICS01 socket: 364 reason# EndOfStream
2025-11-28T14:53:13.545233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789434049958697:2048] [node 1] ICP30 unregister session Session# [2:7577789442639893393:2048] VirtualId# [2:7577789438344926080:0]
2025-11-28T14:53:13.545283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789442639893393:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.545380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:13.545433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:13.555739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:13.556016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789439256381581:2048] [node 1] ICS01 socket: 362 reason# EndOfStream
2025-11-28T14:53:13.556068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789434961414200:2048] [node 1] ICP30 unregister session Session# [4:7577789439256381581:2048] VirtualId# [4:7577789439256381579:0]
2025-11-28T14:53:13.556101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789439256381581:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.556180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:13.556251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:13.560512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:13.560777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789440913766318:2048] [node 1] ICS01 socket: 381 reason# EndOfStream
2025-11-28T14:53:13.560509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:13.560835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789432323831625:2048] [node 1] ICP30 unregister session Session# [5:7577789440913766318:2048] VirtualId# [5:7577789440913766300:0]
2025-11-28T14:53:13.560769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789442228060791:2048] [node 1] ICS01 socket: 356 reason# EndOfStream
2025-11-28T14:53:13.560867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789440913766318:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.560973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:13.561090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:13.560816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789437933093410:2048] [node 1] ICP30 unregister session Session# [6:7577789442228060791:2048] VirtualId# [6:7577789437933093490:0]
2025-11-28T14:53:13.560856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789442228060791:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.560969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:13.561023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:13.673135Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:13.673380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789441536673696:2049] [node 2] ICS01 socket: 349 reason# EndOfStream
2025-11-28T14:53:13.673429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789432946739021:2049] [node 2] ICP30 unregister session Session# [3:7577789441536673696:2049] VirtualId# [3:7577789437241706396:0]
2025-11-28T14:53:13.673466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789441536673696:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.673576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:13.673617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:13.680139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:13.680401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789442228060789:2049] [node 2] ICS01 socket: 368 reason# EndOfStream
2025-11-28T14:53:13.680445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789437933093411:2049] [node 2] ICP30 unregister session Session# [6:7577789442228060789:2049] VirtualId# [6:7577789437933093489:0]
2025-11-28T14:53:13.680482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789442228060789:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.680607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:13.680662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:13.680563Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:13.680784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789443551348895:2049] [node 2] ICS01 socket: 396 reason# EndOfStream
2025-11-28T14:53:13.680822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789434961414201:2049] [node 2] ICP30 unregister session Session# [4:7577789443551348895:2049] VirtualId# [4:7577789443551348891:0]
2025-11-28T14:53:13.680853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789443551348895:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.680926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:13.680967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:13.681061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:13.681242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789440913766316:2049] [node 2] ICS01 socket: 338 reason# EndOfStream
2025-11-28T14:53:13.681276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789432323831626:2049] [node 2] ICP30 unregister session Session# [5:7577789440913766316:2049] VirtualId# [5:7577789440913766309:0]
2025-11-28T14:53:13.681303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789440913766316:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.681355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:13.681395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:13.757668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:13.757866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789443551348889:2050] [node 3] ICS01 socket: 379 reason# EndOfStream
2025-11-28T14:53:13.757901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789434961414202:2050] [node 3] ICP30 unregister session Session# [4:7577789443551348889:2050] VirtualId# [4:7577789443551348881:0]
2025-11-28T14:53:13.757933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789443551348889:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.758038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:13.758077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:13.756554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:13.756792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789442228060807:2050] [node 3] ICS01 socket: 371 reason# EndOfStream
2025-11-28T14:53:13.756842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789437933093412:2050] [node 3] ICP30 unregister session Session# [6:7577789442228060807:2050] VirtualId# [6:7577789442228060805:0]
2025-11-28T14:53:13.756885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789442228060807:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.756972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:13.757016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:13.756640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:13.756829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789440913766302:2050] [node 3] ICS07 socket disconnect 361 reason# EndOfStream
2025-11-28T14:53:13.756858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789440913766302:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.756965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:13.757009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789440913766302:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:13.758816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789432323831627:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19700) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:13.758877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789440913766302:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:13.758907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789432323831627:2050] [node 3] ICP30 unregister session Session# [5:7577789440913766302:2050] VirtualId# [5:7577789436618798999:0]
2025-11-28T14:53:13.759007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:13.759039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789432323831627:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:13.759052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:13.769255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789432323831627:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:13.914185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:13.914518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789442228060809:2074] [node 4] ICS01 socket: 400 reason# EndOfStream
2025-11-28T14:53:13.914560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789437933093444:2074] [node 4] ICP30 unregister session Session# [6:7577789442228060809:2074] VirtualId# [6:7577789442228060800:0]
2025-11-28T14:53:13.914603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789442228060809:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.914688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:13.914736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:13.916554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:13.916754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789440913766314:2073] [node 4] ICS01 socket: 335 reason# EndOfStream
2025-11-28T14:53:13.916797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789436618798969:2073] [node 4] ICP30 unregister session Session# [5:7577789440913766314:2073] VirtualId# [5:7577789440913766308:0]
2025-11-28T14:53:13.916833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789440913766314:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:13.916916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:13.916956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:14.066022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:14.066258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789442228060801:2086] [node 5] ICS01 socket: 340 reason# EndOfStream
2025-11-28T14:53:14.066295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789437933093484:2086] [node 5] ICP30 unregister session Session# [6:7577789442228060801:2086] VirtualId# [6:7577789442228060795:0]
2025-11-28T14:53:14.066335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789442228060801:2086] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:14.066426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:14.066468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:14.284716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789448402657350:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:14.284782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789448402657351:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:14.374973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789449010856024:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:14.375023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789449010856025:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:14.428821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789449494307404:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:14.429627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789449494307405:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:14.492045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789445582287235:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:14.492113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789445582287236:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:14.492154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789445582287237:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:14.556863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789445551130258:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:14.556912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789445551130259:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:14.556948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789445551130260:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:14.744814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789447778612104:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:14.744876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789447778612105:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:14.744913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789447778612106:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:14.952664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789449010856050:2070] [node 5] ICP01 ready to work
2025-11-28T14:53:14.953043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789449010856051:2072] [node 6] ICP01 ready to work
2025-11-28T14:53:15.024446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789449877254552:2063] [node 6] ICP01 ready to work
2025-11-28T14:53:15.002847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789448402657407:2094] [node 4] ICP01 ready to work
2025-11-28T14:53:15.059485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789453789274725:2064] [node 5] ICP01 ready to work
2025-11-28T14:53:15.084621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789452697624709:2096] [node 6] ICP01 ready to work
2025-11-28T14:53:15.088984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789453789274761:2065] [node 4] ICP01 ready to work
2025-11-28T14:53:15.132540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789449846097612:2069] [node 4] ICP01 ready to work
2025-11-28T14:53:15.130966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789452073579433:2063] [node 5] ICP01 ready to work
2025-11-28T14:53:15.180683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789452073579461:2064] [node 4] ICP01 ready to work
2025-11-28T14:53:15.116927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789453789274765:2074] [node 6] ICP01 ready to work
2025-11-28T14:53:15.161927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789452697624711:2097] [node 5] ICP01 ready to work
2025-11-28T14:53:15.136448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789449877254570:2066] [node 5] ICP01 ready to work
2025-11-28T14:53:15.200620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789449846097618:2070] [node 6] ICP01 ready to work
2025-11-28T14:53:15.201396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789449010856050:2070] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(248041398ns) {Configure:104(248036466ns) {ProcessConfigured:135(59886ns) {StartInitialHandshake:157(57510ns) {DropHandshakes:470(7165ns) {DropIncomingHandshake:442(715ns) DropOutgoingHandshake:457(318ns)} GenerateSessionVirtualId:988(650ns) PrepareNewSessionHandshake:477(1383ns) {SwitchToState:179(605ns)}}}}}
2025-11-28T14:53:15.204145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789453305823389:2076] [node 4] ICP01 ready to work
2025-11-28T14:53:15.184388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789453789274761:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(34003512ns) {Configure:104(33993702ns) {ProcessConfigured:135(93447ns) {StartInitialHandshake:157(90460ns) {DropHandshakes:470(2568ns) {DropIncomingHandshake:442(482ns) DropOutgoingHandshake:457(533ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:477(1934ns) {SwitchToState:179(1067ns)}}}}}
2025-11-28T14:53:15.240822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789449846097612:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29588168ns) {Configure:104(29584438ns) {ProcessConfigured:135(7934287ns) {StartInitialHandshake:157(7930362ns) {DropHandshakes:470(1275ns) {DropIncomingHandshake:442(291ns) DropOutgoingHandshake:457(226ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(3543ns) {SwitchToState:179(188ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.286594Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789453789274765:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(99411277ns) {Configure:104(99407317ns) {ProcessConfigured:135(79682ns) {StartInitialHandshake:157(77219ns) {DropHandshakes:470(2769ns) {DropIncomingHandshake:442(716ns) DropOutgoingHandshake:457(436ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:477(6355ns) {SwitchToState:179(579ns)}}}}}
2025-11-28T14:53:15.282215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789449010856025:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42143815ns) {Configure:104(42140047ns) {ProcessConfigured:135(89395ns) {StartInitialHandshake:157(75899ns) {DropHandshakes:470(3014ns) {DropIncomingHandshake:442(566ns) DropOutgoingHandshake:457(478ns)} GenerateSessionVirtualId:988(877ns) PrepareNewSessionHandshake:477(1644ns) {SwitchToState:179(606ns)}}}}}
2025-11-28T14:53:15.283637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789453305823396:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40012
2025-11-28T14:53:15.305461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789452073579461:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(85679596ns) {Configure:104(85675878ns) {ProcessConfigured:135(99116ns) {StartInitialHandshake:157(97125ns) {DropHandshakes:470(1652ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(1136ns) PrepareNewSessionHandshake:477(1685ns) {SwitchToState:179(457ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.309843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789448402657407:2094] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(77733849ns) {Configure:104(77730031ns) {ProcessConfigured:135(57468ns) {StartInitialHandshake:157(54846ns) {DropHandshakes:470(3669ns) {DropIncomingHandshake:442(475ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:477(780ns) {SwitchToState:179(185ns)}}}}}
2025-11-28T14:53:15.308463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579475:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.337474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789452697624711:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25301834ns) {Configure:104(25297893ns) {ProcessConfigured:135(75187ns) {StartInitialHandshake:157(72504ns) {DropHandshakes:470(9275ns) {DropIncomingHandshake:442(843ns) DropOutgoingHandshake:457(379ns)} GenerateSessionVirtualId:988(924ns) PrepareNewSessionHandshake:477(1570ns) {SwitchToState:179(774ns)}}}}}
2025-11-28T14:53:15.338633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789452697624722:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48675
2025-11-28T14:53:15.338715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789452697624723:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48675
2025-11-28T14:53:15.333301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789453789274779:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25704
2025-11-28T14:53:15.333541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789453789274780:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25704
2025-11-28T14:53:15.342909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789445582287235:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39126643ns) {Configure:104(39122532ns) {ProcessConfigured:135(61018ns) {StartInitialHandshake:157(58754ns) {DropHandshakes:470(8541ns) {DropIncomingHandshake:442(718ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(741ns) PrepareNewSessionHandshake:477(1608ns) {SwitchToState:179(605ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.349077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789449846097632:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16435
2025-11-28T14:53:15.349555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789449846097633:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.350944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789453789274779:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.377221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789453789274781:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25704
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.382801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789445582287237:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35364286ns) {Configure:104(35355818ns) {ProcessConfigured:135(71758ns) {StartInitialHandshake:157(69401ns) {DropHandshakes:470(2598ns) {DropIncomingHandshake:442(821ns) DropOutgoingHandshake:457(524ns)} GenerateSessionVirtualId:988(637ns) PrepareNewSessionHandshake:477(2470ns) {SwitchToState:179(1109ns)}}}}}
2025-11-28T14:53:15.384142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789449877254604:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57718
2025-11-28T14:53:15.384221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789449877254605:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57718
2025-11-28T14:53:15.384444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789449877254606:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57718
2025-11-28T14:53:15.385291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789453305823400:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40012
2025-11-28T14:53:15.404995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789452697624722:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.385470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789453305823401:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40012
2025-11-28T14:53:15.420751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789449846097634:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16435
2025-11-28T14:53:15.428994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789449877254605:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.445947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789453305823400:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.446067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789453305823396:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.446942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789449846097632:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.447062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789445551130259:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.452672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789449846097634:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.452849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789449846097612:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.454367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579479:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
2025-11-28T14:53:15.454502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579480:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
2025-11-28T14:53:15.457535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789453789274780:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.457896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789453305823386:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.457933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789453305823386:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.458609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789453305823401:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.461774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789452073579475:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.461933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789447778612105:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.464897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789452697624723:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.465344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789452697624727:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48675
2025-11-28T14:53:15.465817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789453789274781:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.465927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789449494307404:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.466328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789452697624713:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.466359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789452697624713:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.467555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789453789274785:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25704
2025-11-28T14:53:15.467798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789452697624713:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:15.468431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789453789274781:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:15.468668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789449494307404:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.468597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789452073579479:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.468744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789447778612104:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.457865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789449846097633:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.468939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789448402657351:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.469272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.472449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:15.475848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789449494307404:2048] [node 1] ICP22 created new session: [3:7577789453789274786:2048]
2025-11-28T14:53:15.480591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789453305823386:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:15.462478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789449846097638:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16435
2025-11-28T14:53:15.481553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789449846097632:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:15.481725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789445551130259:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.481783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:15.481834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789445551130259:2049] [node 2] ICP22 created new session: [5:7577789449846097639:2049]
2025-11-28T14:53:15.481877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789449846097639:2049] [node 2] ICS09 handshake done sender: [5:7577789449846097632:2115] self: [5:7577789449846097635:0] peer: [2:7577789453305823385:0] socket: 343 qp: -1
2025-11-28T14:53:15.481913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789449846097639:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:15.481952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:15.487076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579483:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
2025-11-28T14:53:15.485100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789453789274786:2048] [node 1] ICS09 handshake done sender: [3:7577789453789274781:2117] self: [3:7577789453789274784:0] peer: [1:7577789452697624712:0] socket: 368 qp: -1
2025-11-28T14:53:15.472548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789448402657351:2049] [node 3] ICP22 created new session: [1:7577789452697624731:2049]
2025-11-28T14:53:15.472600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789452697624731:2049] [node 3] ICS09 handshake done sender: [1:7577789452697624713:2110] self: [1:7577789452697624712:0] peer: [3:7577789453789274784:0] socket: 360 qp: -1
2025-11-28T14:53:15.472641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789452697624731:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:15.472708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:15.474504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789452697624715:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.474530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789452697624715:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.484819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789452697624715:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:15.485368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789452697624709:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.485410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:15.485460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789452697624709:2096] [node 6] ICP22 created new session: [1:7577789452697624735:2096]
2025-11-28T14:53:15.485519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789452697624735:2096] [node 6] ICS09 handshake done sender: [1:7577789452697624715:2111] self: [1:7577789452697624714:0] peer: [6:7577789452073579482:0] socket: 361 qp: -1
2025-11-28T14:53:15.485559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789452697624735:2096] [node 6] ICS10 traffic start
2025-11-28T14:53:15.485595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:15.487621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789449877254597:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.487307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789453789274786:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:15.488015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:15.489732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789449494307404:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20071309ns) {HandleHandshakeStatus:329(20070028ns) {DropIncomingHandshake:442(1737ns) DropOutgoingHandshake:457(5608ns) SwitchToState:179(399ns) ProcessPendingSessionEvents:495(614133ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 543058ns max# 46096ns}}}
2025-11-28T14:53:15.487656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789449877254597:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.492426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789453305823388:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.492461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789453305823388:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.493231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789449010856050:2070] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.493295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:15.493352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789449010856050:2070] [node 5] ICP22 created new session: [2:7577789453305823407:2070]
2025-11-28T14:53:15.493417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789453305823407:2070] [node 5] ICS09 handshake done sender: [2:7577789453305823386:2110] self: [2:7577789453305823385:0] peer: [5:7577789449846097635:0] socket: 340 qp: -1
2025-11-28T14:53:15.493454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789453305823407:2070] [node 5] ICS10 traffic start
2025-11-28T14:53:15.493502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:15.497253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789452073579479:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:15.497512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789447778612104:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.497574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:15.497622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789447778612104:2048] [node 1] ICP22 created new session: [6:7577789452073579484:2048]
2025-11-28T14:53:15.497678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789452073579484:2048] [node 1] ICS09 handshake done sender: [6:7577789452073579479:2116] self: [6:7577789452073579482:0] peer: [1:7577789452697624714:0] socket: 373 qp: -1
2025-11-28T14:53:15.497716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789452073579484:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:15.497756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.501152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789449877254604:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.501540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789445582287237:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.501880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789445582287236:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.501973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789449877254612:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57718
2025-11-28T14:53:15.508637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789449877254597:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:15.510455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789453789274769:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.510498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789453789274769:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.513572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789452697624727:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.516496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789453305823391:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.516545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789453305823391:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.517103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789453305823406:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40012
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.504395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789452073579480:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.523265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789453305823388:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.523920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789449877254606:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.524072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789449877254601:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.524096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789449877254601:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.524636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789449877254570:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.524698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:15.524757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789449877254570:2066] [node 5] ICP22 created new session: [4:7577789449877254619:2066]
2025-11-28T14:53:15.524831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789449877254619:2066] [node 5] ICS09 handshake done sender: [4:7577789449877254597:2110] self: [4:7577789449877254596:0] peer: [5:7577789449846097636:0] socket: 370 qp: -1
2025-11-28T14:53:15.524889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789449877254619:2066] [node 5] ICS10 traffic start
2025-11-28T14:53:15.524947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:15.527487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789449877254616:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57718
2025-11-28T14:53:15.527631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789449877254617:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57718
2025-11-28T14:53:15.527713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789449877254618:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57718
2025-11-28T14:53:15.527842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789445582287235:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.504866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789452073579461:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.537127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789452697624717:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.537170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789452697624717:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.539278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789452697624739:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48675
2025-11-28T14:53:15.539813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789449877254601:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:15.540081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789452697624717:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.544749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789453789274769:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:15.522749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579488:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
2025-11-28T14:53:15.540630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789449846097642:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.555693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789449846097634:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.556006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789449846097643:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16435
2025-11-28T14:53:15.556097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789449846097612:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.556160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:15.556206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789449846097612:2069] [node 4] ICP22 created new session: [5:7577789449846097644:2069]
2025-11-28T14:53:15.556266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789449846097644:2069] [node 4] ICS09 handshake done sender: [5:7577789449846097634:2117] self: [5:7577789449846097636:0] peer: [4:7577789449877254596:0] socket: 377 qp: -1
2025-11-28T14:53:15.556301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789449846097644:2069] [node 4] ICS10 traffic start
2025-11-28T14:53:15.556333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:15.557062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789448402657407:2094] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.557122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:15.557169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789448402657407:2094] [node 4] ICP22 created new session: [1:7577789452697624741:2094]
2025-11-28T14:53:15.557211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789452697624741:2094] [node 4] ICS09 handshake done sender: [1:7577789452697624717:2112] self: [1:7577789452697624716:0] peer: [4:7577789449877254623:0] socket: 362 qp: -1
2025-11-28T14:53:15.557237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789452697624741:2094] [node 4] ICS10 traffic start
2025-11-28T14:53:15.557280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:15.522856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579489:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
2025-11-28T14:53:15.545373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789452073579475:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:15.549774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789447778612105:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.549848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:15.549913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789447778612105:2049] [node 2] ICP22 created new session: [6:7577789452073579491:2049]
2025-11-28T14:53:15.549959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789452073579491:2049] [node 2] ICS09 handshake done sender: [6:7577789452073579475:2115] self: [6:7577789452073579481:0] peer: [2:7577789453305823387:0] socket: 348 qp: -1
2025-11-28T14:53:15.549996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789452073579491:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:15.550044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:15.551448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579490:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
2025-11-28T14:53:15.552795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789452073579480:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:15.552949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789452073579461:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.553017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:15.553076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789452073579461:2064] [node 4] ICP22 created new session: [6:7577789452073579493:2064]
2025-11-28T14:53:15.553123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789452073579493:2064] [node 4] ICS09 handshake done sender: [6:7577789452073579480:2117] self: [6:7577789452073579487:0] peer: [4:7577789449877254600:0] socket: 376 qp: -1
2025-11-28T14:53:15.553165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789452073579493:2064] [node 4] ICS10 traffic start
2025-11-28T14:53:15.553199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:15.562810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789449877254616:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.562869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789449877254552:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.562915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:15.562972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789449877254552:2063] [node 6] ICP22 created new session: [4:7577789449877254625:2063]
2025-11-28T14:53:15.563019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789449877254625:2063] [node 6] ICS09 handshake done sender: [4:7577789449877254601:2112] self: [4:7577789449877254600:0] peer: [6:7577789452073579487:0] socket: 372 qp: -1
2025-11-28T14:53:15.563052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789449877254625:2063] [node 6] ICS10 traffic start
2025-11-28T14:53:15.563087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:15.565332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789449877254612:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.546376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789453789274761:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.546438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:15.546491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789453789274761:2065] [node 4] ICP22 created new session: [3:7577789453789274790:2065]
2025-11-28T14:53:15.546539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789453789274790:2065] [node 4] ICS09 handshake done sender: [3:7577789453789274769:2111] self: [3:7577789453789274768:0] peer: [4:7577789449877254614:0] socket: 345 qp: -1
2025-11-28T14:53:15.546575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789453789274790:2065] [node 4] ICS10 traffic start
2025-11-28T14:53:15.546613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:15.579500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789452073579488:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.579659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789447778612106:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.567643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789453305823391:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:15.572695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789449877254605:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:15.580991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789449010856051:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.581053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:15.581108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789449010856051:2072] [node 6] ICP22 created new session: [2:7577789453305823413:2072]
2025-11-28T14:53:15.581156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789453305823413:2072] [node 6] ICS09 handshake done sender: [2:7577789453305823388:2111] self: [2:7577789453305823387:0] peer: [6:7577789452073579481:0] socket: 341 qp: -1
2025-11-28T14:53:15.581192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789453305823413:2072] [node 6] ICS10 traffic start
2025-11-28T14:53:15.581227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:15.580206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789449877254552:2063] [node 6] ICP18 handshake (actor [4:7577789449877254616:2119]) is too old
2025-11-28T14:53:15.580463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789449877254624:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57718
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.582787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789449846097643:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.583972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789445551130258:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.584073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789449846097646:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16435
2025-11-28T14:53:15.585736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789453789274773:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.585779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789453789274773:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.586471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789453789274773:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:15.589640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789453305823406:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.589719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789453305823389:2076] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.589793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:15.589848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789453305823389:2076] [node 4] ICP22 created new session: [2:7577789453305823415:2076]
2025-11-28T14:53:15.589897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789453305823415:2076] [node 4] ICS09 handshake done sender: [2:7577789453305823391:2112] self: [2:7577789453305823390:0] peer: [4:7577789449877254615:0] socket: 342 qp: -1
2025-11-28T14:53:15.589933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789453305823415:2076] [node 4] ICS10 traffic start
2025-11-28T14:53:15.589972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:15.586548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789452697624721:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.595662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789449846097649:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16435
2025-11-28T14:53:15.586581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789452697624721:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.596070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789452697624721:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:15.598755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789453789274765:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.598846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:15.598902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789453789274765:2074] [node 6] ICP22 created new session: [3:7577789453789274794:2074]
2025-11-28T14:53:15.598957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789453789274794:2074] [node 6] ICS09 handshake done sender: [3:7577789453789274773:2113] self: [3:7577789453789274772:0] peer: [6:7577789452073579495:0] socket: 346 qp: -1
2025-11-28T14:53:15.598993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789453789274794:2074] [node 6] ICS10 traffic start
2025-11-28T14:53:15.599029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:15.600939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789452697624711:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.601001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:15.601042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789452697624711:2097] [node 5] ICP22 created new session: [1:7577789452697624744:2097]
2025-11-28T14:53:15.601083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789452697624744:2097] [node 5] ICS09 handshake done sender: [1:7577789452697624721:2114] self: [1:7577789452697624720:0] peer: [5:7577789449846097648:0] socket: 389 qp: -1
2025-11-28T14:53:15.601117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789452697624744:2097] [node 5] ICS10 traffic start
2025-11-28T14:53:15.601152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:15.603770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789445582287237:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.603827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:15.603869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789445582287237:2050] [node 3] ICP22 created new session: [4:7577789449877254628:2050]
2025-11-28T14:53:15.603914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789449877254628:2050] [node 3] ICS09 handshake done sender: [4:7577789449877254605:2115] self: [4:7577789449877254614:0] peer: [3:7577789453789274768:0] socket: 366 qp: -1
2025-11-28T14:53:15.603950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789449877254628:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:15.603985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:15.606422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789449877254570:2066] [node 5] ICP18 handshake (actor [4:7577789449877254612:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.608078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579496:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
2025-11-28T14:53:15.608233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579497:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.610493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789453305823412:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40012
2025-11-28T14:53:15.610799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789449010856024:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.611546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789445582287235:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9150) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789449877254606:2116] held: no
2025-11-28T14:53:15.611893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789452073579488:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:15.612108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789447778612106:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.612175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:15.612221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789447778612106:2050] [node 3] ICP22 created new session: [6:7577789452073579498:2050]
2025-11-28T14:53:15.612262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789452073579498:2050] [node 3] ICS09 handshake done sender: [6:7577789452073579488:2119] self: [6:7577789452073579495:0] peer: [3:7577789453789274772:0] socket: 393 qp: -1
2025-11-28T14:53:15.612296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789452073579498:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:15.612329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:15.616891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789452697624719:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.616925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789452697624719:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:15.618285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789449846097646:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.620743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789453305823419:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40012
2025-11-28T14:53:15.621996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789452697624719:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:15.622450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789452697624739:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.622503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789448402657350:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.622552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:15.622602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789448402657350:2048] [node 2] ICP22 created new session: [1:7577789452697624747:2048]
2025-11-28T14:53:15.622648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789452697624747:2048] [node 2] ICS09 handshake done sender: [1:7577789452697624719:2113] self: [1:7577789452697624718:0] peer: [2:7577789453305823418:0] socket: 382 qp: -1
2025-11-28T14:53:15.622678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789452697624747:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:15.622714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:15.623925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789448402657350:2048] [node 2] ICP18 handshake (actor [1:7577789452697624739:2118]) is too old
2025-11-28T14:53:15.624886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789445551130258:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9150) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789449846097643:2120] held: no
2025-11-28T14:53:15.625564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789449846097643:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:15.627780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789453305823406:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:15.628328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789445551130258:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.628710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789449010856024:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.628777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:15.628830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789449010856024:2048] [node 1] ICP22 created new session: [2:7577789453305823420:2048]
2025-11-28T14:53:15.628873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789453305823420:2048] [node 1] ICS09 handshake done sender: [2:7577789453305823406:2118] self: [2:7577789453305823418:0] peer: [1:7577789452697624718:0] socket: 384 qp: -1
2025-11-28T14:53:15.628911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789453305823420:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:15.628940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:15.632451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789452073579496:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.632780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789452073579433:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.633638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:15.633704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789445551130258:2048] [node 1] ICP22 created new session: [5:7577789449846097650:2048]
2025-11-28T14:53:15.633759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789449846097650:2048] [node 1] ICS09 handshake done sender: [5:7577789449846097643:2120] self: [5:7577789449846097648:0] peer: [1:7577789452697624720:0] socket: 400 qp: -1
2025-11-28T14:53:15.633796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789449846097650:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:15.633834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:15.634924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789445551130260:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.635066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789449846097628:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.635086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789449846097628:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.650303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789449846097628:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:15.651805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789453789274775:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.651855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789453789274775:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.651312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789449877254606:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:15.651663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789452073579501:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32305
2025-11-28T14:53:15.657340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789449846097618:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.657430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:15.657476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789449846097618:2070] [node 6] ICP22 created new session: [5:7577789449846097654:2070]
2025-11-28T14:53:15.657523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789449846097654:2070] [node 6] ICS09 handshake done sender: [5:7577789449846097628:2114] self: [5:7577789449846097627:0] peer: [6:7577789452073579500:0] socket: 388 qp: -1
2025-11-28T14:53:15.657560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789449846097654:2070] [node 6] ICS10 traffic start
2025-11-28T14:53:15.657596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:15.655826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789453789274796:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25704
2025-11-28T14:53:15.656006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789453789274797:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25704
2025-11-28T14:53:15.660409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789452073579496:2122] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:15.660670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789452073579433:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.660735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:15.660787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789452073579433:2063] [node 5] ICP22 created new session: [6:7577789452073579502:2063]
2025-11-28T14:53:15.660840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789452073579502:2063] [node 5] ICS09 handshake done sender: [6:7577789452073579496:2122] self: [6:7577789452073579500:0] peer: [5:7577789449846097627:0] socket: 408 qp: -1
2025-11-28T14:53:15.660880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789452073579502:2063] [node 5] ICS10 traffic start
2025-11-28T14:53:15.660916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:15.663512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789453789274775:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:15.664139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789449877254604:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:15.664570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789453305823412:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.665887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789445582287235:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.665942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:15.665988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789445582287235:2048] [node 1] ICP22 created new session: [4:7577789449877254630:2048]
2025-11-28T14:53:15.666043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789449877254630:2048] [node 1] ICS09 handshake done sender: [4:7577789449877254606:2116] self: [4:7577789449877254623:0] peer: [1:7577789452697624716:0] socket: 367 qp: -1
2025-11-28T14:53:15.666074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789449877254630:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:15.666111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:15.666941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789445582287236:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.666992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:15.667035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789445582287236:2049] [node 2] ICP22 created new session: [4:7577789449877254632:2049]
2025-11-28T14:53:15.667077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789449877254632:2049] [node 2] ICS09 handshake done sender: [4:7577789449877254604:2114] self: [4:7577789449877254615:0] peer: [2:7577789453305823390:0] socket: 365 qp: -1
2025-11-28T14:53:15.667112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789449877254632:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:15.667153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:15.664498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789453789274725:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.664583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:15.664626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789453789274725:2064] [node 5] ICP22 created new session: [3:7577789453789274799:2064]
2025-11-28T14:53:15.664672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789453789274799:2064] [node 5] ICS09 handshake done sender: [3:7577789453789274775:2114] self: [3:7577789453789274774:0] peer: [5:7577789449846097652:0] socket: 349 qp: -1
2025-11-28T14:53:15.664703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789453789274799:2064] [node 5] ICS10 traffic start
2025-11-28T14:53:15.664866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:15.664912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789453305823389:2076] [node 4] ICP18 handshake (actor [2:7577789453305823412:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.672012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789449846097656:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16435
2025-11-28T14:53:15.674674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789453789274796:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.674885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789449494307405:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:15.675329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789453305823395:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:15.675364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789453305823395:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:15.676544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789453789274802:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25704
2025-11-28T14:53:15.677743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789453305823395:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:15.678224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789453789274796:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:15.679846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789449010856025:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:15.679937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:15.679992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789449010856025:2049] [node 3] ICP22 created new session: [2:7577789453305823423:2049]
2025-11-28T14:53:15.680038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789453305823423:2049] [node 3] ICS09 handshake done sender: [2:7577789453305823395:2114] self: [2:7577789453305823394:0] peer: [3:7577789453789274801:0] socket: 363 qp: -1
2025-11-28T14:53:15.680075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789453305823423:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:15.680115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:15.681838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789449494307405:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.681905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:15.681952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789449494307405:2049] [node 2] ICP22 created new session: [3:7577789453789274803:2049]
2025-11-28T14:53:15.682002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789453789274803:2049] [node 2] ICS09 handshake done sender: [3:7577789453789274796:2119] self: [3:7577789453789274801:0] peer: [2:7577789453305823394:0] socket: 350 qp: -1
2025-11-28T14:53:15.682039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789453789274803:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:15.682092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:15.685325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789445551130260:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26724) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789449846097646:2121] held: no
2025-11-28T14:53:15.685988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789449846097646:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:15.689706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789445551130260:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:15.689785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:15.689839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789445551130260:2050] [node 3] ICP22 created new session: [5:7577789449846097657:2050]
2025-11-28T14:53:15.689892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789449846097657:2050] [node 3] ICS09 handshake done sender: [5:7577789449846097646:2121] self: [5:7577789449846097652:0] peer: [3:7577789453789274774:0] socket: 338 qp: -1
2025-11-28T14:53:15.689933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789449846097657:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:15.689968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:15.717109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:15.717326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789453305823420:2048] [node 1] ICS07 socket disconnect 384 reason# EndOfStream
2025-11-28T14:53:15.717361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789453305823420:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.717468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:15.717509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789453305823420:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:15.722633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789449010856024:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9150) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:15.722677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789453305823420:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:15.732543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:15.736712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789449846097650:2048] [node 1] ICS01 socket: 400 reason# EndOfStream
2025-11-28T14:53:15.736772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789445551130258:2048] [node 1] ICP30 unregister session Session# [5:7577789449846097650:2048] VirtualId# [5:7577789449846097648:0]
2025-11-28T14:53:15.736813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789449846097650:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.736920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:15.736982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:15.732565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:15.737955Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:15.732478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:15.749686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789449010856024:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:15.749756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789453305823420:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:15.749802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789449010856024:2048] [node 1] ICP30 unregister session Session# [2:7577789453305823420:2048] VirtualId# [2:7577789453305823418:0]
2025-11-28T14:53:15.749842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:15.749875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789449010856024:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.749896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.744788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789449877254630:2048] [node 1] ICS01 socket: 367 reason# EndOfStream
2025-11-28T14:53:15.744984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789453789274786:2048] [node 1] ICS07 socket disconnect 368 reason# EndOfStream
2025-11-28T14:53:15.745021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789453789274786:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.745163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:15.745209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789453789274786:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:15.749327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789445582287235:2048] [node 1] ICP30 unregister session Session# [4:7577789449877254630:2048] VirtualId# [4:7577789449877254623:0]
2025-11-28T14:53:15.749375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789449877254630:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.749484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:15.749530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:15.746174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789452073579484:2048] [node 1] ICS01 socket: 373 reason# EndOfStream
2025-11-28T14:53:15.760831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789449494307404:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:15.760910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789453789274786:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:15.760963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789449494307404:2048] [node 1] ICP30 unregister session Session# [3:7577789453789274786:2048] VirtualId# [3:7577789453789274784:0]
2025-11-28T14:53:15.761017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:15.761054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789449494307404:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.761071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.746218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789447778612104:2048] [node 1] ICP30 unregister session Session# [6:7577789452073579484:2048] VirtualId# [6:7577789452073579482:0]
2025-11-28T14:53:15.746252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789452073579484:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.746330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:15.746373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:15.761977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789449010856024:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:15.774631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789449494307404:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:15.821287Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:15.821551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789453789274803:2049] [node 2] ICS01 socket: 350 reason# EndOfStream
2025-11-28T14:53:15.821613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789449494307405:2049] [node 2] ICP30 unregister session Session# [3:7577789453789274803:2049] VirtualId# [3:7577789453789274801:0]
2025-11-28T14:53:15.821647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789453789274803:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.821726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:15.821764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:15.828154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:15.828429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789449877254632:2049] [node 2] ICS07 socket disconnect 365 reason# EndOfStream
2025-11-28T14:53:15.828466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789449877254632:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.828565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:15.828619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789449877254632:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:15.828230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:15.828450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:15.852773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789445582287236:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:15.852841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789449877254632:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:15.852877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789445582287236:2049] [node 2] ICP30 unregister session Session# [4:7577789449877254632:2049] VirtualId# [4:7577789449877254615:0]
2025-11-28T14:53:15.852927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:15.852970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789445582287236:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.852987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.828469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789449846097639:2049] [node 2] ICS07 socket disconnect 343 reason# EndOfStream
2025-11-28T14:53:15.828497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789449846097639:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.828593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:15.828629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789449846097639:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:15.837728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789445551130259:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:15.837782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789449846097639:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:15.837818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789445551130259:2049] [node 2] ICP30 unregister session Session# [5:7577789449846097639:2049] VirtualId# [5:7577789449846097635:0]
2025-11-28T14:53:15.837869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:15.837908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789445551130259:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.837923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.828641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789452073579491:2049] [node 2] ICS07 socket disconnect 348 reason# EndOfStream
2025-11-28T14:53:15.828676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789452073579491:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.828736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:15.828782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789452073579491:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:15.854272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789445551130259:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:15.855363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789447778612105:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:15.855432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789452073579491:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:15.855473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789447778612105:2049] [node 2] ICP30 unregister session Session# [6:7577789452073579491:2049] VirtualId# [6:7577789452073579481:0]
2025-11-28T14:53:15.855515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:15.855545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789447778612105:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.855560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.864653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789445582287236:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:15.864930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789447778612105:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:15.908607Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:15.908836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789449846097657:2050] [node 3] ICS01 socket: 338 reason# EndOfStream
2025-11-28T14:53:15.908875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789445551130260:2050] [node 3] ICP30 unregister session Session# [5:7577789449846097657:2050] VirtualId# [5:7577789449846097652:0]
2025-11-28T14:53:15.908909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789449846097657:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.909000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:15.909045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:15.909145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:15.909324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789452073579498:2050] [node 3] ICS07 socket disconnect 393 reason# EndOfStream
2025-11-28T14:53:15.909357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789452073579498:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.909414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:15.909448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789452073579498:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:15.911176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789447778612106:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26724) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:15.911251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789452073579498:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:15.911285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789447778612106:2050] [node 3] ICP30 unregister session Session# [6:7577789452073579498:2050] VirtualId# [6:7577789452073579495:0]
2025-11-28T14:53:15.911323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:15.911354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789447778612106:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26724) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.911370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26724) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.912968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:15.913189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789449877254628:2050] [node 3] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-28T14:53:15.913219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789449877254628:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:15.913308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:15.913349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789449877254628:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:15.921381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789447778612106:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:15.921491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789445582287237:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26724) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:15.921586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789449877254628:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:15.921660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789445582287237:2050] [node 3] ICP30 unregister session Session# [4:7577789449877254628:2050] VirtualId# [4:7577789449877254614:0]
2025-11-28T14:53:15.921761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:15.921836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789445582287237:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26724) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:15.921860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26724) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:16.124834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:16.125056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789449846097644:2069] [node 4] ICS07 socket disconnect 377 reason# EndOfStream
2025-11-28T14:53:16.125095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789449846097644:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:16.125197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:16.125244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789449846097644:2069] [node 4] ICS15 start handshake
2025-11-28T14:53:16.124730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:16.125004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789452073579493:2064] [node 4] ICS07 socket disconnect 376 reason# EndOfStream
2025-11-28T14:53:16.125058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789452073579493:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:16.125198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:16.125243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789452073579493:2064] [node 4] ICS15 start handshake
2025-11-28T14:53:16.126306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789449846097612:2069] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30433) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:16.126366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789449846097644:2069] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:16.126422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789449846097612:2069] [node 4] ICP30 unregister session Session# [5:7577789449846097644:2069] VirtualId# [5:7577789449846097636:0]
2025-11-28T14:53:16.126464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:16.126522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789449846097612:2069] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:16.126540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:16.126579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789452073579461:2064] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30433) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:16.126629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789452073579493:2064] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:16.126670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789452073579461:2064] [node 4] ICP30 unregister session Session# [6:7577789452073579493:2064] VirtualId# [6:7577789452073579487:0]
2025-11-28T14:53:16.126703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:16.126733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789452073579461:2064] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:16.126747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:16.144618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789452073579461:2064] [node 4] ICP33 wake up from error state
2025-11-28T14:53:16.229321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:16.229567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789452073579502:2063] [node 5] ICS01 socket: 408 reason# EndOfStream
2025-11-28T14:53:16.229617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789452073579433:2063] [node 5] ICP30 unregister session Session# [6:7577789452073579502:2063] VirtualId# [6:7577789452073579500:0]
2025-11-28T14:53:16.229656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789452073579502:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:16.229730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:16.229769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:16.764812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789457792386896:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:16.764876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789457792386897:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:16.850441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789454644130041:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:16.850509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789454644130042:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:16.944950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789457622810554:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:16.945006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789457622810555:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:17.068712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789458863551167:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:17.068756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789458863551168:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:17.068801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789458863551169:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:17.152703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789460559659288:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:17.152755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789460559659289:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:17.152789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789460559659290:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:17.560902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789458939097372:2074] [node 4] ICP01 ready to work
2025-11-28T14:53:17.561325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789458939097373:2078] [node 6] ICP01 ready to work
2025-11-28T14:53:17.624824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789460559659329:2063] [node 4] ICP01 ready to work
2025-11-28T14:53:17.200870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789461315683860:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:17.679472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789458863551201:2066] [node 6] ICP01 ready to work
2025-11-28T14:53:17.700685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789462087354255:2104] [node 4] ICP01 ready to work
2025-11-28T14:53:17.200925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789461315683861:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:17.700424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789461917777913:2074] [node 4] ICP01 ready to work
2025-11-28T14:53:17.709079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789458939097404:2069] [node 5] ICP01 ready to work
2025-11-28T14:53:17.200959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789461315683862:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:17.701296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789462087354256:2105] [node 5] ICP01 ready to work
2025-11-28T14:53:17.689762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789458863551167:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(249970568ns) {Configure:104(249964897ns) {ProcessConfigured:135(8633831ns) {StartInitialHandshake:157(8630559ns) {DropHandshakes:470(2671ns) {DropIncomingHandshake:442(711ns) DropOutgoingHandshake:457(436ns)} GenerateSessionVirtualId:988(842ns) PrepareNewSessionHandshake:477(4238ns) {SwitchToState:179(2500ns)}}}}}
2025-11-28T14:53:17.736663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789460559659352:2064] [node 6] ICP01 ready to work
2025-11-28T14:53:17.725060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789461917777914:2076] [node 5] ICP01 ready to work
2025-11-28T14:53:17.690272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789458863551233:2067] [node 5] ICP01 ready to work
2025-11-28T14:53:17.713365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789461315683926:2083] [node 4] ICP01 ready to work
2025-11-28T14:53:17.749661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789458939097372:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(156000008ns) {Configure:104(155995073ns) {ProcessConfigured:135(22612758ns) {StartInitialHandshake:157(22610558ns) {DropHandshakes:470(1991ns) {DropIncomingHandshake:442(671ns) DropOutgoingHandshake:457(421ns)} GenerateSessionVirtualId:988(445ns) PrepareNewSessionHandshake:477(2835ns) {SwitchToState:179(203ns)}}}}}
2025-11-28T14:53:17.728891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789457792386896:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26507386ns) {Configure:104(26503171ns) {ProcessConfigured:135(24230532ns) {StartInitialHandshake:157(24227317ns) {DropHandshakes:470(24121479ns) {DropIncomingHandshake:442(1223ns) DropOutgoingHandshake:457(4700ns)} GenerateSessionVirtualId:988(2849ns) PrepareNewSessionHandshake:477(3446ns) {SwitchToState:179(1110ns)}}}}}
2025-11-28T14:53:17.738484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789458863551169:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42289530ns) {Configure:104(42285303ns) {ProcessConfigured:135(82356ns) {StartInitialHandshake:157(79505ns) {DropHandshakes:470(2740ns) {DropIncomingHandshake:442(700ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(790ns) PrepareNewSessionHandshake:477(1454ns) {SwitchToState:179(564ns)}}}}}
2025-11-28T14:53:17.728451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789462087354257:2103] [node 6] ICP01 ready to work
2025-11-28T14:53:17.757722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789461315683929:2069] [node 5] ICP01 ready to work
2025-11-28T14:53:17.785899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789461315683860:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(69796829ns) {Configure:104(69791129ns) {ProcessConfigured:135(89881ns) {StartInitialHandshake:157(86165ns) {DropHandshakes:470(2506ns) {DropIncomingHandshake:442(744ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(993ns) PrepareNewSessionHandshake:477(9644ns) {SwitchToState:179(894ns)}}}}}
2025-11-28T14:53:17.764664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789461917777915:2080] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.784904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789457792386897:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20041339ns) {Configure:104(20033781ns) {ProcessConfigured:135(18081940ns) {StartInitialHandshake:157(18077419ns) {DropHandshakes:470(3707ns) {DropIncomingHandshake:442(1117ns) DropOutgoingHandshake:457(611ns)} GenerateSessionVirtualId:988(1184ns) PrepareNewSessionHandshake:477(9521ns) {SwitchToState:179(5776ns)}}}}}
2025-11-28T14:53:17.794196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789457622810555:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44937786ns) {Configure:104(44929190ns) {ProcessConfigured:135(88847ns) {StartInitialHandshake:157(85832ns) {DropHandshakes:470(3207ns) {DropIncomingHandshake:442(1052ns) DropOutgoingHandshake:457(372ns)} GenerateSessionVirtualId:988(759ns) PrepareNewSessionHandshake:477(2704ns) {SwitchToState:179(990ns)}}}}}
2025-11-28T14:53:17.856193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789460559659353:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55150
2025-11-28T14:53:17.856794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789460559659362:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55150
2025-11-28T14:53:17.876729Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789461315683929:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(89823086ns) {Configure:104(89817082ns) {ProcessConfigured:135(18744390ns) {StartInitialHandshake:157(18739813ns) {DropHandshakes:470(4129101ns) {DropIncomingHandshake:442(3255ns) DropOutgoingHandshake:457(541ns)} GenerateSessionVirtualId:988(2011ns) PrepareNewSessionHandshake:477(5258ns) {SwitchToState:179(2070ns)}}}}}
2025-11-28T14:53:17.880411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789462087354255:2104] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40813500ns) {Configure:104(40806930ns) {ProcessConfigured:135(39077033ns) {StartInitialHandshake:157(39074326ns) {DropHandshakes:470(2800ns) {DropIncomingHandshake:442(858ns) DropOutgoingHandshake:457(615ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:477(5568ns) {SwitchToState:179(2097ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.877637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789460559659288:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(98091615ns) {Configure:104(98084860ns) {ProcessConfigured:135(8498193ns) {StartInitialHandshake:157(8494849ns) {DropHandshakes:470(3087ns) {DropIncomingHandshake:442(694ns) DropOutgoingHandshake:457(385ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:477(3224ns) {SwitchToState:179(2163ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.881361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789462087354266:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17510
2025-11-28T14:53:17.797219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789461917777914:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28974375ns) {Configure:104(28970191ns) {ProcessConfigured:135(59784ns) {StartInitialHandshake:157(56967ns) {DropHandshakes:470(2652ns) {DropIncomingHandshake:442(559ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(668ns) PrepareNewSessionHandshake:477(1280ns) {SwitchToState:179(410ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.929556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789458863551246:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16651
2025-11-28T14:53:17.890794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789461917777927:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62257
2025-11-28T14:53:17.897347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789462087354256:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(140236952ns) {Configure:104(140228868ns) {ProcessConfigured:135(82839ns) {StartInitialHandshake:157(79953ns) {DropHandshakes:470(3314ns) {DropIncomingHandshake:442(830ns) DropOutgoingHandshake:457(482ns)} GenerateSessionVirtualId:988(1208ns) PrepareNewSessionHandshake:477(2023ns) {SwitchToState:179(979ns)}}}}}
2025-11-28T14:53:17.948900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789462087354266:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:17.909010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789454644130041:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(199599753ns) {Configure:104(199594490ns) {ProcessConfigured:135(87272ns) {StartInitialHandshake:157(83169ns) {DropHandshakes:470(8552ns) {DropIncomingHandshake:442(840ns) DropOutgoingHandshake:457(416ns)} GenerateSessionVirtualId:988(1096ns) PrepareNewSessionHandshake:477(1834ns) {SwitchToState:179(634ns)}}}}}
2025-11-28T14:53:17.948910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789460559659353:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:17.972761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789460559659362:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:17.980561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789460559659329:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:17.944826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789458863551247:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16651
2025-11-28T14:53:17.890859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789461917777928:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62257
2025-11-28T14:53:17.986765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789460559659290:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.994481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789461315683932:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.913847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789458939097409:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22905
2025-11-28T14:53:17.962282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789458863551246:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:17.959621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789461917777929:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62257
2025-11-28T14:53:17.913944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789458939097410:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22905
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.017000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789461315683926:2083] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35098641ns) {Configure:104(35091839ns) {ProcessConfigured:135(16085802ns) {StartInitialHandshake:157(16082124ns) {DropHandshakes:470(2704ns) {DropIncomingHandshake:442(740ns) DropOutgoingHandshake:457(358ns)} GenerateSessionVirtualId:988(853ns) PrepareNewSessionHandshake:477(3244ns) {SwitchToState:179(1558ns)}}}}}
2025-11-28T14:53:18.017531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789461315683932:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.017848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789461315683935:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-28T14:53:18.018193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789461315683926:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.975635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789461917777929:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:17.980695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789458863551247:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.037145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789466382321570:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17510
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.914168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789458939097411:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22905
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.053266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789461315683935:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.053861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789461315683860:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.053942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789465610651237:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-28T14:53:17.984706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789458863551168:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.028838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789464854626666:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55150
2025-11-28T14:53:17.980645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789461917777927:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:17.996569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789458863551237:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:17.996579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789458863551245:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16651
2025-11-28T14:53:17.996602Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789458863551237:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.019894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789463158518546:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16651
2025-11-28T14:53:18.028716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789458863551245:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.047740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789458863551235:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.047765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789458863551235:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.048834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789458863551169:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.063418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789458863551237:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:18.069365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789458863551235:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:18.068938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789464854626667:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55150
2025-11-28T14:53:17.914298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789458939097412:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22905
2025-11-28T14:53:18.043112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789462087354261:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:17.997703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789457622810555:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.009074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789457622810554:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(214479296ns) {Configure:104(214473473ns) {ProcessConfigured:135(20384258ns) {StartInitialHandshake:157(20380509ns) {DropHandshakes:470(2123603ns) {DropIncomingHandshake:442(1640ns) DropOutgoingHandshake:457(561ns)} GenerateSessionVirtualId:988(1677ns) PrepareNewSessionHandshake:477(5201ns) {SwitchToState:179(2301ns)}}}}}
2025-11-28T14:53:18.041054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789461917777928:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.041470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789457622810554:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.041572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789466212745230:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62257
2025-11-28T14:53:18.047528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789461917777921:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.047573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789461917777921:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.052700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789466212745232:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62257
2025-11-28T14:53:18.064379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789466212745230:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.073665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789461917777928:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:18.073890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789461917777917:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.073926Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789461917777917:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.078067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789458863551233:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.078153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:18.078203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789458863551233:2067] [node 5] ICP22 created new session: [4:7577789463158518551:2067]
2025-11-28T14:53:18.078254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789463158518551:2067] [node 5] ICS09 handshake done sender: [4:7577789458863551237:2112] self: [4:7577789458863551236:0] peer: [5:7577789460559659368:0] socket: 345 qp: -1
2025-11-28T14:53:18.078297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789463158518551:2067] [node 5] ICS10 traffic start
2025-11-28T14:53:18.078345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:18.079516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789458863551247:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:18.085056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789461917777921:2112] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:18.043164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789462087354261:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.093479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789458863551201:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.093555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:18.093609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789458863551201:2066] [node 6] ICP22 created new session: [4:7577789463158518554:2066]
2025-11-28T14:53:18.093663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789463158518554:2066] [node 6] ICS09 handshake done sender: [4:7577789458863551235:2111] self: [4:7577789458863551234:0] peer: [6:7577789465610651236:0] socket: 344 qp: -1
2025-11-28T14:53:18.093697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789463158518554:2066] [node 6] ICS10 traffic start
2025-11-28T14:53:18.093745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:18.094877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789458863551168:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.094935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:18.094980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789458863551168:2049] [node 2] ICP22 created new session: [4:7577789463158518556:2049]
2025-11-28T14:53:18.095036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789463158518556:2049] [node 2] ICS09 handshake done sender: [4:7577789458863551247:2117] self: [4:7577789458863551249:0] peer: [2:7577789458939097405:0] socket: 375 qp: -1
2025-11-28T14:53:18.095072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789463158518556:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:18.095104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:17.965417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789458939097373:2078] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(215434421ns) {Configure:104(215429713ns) {ProcessConfigured:135(88449ns) {StartInitialHandshake:157(86061ns) {DropHandshakes:470(2970ns) {DropIncomingHandshake:442(711ns) DropOutgoingHandshake:457(434ns)} GenerateSessionVirtualId:988(1098ns) PrepareNewSessionHandshake:477(2485ns) {SwitchToState:179(824ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.110272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789460559659329:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2881) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789460559659353:2110] held: no
2025-11-28T14:53:18.110997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789460559659353:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:18.111577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789460559659329:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.111648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:18.111701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789460559659329:2063] [node 4] ICP22 created new session: [5:7577789464854626668:2063]
2025-11-28T14:53:18.111757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789464854626668:2063] [node 4] ICS09 handshake done sender: [5:7577789460559659353:2110] self: [5:7577789460559659368:0] peer: [4:7577789458863551236:0] socket: 346 qp: -1
2025-11-28T14:53:18.111789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789464854626668:2063] [node 4] ICS10 traffic start
2025-11-28T14:53:18.111831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.044083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789462087354261:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.965850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789458939097411:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.105189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789463158518550:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16651
2025-11-28T14:53:18.103778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789461917777917:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.044410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789457792386897:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.154325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789463158518553:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16651
2025-11-28T14:53:18.154593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789458863551245:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:18.128641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789461315683932:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:18.125875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789460559659290:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12787) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789460559659362:2115] held: no
2025-11-28T14:53:17.966799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789458939097410:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.164789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789458863551169:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.164882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:18.164935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789458863551169:2050] [node 3] ICP22 created new session: [4:7577789463158518560:2050]
2025-11-28T14:53:18.164978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789463158518560:2050] [node 3] ICS09 handshake done sender: [4:7577789458863551245:2115] self: [4:7577789463158518549:0] peer: [3:7577789461917777916:0] socket: 352 qp: -1
2025-11-28T14:53:18.165013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789463158518560:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:18.165061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:18.112582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789457622810554:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.047407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789462087354272:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17510
2025-11-28T14:53:18.140191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789465610651239:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-28T14:53:18.148880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789461315683926:2083] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.148939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:18.126658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789464854626667:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.184688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789463158518553:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:17.997732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789458939097406:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.193478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789458863551167:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.049142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:18.112636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:18.148994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789461315683926:2083] [node 4] ICP22 created new session: [6:7577789465610651242:2083]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.200687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789463158518563:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16651
2025-11-28T14:53:18.149045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789465610651242:2083] [node 4] ICS09 handshake done sender: [6:7577789461315683932:2113] self: [6:7577789465610651236:0] peer: [4:7577789458863551234:0] socket: 347 qp: -1
2025-11-28T14:53:18.149082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789465610651242:2083] [node 4] ICS10 traffic start
2025-11-28T14:53:18.149136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:18.151144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789465610651240:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-28T14:53:18.151296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789465610651241:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-28T14:53:18.151914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789461315683935:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:18.157434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789461315683860:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.157484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:18.180660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789461315683860:2048] [node 1] ICP22 created new session: [6:7577789465610651245:2048]
2025-11-28T14:53:18.185676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789465610651245:2048] [node 1] ICS09 handshake done sender: [6:7577789461315683935:2115] self: [6:7577789465610651238:0] peer: [1:7577789462087354262:0] socket: 378 qp: -1
2025-11-28T14:53:18.185718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789465610651245:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:18.185759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:18.196380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789465610651241:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.196490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789465610651240:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.198642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789461315683929:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.198715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789461315683861:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.198792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789465610651247:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:17.997766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789458939097406:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.127185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789464854626671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55150
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.112686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789457622810554:2048] [node 1] ICP22 created new session: [3:7577789466212745237:2048]
2025-11-28T14:53:18.238414Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789461315683860:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(80134654ns) {HandleHandshakeStatus:329(80131985ns) {DropIncomingHandshake:442(1674ns) DropOutgoingHandshake:457(4881ns) SwitchToState:179(1004ns) ProcessPendingSessionEvents:495(51685737ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 51611351ns max# 27938306ns}}}
2025-11-28T14:53:18.060711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789457792386897:2049] [node 3] ICP22 created new session: [1:7577789466382321572:2049]
2025-11-28T14:53:18.004643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789458939097406:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:18.245511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789461315683861:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31065) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789465610651240:2119] held: no
2025-11-28T14:53:18.127656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789460559659289:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.248422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789463158518553:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:18.112741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789466212745237:2048] [node 1] ICS09 handshake done sender: [3:7577789461917777928:2115] self: [3:7577789466212745231:0] peer: [1:7577789462087354260:0] socket: 365 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.075393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789466382321572:2049] [node 3] ICS09 handshake done sender: [1:7577789462087354261:2111] self: [1:7577789462087354260:0] peer: [3:7577789466212745231:0] socket: 363 qp: -1
2025-11-28T14:53:18.157129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789460559659362:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:18.194208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789460559659290:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.194265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:18.272705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789458863551167:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26180) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789463158518553:2120] held: no
2025-11-28T14:53:18.273320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789458863551167:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.273366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:18.273410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789458863551167:2048] [node 1] ICP22 created new session: [4:7577789463158518564:2048]
2025-11-28T14:53:18.273456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789463158518564:2048] [node 1] ICS09 handshake done sender: [4:7577789463158518553:2120] self: [4:7577789463158518562:0] peer: [1:7577789462087354264:0] socket: 358 qp: -1
2025-11-28T14:53:18.273488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789463158518564:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:18.273523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:18.028482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789458939097412:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.112776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789466212745237:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:18.112810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:18.114014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789461917777914:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.114068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:18.114108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789461917777914:2076] [node 5] ICP22 created new session: [3:7577789466212745239:2076]
2025-11-28T14:53:18.114156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789466212745239:2076] [node 5] ICS09 handshake done sender: [3:7577789461917777921:2112] self: [3:7577789461917777920:0] peer: [5:7577789460559659369:0] socket: 351 qp: -1
2025-11-28T14:53:18.114193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789466212745239:2076] [node 5] ICS10 traffic start
2025-11-28T14:53:18.114221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:18.140814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789466212745236:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62257
2025-11-28T14:53:18.181213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789461917777913:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.181284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:18.181345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789461917777913:2074] [node 4] ICP22 created new session: [3:7577789466212745242:2074]
2025-11-28T14:53:18.181399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789466212745242:2074] [node 4] ICS09 handshake done sender: [3:7577789461917777917:2110] self: [3:7577789461917777916:0] peer: [4:7577789463158518549:0] socket: 349 qp: -1
2025-11-28T14:53:18.181440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789466212745242:2074] [node 4] ICS10 traffic start
2025-11-28T14:53:18.181475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:18.183452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789457622810555:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31065) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789461917777929:2116] held: no
2025-11-28T14:53:18.190344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789461917777929:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:18.225022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789457622810555:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.225098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:18.225155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789457622810555:2049] [node 2] ICP22 created new session: [3:7577789466212745245:2049]
2025-11-28T14:53:18.225196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789466212745245:2049] [node 2] ICS09 handshake done sender: [3:7577789461917777929:2116] self: [3:7577789461917777931:0] peer: [2:7577789458939097374:0] socket: 377 qp: -1
2025-11-28T14:53:18.225231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789466212745245:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:18.225271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:18.230047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789466212745244:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62257
2025-11-28T14:53:18.268712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789465610651250:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-28T14:53:18.075450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789466382321572:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:18.047483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789454644130041:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.056706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789458939097372:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.056773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:18.056842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789458939097372:2074] [node 4] ICP22 created new session: [2:7577789463234064720:2074]
2025-11-28T14:53:18.056895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789463234064720:2074] [node 4] ICS09 handshake done sender: [2:7577789458939097406:2111] self: [2:7577789458939097405:0] peer: [4:7577789458863551249:0] socket: 372 qp: -1
2025-11-28T14:53:18.056937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789463234064720:2074] [node 4] ICS10 traffic start
2025-11-28T14:53:18.056977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:18.069395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789458939097375:2096] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.069434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789458939097375:2096] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.074787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789458939097409:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.084743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789458939097375:2096] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:18.101468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789454644130042:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.101560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:18.101625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789454644130042:2049] [node 3] ICP22 created new session: [2:7577789463234064726:2049]
2025-11-28T14:53:18.101672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789463234064726:2049] [node 3] ICS09 handshake done sender: [2:7577789458939097375:2096] self: [2:7577789458939097374:0] peer: [3:7577789461917777931:0] socket: 371 qp: -1
2025-11-28T14:53:18.101708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789463234064726:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:18.101746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:18.124799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789463234064725:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22905
2025-11-28T14:53:18.145760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789458939097414:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.145796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789458939097414:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.166911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789458939097414:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:18.169693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789458939097412:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:18.205272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789454644130041:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.205340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:18.205402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789454644130041:2048] [node 1] ICP22 created new session: [2:7577789463234064730:2048]
2025-11-28T14:53:18.205454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789463234064730:2048] [node 1] ICS09 handshake done sender: [2:7577789458939097412:2116] self: [2:7577789463234064719:0] peer: [1:7577789462087354258:0] socket: 370 qp: -1
2025-11-28T14:53:18.205490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789463234064730:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:18.205531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:18.206265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789454644130041:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:18.207554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789458939097419:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.207580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789458939097419:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.208018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789458939097404:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.208074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:18.208117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789458939097404:2069] [node 5] ICP22 created new session: [2:7577789463234064732:2069]
2025-11-28T14:53:18.208163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789463234064732:2069] [node 5] ICS09 handshake done sender: [2:7577789458939097414:2117] self: [2:7577789458939097413:0] peer: [5:7577789464854626672:0] socket: 392 qp: -1
2025-11-28T14:53:18.208197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789463234064732:2069] [node 5] ICS10 traffic start
2025-11-28T14:53:18.208233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:18.220124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789458939097419:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:18.220376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789458939097373:2078] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.220461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:18.220517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789458939097373:2078] [node 6] ICP22 created new session: [2:7577789463234064735:2078]
2025-11-28T14:53:18.220575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789463234064735:2078] [node 6] ICS09 handshake done sender: [2:7577789458939097419:2118] self: [2:7577789458939097418:0] peer: [6:7577789465610651249:0] socket: 355 qp: -1
2025-11-28T14:53:18.220607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789463234064735:2078] [node 6] ICS10 traffic start
2025-11-28T14:53:18.220642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:18.194309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789460559659290:2050] [node 3] ICP22 created new session: [5:7577789464854626677:2050]
2025-11-28T14:53:18.194355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789464854626677:2050] [node 3] ICS09 handshake done sender: [5:7577789460559659362:2115] self: [5:7577789460559659369:0] peer: [3:7577789461917777920:0] socket: 354 qp: -1
2025-11-28T14:53:18.194385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789464854626677:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:18.194452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:18.195539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789464854626674:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55150
2025-11-28T14:53:18.208717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789464854626675:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55150
2025-11-28T14:53:18.208857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789464854626676:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55150
2025-11-28T14:53:18.218403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789460559659361:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.218433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789460559659361:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.219042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789464854626667:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:18.241949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789460559659289:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.242014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:18.242063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789460559659289:2049] [node 2] ICP22 created new session: [5:7577789464854626680:2049]
2025-11-28T14:53:18.242109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789464854626680:2049] [node 2] ICS09 handshake done sender: [5:7577789464854626667:2118] self: [5:7577789464854626672:0] peer: [2:7577789458939097413:0] socket: 393 qp: -1
2025-11-28T14:53:18.242143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789464854626680:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:18.242178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:18.242869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789460559659289:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:18.247426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789464854626675:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.247575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789464854626676:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.260705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789460559659361:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:18.264847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789460559659288:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.269962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789460559659352:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.270024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:18.270070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789460559659352:2064] [node 6] ICP22 created new session: [5:7577789464854626685:2064]
2025-11-28T14:53:18.270113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789464854626685:2064] [node 6] ICS09 handshake done sender: [5:7577789460559659361:2114] self: [5:7577789460559659360:0] peer: [6:7577789465610651248:0] socket: 359 qp: -1
2025-11-28T14:53:18.270139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789464854626685:2064] [node 6] ICS10 traffic start
2025-11-28T14:53:18.270192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:18.283643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789464854626684:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55150
2025-11-28T14:53:18.284032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789460559659288:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26180) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789464854626676:2122] held: no
2025-11-28T14:53:18.271635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789465610651251:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-28T14:53:18.304928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789465610651247:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.306042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789465610651241:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:18.309551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789460559659288:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24737703ns) {HandleHandshakeStatus:395(24733290ns) {LogHandshakeFail:472(18414ns) {UpdateErrorStateLog:545(6095ns)} DropOutgoingHandshake:457(749ns) ProcessPendingSessionEvents:495(24661794ns) {unordered DropSessionEvent:512 num# 100 sum# 24618646ns max# 8024979ns}}}
2025-11-28T14:53:18.309956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789464854626676:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:18.075495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:18.077236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789462087354263:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.077273Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789462087354263:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.085679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789462087354259:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.085708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789462087354259:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.086743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789462087354272:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.092713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789462087354263:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:18.156515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789462087354259:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:18.161776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789457792386897:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(116480096ns) {HandleHandshakeStatus:329(116476463ns) {DropIncomingHandshake:442(602ns) DropOutgoingHandshake:457(456ns) SwitchToState:179(6195ns) ProcessPendingSessionEvents:495(83370823ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 83222298ns max# 11929636ns}}}
2025-11-28T14:53:18.161956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789466382321570:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.162016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789462087354257:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.162063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:18.162108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789462087354257:2103] [node 6] ICP22 created new session: [1:7577789466382321578:2103]
2025-11-28T14:53:18.162161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789466382321578:2103] [node 6] ICS09 handshake done sender: [1:7577789462087354263:2112] self: [1:7577789462087354262:0] peer: [6:7577789465610651238:0] socket: 364 qp: -1
2025-11-28T14:53:18.162195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789466382321578:2103] [node 6] ICS10 traffic start
2025-11-28T14:53:18.162240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:18.171569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789457792386896:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.171642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:18.171684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789457792386896:2048] [node 2] ICP22 created new session: [1:7577789466382321581:2048]
2025-11-28T14:53:18.171735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789466382321581:2048] [node 2] ICS09 handshake done sender: [1:7577789462087354259:2110] self: [1:7577789462087354258:0] peer: [2:7577789463234064719:0] socket: 362 qp: -1
2025-11-28T14:53:18.171766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789466382321581:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:18.171812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:18.173148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789462087354257:2103] [node 6] ICP18 handshake (actor [1:7577789466382321570:2117]) is too old
2025-11-28T14:53:18.174546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789466382321580:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17510
2025-11-28T14:53:18.199461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789462087354265:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.199485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789462087354265:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.199952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789466382321580:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.214031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789462087354265:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:18.256983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789462087354255:2104] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.257053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:18.257115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789462087354255:2104] [node 4] ICP22 created new session: [1:7577789466382321586:2104]
2025-11-28T14:53:18.257171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789466382321586:2104] [node 4] ICS09 handshake done sender: [1:7577789462087354265:2113] self: [1:7577789462087354264:0] peer: [4:7577789463158518562:0] socket: 399 qp: -1
2025-11-28T14:53:18.257205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789466382321586:2104] [node 4] ICS10 traffic start
2025-11-28T14:53:18.257255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:18.266269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789462087354271:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.266294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789462087354271:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:18.267099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789462087354271:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:18.267299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789462087354256:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.267362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:18.267410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789462087354256:2105] [node 5] ICP22 created new session: [1:7577789466382321589:2105]
2025-11-28T14:53:18.267458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789466382321589:2105] [node 5] ICS09 handshake done sender: [1:7577789462087354271:2115] self: [1:7577789462087354270:0] peer: [5:7577789464854626683:0] socket: 376 qp: -1
2025-11-28T14:53:18.267492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789466382321589:2105] [node 5] ICS10 traffic start
2025-11-28T14:53:18.267533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:18.320903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789460559659288:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.320983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:18.321045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789460559659288:2048] [node 1] ICP22 created new session: [5:7577789464854626687:2048]
2025-11-28T14:53:18.321105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789464854626687:2048] [node 1] ICS09 handshake done sender: [5:7577789464854626676:2122] self: [5:7577789464854626683:0] peer: [1:7577789462087354270:0] socket: 386 qp: -1
2025-11-28T14:53:18.321155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789464854626687:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:18.321205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:18.339983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789461917777923:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:18.340031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789461917777923:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:18.312635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789461315683929:2069] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28375) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789465610651241:2120] held: no
2025-11-28T14:53:18.313292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789461315683929:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.313344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:18.313398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789461315683929:2069] [node 5] ICP22 created new session: [6:7577789465610651252:2069]
2025-11-28T14:53:18.313451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789465610651252:2069] [node 5] ICS09 handshake done sender: [6:7577789465610651241:2120] self: [6:7577789465610651248:0] peer: [5:7577789460559659360:0] socket: 405 qp: -1
2025-11-28T14:53:18.313491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789465610651252:2069] [node 5] ICS10 traffic start
2025-11-28T14:53:18.313565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:18.314670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789465610651240:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:18.337336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789461315683861:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.337394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:18.337446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789461315683861:2049] [node 2] ICP22 created new session: [6:7577789465610651254:2049]
2025-11-28T14:53:18.337493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789465610651254:2049] [node 2] ICS09 handshake done sender: [6:7577789465610651240:2119] self: [6:7577789465610651249:0] peer: [2:7577789458939097418:0] socket: 368 qp: -1
2025-11-28T14:53:18.337534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789465610651254:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:18.337568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:18.338623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789461315683862:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:18.349238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789465610651257:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-28T14:53:18.348635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789461917777923:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:18.350536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789461917777915:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:18.350625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:18.350672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789461917777915:2080] [node 6] ICP22 created new session: [3:7577789466212745248:2080]
2025-11-28T14:53:18.350719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789466212745248:2080] [node 6] ICS09 handshake done sender: [3:7577789461917777923:2113] self: [3:7577789461917777922:0] peer: [6:7577789465610651256:0] socket: 350 qp: -1
2025-11-28T14:53:18.350751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789466212745248:2080] [node 6] ICS10 traffic start
2025-11-28T14:53:18.350784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:18.356614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789465610651247:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:18.361652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789461315683862:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12787) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789465610651247:2121] held: no
2025-11-28T14:53:18.362308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789461315683862:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:18.362392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:18.362448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789461315683862:2050] [node 3] ICP22 created new session: [6:7577789465610651258:2050]
2025-11-28T14:53:18.362495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789465610651258:2050] [node 3] ICS09 handshake done sender: [6:7577789465610651247:2121] self: [6:7577789465610651256:0] peer: [3:7577789461917777922:0] socket: 383 qp: -1
2025-11-28T14:53:18.362534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789465610651258:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:18.362573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:18.407764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:18.408018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789466212745237:2048] [node 1] ICS01 socket: 365 reason# EndOfStream
2025-11-28T14:53:18.408064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789457622810554:2048] [node 1] ICP30 unregister session Session# [3:7577789466212745237:2048] VirtualId# [3:7577789466212745231:0]
2025-11-28T14:53:18.408100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789466212745237:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.408200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:18.408248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:18.408465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:18.408675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789463234064730:2048] [node 1] ICS07 socket disconnect 370 reason# EndOfStream
2025-11-28T14:53:18.408702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789463234064730:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.408768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:18.408817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789463234064730:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:18.413132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789454644130041:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26180) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:18.413166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789463234064730:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:18.413917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789454644130041:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26180) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:18.413983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789463234064730:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:18.414014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789454644130041:2048] [node 1] ICP30 unregister session Session# [2:7577789463234064730:2048] VirtualId# [2:7577789463234064719:0]
2025-11-28T14:53:18.414056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:18.414090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789454644130041:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26180) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.414105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26180) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.420591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:18.420813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789465610651245:2048] [node 1] ICS07 socket disconnect 378 reason# EndOfStream
2025-11-28T14:53:18.420851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789465610651245:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.420937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:18.420986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789465610651245:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:18.428654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789454644130041:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:18.428781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:18.429702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:18.429926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789464854626687:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-28T14:53:18.429962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789460559659288:2048] [node 1] ICP30 unregister session Session# [5:7577789464854626687:2048] VirtualId# [5:7577789464854626683:0]
2025-11-28T14:53:18.429995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789464854626687:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.430076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:18.430118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:18.430793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789463158518564:2048] [node 1] ICS01 socket: 358 reason# EndOfStream
2025-11-28T14:53:18.430848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789458863551167:2048] [node 1] ICP30 unregister session Session# [4:7577789463158518564:2048] VirtualId# [4:7577789463158518562:0]
2025-11-28T14:53:18.430875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789463158518564:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.430945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:18.430990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:18.431365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789461315683860:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26180) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:18.431440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789465610651245:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:18.431468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789461315683860:2048] [node 1] ICP30 unregister session Session# [6:7577789465610651245:2048] VirtualId# [6:7577789465610651238:0]
2025-11-28T14:53:18.431575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:18.431607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789461315683860:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26180) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.431619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26180) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.488710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789461315683860:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:18.488865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:18.489075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789466212745245:2049] [node 2] ICS01 socket: 377 reason# EndOfStream
2025-11-28T14:53:18.489118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789457622810555:2049] [node 2] ICP30 unregister session Session# [3:7577789466212745245:2049] VirtualId# [3:7577789461917777931:0]
2025-11-28T14:53:18.489140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789466212745245:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.489213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:18.489258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:18.496498Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:18.496661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789464854626680:2049] [node 2] ICS07 socket disconnect 393 reason# EndOfStream
2025-11-28T14:53:18.496685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789464854626680:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.496764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:18.496805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789464854626680:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:18.497553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789460559659289:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31065) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:18.497606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789464854626680:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:18.497640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789460559659289:2049] [node 2] ICP30 unregister session Session# [5:7577789464854626680:2049] VirtualId# [5:7577789464854626672:0]
2025-11-28T14:53:18.497681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:18.497708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789460559659289:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.497721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.497327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:18.497540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789463158518556:2049] [node 2] ICS07 socket disconnect 375 reason# EndOfStream
2025-11-28T14:53:18.497564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789463158518556:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.497625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:18.497661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789463158518556:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:18.498399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:18.498558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789465610651254:2049] [node 2] ICS01 socket: 368 reason# EndOfStream
2025-11-28T14:53:18.498585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789461315683861:2049] [node 2] ICP30 unregister session Session# [6:7577789465610651254:2049] VirtualId# [6:7577789465610651249:0]
2025-11-28T14:53:18.498604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789465610651254:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.498649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:18.498675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:18.499360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789458863551168:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31065) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:18.499393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789463158518556:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:18.499416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789458863551168:2049] [node 2] ICP30 unregister session Session# [4:7577789463158518556:2049] VirtualId# [4:7577789458863551249:0]
2025-11-28T14:53:18.499439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:18.499465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789458863551168:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.499479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.512868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789458863551168:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:18.512868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789460559659289:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:18.547762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:18.548028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789463158518560:2050] [node 3] ICS07 socket disconnect 352 reason# EndOfStream
2025-11-28T14:53:18.548056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789463158518560:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.548156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:18.548221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789463158518560:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:18.548997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:18.548998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:18.549362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789464854626677:2050] [node 3] ICS01 socket: 354 reason# EndOfStream
2025-11-28T14:53:18.549181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789465610651258:2050] [node 3] ICS01 socket: 383 reason# EndOfStream
2025-11-28T14:53:18.549416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789460559659290:2050] [node 3] ICP30 unregister session Session# [5:7577789464854626677:2050] VirtualId# [5:7577789460559659369:0]
2025-11-28T14:53:18.549223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789461315683862:2050] [node 3] ICP30 unregister session Session# [6:7577789465610651258:2050] VirtualId# [6:7577789465610651256:0]
2025-11-28T14:53:18.549441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789464854626677:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.549255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789465610651258:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.549315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:18.549354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:18.549530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:18.549605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:18.560560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789458863551169:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12787) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:18.560645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789463158518560:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:18.560682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789458863551169:2050] [node 3] ICP30 unregister session Session# [4:7577789463158518560:2050] VirtualId# [4:7577789463158518549:0]
2025-11-28T14:53:18.560733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:18.560770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789458863551169:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12787) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.560786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12787) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:18.572500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789458863551169:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:18.638556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:18.638803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789465610651242:2083] [node 4] ICS01 socket: 347 reason# EndOfStream
2025-11-28T14:53:18.638847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789461315683926:2083] [node 4] ICP30 unregister session Session# [6:7577789465610651242:2083] VirtualId# [6:7577789465610651236:0]
2025-11-28T14:53:18.638889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789465610651242:2083] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.638962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:18.639005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:18.644563Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:18.644810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789464854626668:2063] [node 4] ICS01 socket: 346 reason# EndOfStream
2025-11-28T14:53:18.644862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789460559659329:2063] [node 4] ICP30 unregister session Session# [5:7577789464854626668:2063] VirtualId# [5:7577789460559659368:0]
2025-11-28T14:53:18.644900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789464854626668:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.645005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:18.645055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:18.738939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:18.739179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789465610651252:2069] [node 5] ICS01 socket: 405 reason# EndOfStream
2025-11-28T14:53:18.739222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789461315683929:2069] [node 5] ICP30 unregister session Session# [6:7577789465610651252:2069] VirtualId# [6:7577789465610651248:0]
2025-11-28T14:53:18.739257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789465610651252:2069] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:18.739347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:18.739388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:18.940875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789463886003385:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:18.940959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789463886003386:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:18.980794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789464194028317:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:18.980849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789464194028318:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:19.022721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789466852804500:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:19.022778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789466852804501:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:19.092839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789470332840246:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:19.092896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789470332840247:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:19.092937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789470332840248:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:19.176812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789467983175479:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:19.176880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789467983175480:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:19.176916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789467983175481:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:19.324203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789468488995628:2061] [node 6] ICP01 ready to work
2025-11-28T14:53:19.357693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789466852804512:2058] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.420772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789466852804520:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49950
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.484741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789467983175501:2063] [node 4] ICP01 ready to work
2025-11-28T14:53:19.537025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789466852804567:2067] [node 4] ICP01 ready to work
2025-11-28T14:53:19.604534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789468488995676:2064] [node 4] ICP01 ready to work
2025-11-28T14:53:19.537562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789466852804568:2069] [node 6] ICP01 ready to work
2025-11-28T14:53:19.416773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789470746891430:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:19.616022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789470332840307:2062] [node 6] ICP01 ready to work
2025-11-28T14:53:19.580424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789468180970739:2065] [node 4] ICP01 ready to work
2025-11-28T14:53:19.569462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789467983175480:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(238819695ns) {Configure:104(238809099ns) {ProcessConfigured:135(102572368ns) {StartInitialHandshake:157(102567328ns) {DropHandshakes:470(3563ns) {DropIncomingHandshake:442(744ns) DropOutgoingHandshake:457(809ns)} GenerateSessionVirtualId:988(1612ns) PrepareNewSessionHandshake:477(5316ns) {SwitchToState:179(2771ns)}}}}}
2025-11-28T14:53:19.468437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789470746891428:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:19.552685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789470746891429:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:19.659915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789470746891488:2069] [node 5] ICP01 ready to work
2025-11-28T14:53:19.660455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789470746891489:2074] [node 4] ICP01 ready to work
2025-11-28T14:53:19.569973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789467983175524:2066] [node 6] ICP01 ready to work
2025-11-28T14:53:19.714750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789468180970745:2075] [node 6] ICP01 ready to work
2025-11-28T14:53:19.596095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789467983175532:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-11-28T14:53:19.710479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789464194028318:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(297674490ns) {Configure:104(297668583ns) {ProcessConfigured:135(95015ns) {StartInitialHandshake:157(91753ns) {DropHandshakes:470(3402ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(517ns)} GenerateSessionVirtualId:988(1365ns) PrepareNewSessionHandshake:477(2461ns) {SwitchToState:179(1301ns)}}}}}
2025-11-28T14:53:19.717054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789468180970746:2079] [node 5] ICP01 ready to work
2025-11-28T14:53:19.711064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789468488995680:2069] [node 5] ICP01 ready to work
2025-11-28T14:53:19.737132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789468488995628:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(412532924ns) {Configure:104(412527549ns) {ProcessConfigured:135(86672ns) {StartInitialHandshake:157(83300ns) {DropHandshakes:470(3529ns) {DropIncomingHandshake:442(588ns) DropOutgoingHandshake:457(838ns)} GenerateSessionVirtualId:988(908ns) PrepareNewSessionHandshake:477(2067ns) {SwitchToState:179(1144ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.719094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789466852804568:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(175260373ns) {Configure:104(175253024ns) {ProcessConfigured:135(10565021ns) {StartInitialHandshake:157(10561316ns) {DropHandshakes:470(5636824ns) {DropIncomingHandshake:442(2454ns) DropOutgoingHandshake:457(465ns)} GenerateSessionVirtualId:988(2473ns) PrepareNewSessionHandshake:477(3175ns) {SwitchToState:179(1688ns)}}}}}
2025-11-28T14:53:19.719397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789466852804514:2059] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.719433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789466852804514:2059] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:19.723730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789470746891429:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(98204894ns) {Configure:104(98198799ns) {ProcessConfigured:135(64062ns) {StartInitialHandshake:157(61745ns) {DropHandshakes:470(3176ns) {DropIncomingHandshake:442(822ns) DropOutgoingHandshake:457(565ns)} GenerateSessionVirtualId:988(1121ns) PrepareNewSessionHandshake:477(1755ns) {SwitchToState:179(801ns)}}}}}
2025-11-28T14:53:19.620941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789467983175532:2096] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.730060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789470332840248:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(288991659ns) {Configure:104(288985195ns) {ProcessConfigured:135(65348ns) {StartInitialHandshake:157(63160ns) {DropHandshakes:470(3188ns) {DropIncomingHandshake:442(729ns) DropOutgoingHandshake:457(577ns)} GenerateSessionVirtualId:988(1430ns) PrepareNewSessionHandshake:477(1927ns) {SwitchToState:179(686ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.771517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789466852804578:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49950
2025-11-28T14:53:19.762173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789468488995680:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24359031ns) {Configure:104(24352986ns) {ProcessConfigured:135(85787ns) {StartInitialHandshake:157(82614ns) {DropHandshakes:470(3691ns) {DropIncomingHandshake:442(711ns) DropOutgoingHandshake:457(845ns)} GenerateSessionVirtualId:988(986ns) PrepareNewSessionHandshake:477(2352ns) {SwitchToState:179(1029ns)}}}}}
2025-11-28T14:53:19.763003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789468488995681:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10874
2025-11-28T14:53:19.732637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789470332840310:2085] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.769198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789470746891488:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(108013958ns) {Configure:104(107990944ns) {ProcessConfigured:135(97857ns) {StartInitialHandshake:157(94886ns) {DropHandshakes:470(3851ns) {DropIncomingHandshake:442(778ns) DropOutgoingHandshake:457(543ns)} GenerateSessionVirtualId:988(1131ns) PrepareNewSessionHandshake:477(4762ns) {SwitchToState:179(539ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.797506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789468488995676:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(85556266ns) {Configure:104(85550992ns) {ProcessConfigured:135(52286ns) {StartInitialHandshake:157(48330ns) {DropHandshakes:470(1547ns) {DropIncomingHandshake:442(212ns) DropOutgoingHandshake:457(455ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(983ns) {SwitchToState:179(443ns)}}}}}
2025-11-28T14:53:19.797750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789468488995681:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.798168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789468488995691:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10874
2025-11-28T14:53:19.798296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789468488995692:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10874
2025-11-28T14:53:19.684422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789467983175479:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(354908557ns) {Configure:104(354903099ns) {ProcessConfigured:135(38851753ns) {StartInitialHandshake:157(38848122ns) {DropHandshakes:470(3404ns) {DropIncomingHandshake:442(669ns) DropOutgoingHandshake:457(715ns)} GenerateSessionVirtualId:988(935ns) PrepareNewSessionHandshake:477(4943ns) {SwitchToState:179(2414ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.769321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789470746891487:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:19.784511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789466852804520:2062] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.740985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789470332840247:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(329430182ns) {Configure:104(329420987ns) {ProcessConfigured:135(77814ns) {StartInitialHandshake:157(73823ns) {DropHandshakes:470(2973ns) {DropIncomingHandshake:442(625ns) DropOutgoingHandshake:457(491ns)} GenerateSessionVirtualId:988(950ns) PrepareNewSessionHandshake:477(2527ns) {SwitchToState:179(1006ns)}}}}}
2025-11-28T14:53:19.793215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789466852804514:2059] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.770532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789470746891501:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:19.706103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789467983175481:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(133544021ns) {Configure:104(133536720ns) {ProcessConfigured:135(107088ns) {StartInitialHandshake:157(103329ns) {DropHandshakes:470(3779ns) {DropIncomingHandshake:442(776ns) DropOutgoingHandshake:457(489ns)} GenerateSessionVirtualId:988(1630ns) PrepareNewSessionHandshake:477(2109ns) {SwitchToState:179(864ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.821381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789468488995691:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.744208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789470332840313:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13860
2025-11-28T14:53:19.795850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789470746891504:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:19.796899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789470746891487:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.797024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789470746891488:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.706369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789467983175481:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.848732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789463886003386:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(450753955ns) {Configure:104(437141190ns) {ProcessConfigured:135(19827692ns) {StartInitialHandshake:157(19823856ns) {DropHandshakes:470(3570ns) {DropIncomingHandshake:442(789ns) DropOutgoingHandshake:457(473ns)} GenerateSessionVirtualId:988(1142ns) PrepareNewSessionHandshake:477(4128ns) {SwitchToState:179(2503ns)}}}}}
2025-11-28T14:53:19.876742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789464194028317:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(250798456ns) {Configure:104(250790462ns) {ProcessConfigured:135(7428971ns) {StartInitialHandshake:157(7425538ns) {DropHandshakes:470(2582ns) {DropIncomingHandshake:442(728ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(1085ns) PrepareNewSessionHandshake:477(3879ns) {SwitchToState:179(2021ns)}}}}}
2025-11-28T14:53:19.788478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789470332840314:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13860
2025-11-28T14:53:19.817124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789470746891489:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(93156742ns) {Configure:104(93150818ns) {ProcessConfigured:135(69659ns) {StartInitialHandshake:157(67742ns) {DropHandshakes:470(3034ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(593ns)} GenerateSessionVirtualId:988(1413ns) PrepareNewSessionHandshake:477(2168ns) {SwitchToState:179(1086ns)}}}}}
2025-11-28T14:53:19.892780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789468488995692:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.817595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789467983175560:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-11-28T14:53:19.833463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789470746891428:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(344755878ns) {Configure:104(344745763ns) {ProcessConfigured:135(58923ns) {StartInitialHandshake:157(56249ns) {DropHandshakes:470(3227ns) {DropIncomingHandshake:442(673ns) DropOutgoingHandshake:457(579ns)} GenerateSessionVirtualId:988(764ns) PrepareNewSessionHandshake:477(2242ns) {SwitchToState:179(700ns)}}}}}
2025-11-28T14:53:19.834116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789470746891510:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:19.834821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789470746891487:2104] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:19.835008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789470746891488:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:19.835053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:19.835108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789470746891488:2069] [node 5] ICP22 created new session: [6:7577789470746891511:2069]
2025-11-28T14:53:19.835165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789470746891511:2069] [node 5] ICS09 handshake done sender: [6:7577789470746891487:2104] self: [6:7577789470746891505:0] peer: [5:7577789467983175533:0] socket: 360 qp: -1
2025-11-28T14:53:19.835198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789470746891511:2069] [node 5] ICS10 traffic start
2025-11-28T14:53:19.835244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:19.883619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789470746891504:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.884680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789470746891501:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.869149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789468180970747:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15960
2025-11-28T14:53:19.904461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789470746891429:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.877190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789466852804501:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(386411536ns) {Configure:104(386406246ns) {ProcessConfigured:135(22590503ns) {StartInitialHandshake:157(22587832ns) {DropHandshakes:470(2957974ns) {DropIncomingHandshake:442(780ns) DropOutgoingHandshake:457(871ns)} GenerateSessionVirtualId:988(1047ns) PrepareNewSessionHandshake:477(2928ns) {SwitchToState:179(1685ns)}}}}}
2025-11-28T14:53:19.881678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789466852804512:2058] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:19.881751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:19.881802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789466852804512:2058] [node 5] ICP22 created new session: [3:7577789466852804583:2058]
2025-11-28T14:53:19.881858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789466852804583:2058] [node 5] ICS09 handshake done sender: [3:7577789466852804514:2059] self: [3:7577789466852804513:0] peer: [5:7577789467983175557:0] socket: 344 qp: -1
2025-11-28T14:53:19.881893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789466852804583:2058] [node 5] ICS10 traffic start
2025-11-28T14:53:19.881948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:19.904589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789466852804578:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.917846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789468180970747:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.920783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789468180970754:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15960
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.788785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789470332840313:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.896465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789468488995685:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.896498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789468488995685:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.817819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789467983175534:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.828718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789470332840314:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.913342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789470746891514:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:19.928778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789468488995683:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.817842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789467983175534:2097] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:19.880726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789470332840248:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.945137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789466852804571:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.925284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789470746891489:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.928812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789468488995683:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:19.818654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789467983175562:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-11-28T14:53:19.821780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789467983175534:2097] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:19.824526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789467983175559:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-11-28T14:53:19.929375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789468488995685:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:19.930323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789468488995680:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:19.930382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:19.930432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789468488995680:2069] [node 5] ICP22 created new session: [2:7577789468488995700:2069]
2025-11-28T14:53:19.930483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789468488995700:2069] [node 5] ICS09 handshake done sender: [2:7577789468488995685:2113] self: [2:7577789468488995684:0] peer: [5:7577789467983175566:0] socket: 367 qp: -1
2025-11-28T14:53:19.930531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789468488995700:2069] [node 5] ICS10 traffic start
2025-11-28T14:53:19.930566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:19.972636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789468488995679:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.972667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789468488995679:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:19.973239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789468488995683:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:19.990155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789468488995679:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:19.945190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789466852804571:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.913528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789470332840248:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(23758676ns) {IncomingHandshake:261(8275620ns) {DropIncomingHandshake:442(421ns) PrepareNewSessionHandshake:477(2302ns) {SwitchToState:179(1027ns)} GenerateSessionVirtualId:988(910ns) IssueIncomingHandshakeReply:193(55966ns) {SwitchToState:179(459ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.008912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789468488995628:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.009014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:20.009060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789468488995628:2061] [node 6] ICP22 created new session: [2:7577789472783963001:2061]
2025-11-28T14:53:20.009110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789472783963001:2061] [node 6] ICS09 handshake done sender: [2:7577789468488995683:2112] self: [2:7577789468488995682:0] peer: [6:7577789470746891515:0] socket: 366 qp: -1
2025-11-28T14:53:20.009145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789472783963001:2061] [node 6] ICS10 traffic start
2025-11-28T14:53:20.009183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:20.011020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789472783963000:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10874
2025-11-28T14:53:19.884489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789467983175559:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.944897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789470746891517:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:20.012167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789464194028318:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.012211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:19.921787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789470332840329:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13860
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.893140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789467983175524:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:19.997361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789463886003385:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(200640963ns) {Configure:104(200635237ns) {ProcessConfigured:135(78154ns) {StartInitialHandshake:157(75023ns) {DropHandshakes:470(3494ns) {DropIncomingHandshake:442(968ns) DropOutgoingHandshake:457(408ns)} GenerateSessionVirtualId:988(1394ns) PrepareNewSessionHandshake:477(2843ns) {SwitchToState:179(1463ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.957212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789466852804501:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.893192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:19.966324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789470746891514:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.968696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789466852804571:2110] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:20.020864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789464194028318:2049] [node 3] ICP22 created new session: [2:7577789472783963004:2049]
2025-11-28T14:53:19.956415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789470332840316:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.956480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789470332840316:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:19.959282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789470332840320:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.959320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789470332840320:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:19.965876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789470332840330:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13860
2025-11-28T14:53:19.965942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789470332840331:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13860
2025-11-28T14:53:19.966109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789470332840316:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:20.004594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789470332840328:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13860
2025-11-28T14:53:20.020393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789470332840329:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.020566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789470332840320:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:20.046394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789470332840307:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.046483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:20.046542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789470332840307:2062] [node 6] ICP22 created new session: [4:7577789474627807630:2062]
2025-11-28T14:53:20.046597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789474627807630:2062] [node 6] ICS09 handshake done sender: [4:7577789470332840316:2113] self: [4:7577789470332840315:0] peer: [6:7577789470746891516:0] socket: 376 qp: -1
2025-11-28T14:53:20.046631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789474627807630:2062] [node 6] ICS10 traffic start
2025-11-28T14:53:20.046666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:20.047823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789470332840310:2085] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.047900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:20.047954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789470332840310:2085] [node 5] ICP22 created new session: [4:7577789474627807632:2085]
2025-11-28T14:53:20.047995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789474627807632:2085] [node 5] ICS09 handshake done sender: [4:7577789470332840320:2115] self: [4:7577789470332840319:0] peer: [5:7577789467983175568:0] socket: 378 qp: -1
2025-11-28T14:53:20.048022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789474627807632:2085] [node 5] ICS10 traffic start
2025-11-28T14:53:20.048054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:20.049210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789470332840314:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:20.016719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789468180970754:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.038714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789472783963004:2049] [node 3] ICS09 handshake done sender: [2:7577789468488995679:2110] self: [2:7577789468488995678:0] peer: [3:7577789466852804586:0] socket: 365 qp: -1
2025-11-28T14:53:20.040530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789472783963004:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:20.052503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:19.982402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789470746891518:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:20.069155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789472783963000:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.068492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789470332840246:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.984559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789466852804567:2067] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:19.893241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789467983175524:2066] [node 6] ICP22 created new session: [5:7577789467983175564:2066]
2025-11-28T14:53:20.024774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789468180970756:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.893289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789467983175564:2066] [node 6] ICS09 handshake done sender: [5:7577789467983175534:2097] self: [5:7577789467983175533:0] peer: [6:7577789470746891505:0] socket: 358 qp: -1
2025-11-28T14:53:19.984659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:20.082645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789464194028318:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(69522723ns) {HandleHandshakeStatus:329(69519622ns) {DropIncomingHandshake:442(754ns) DropOutgoingHandshake:457(660ns) SwitchToState:179(1097ns) ProcessPendingSessionEvents:495(17170866ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17106532ns max# 11297162ns}}}
2025-11-28T14:53:19.984591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789470746891430:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:20.024804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789468180970756:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:19.989116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789470746891501:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:20.083876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789470332840248:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:19.984713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789466852804567:2067] [node 4] ICP22 created new session: [3:7577789466852804589:2067]
2025-11-28T14:53:19.893324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789467983175564:2066] [node 6] ICS10 traffic start
2025-11-28T14:53:20.025832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789472475938061:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15960
2025-11-28T14:53:20.090227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789470332840330:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.085214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789464194028318:2049] [node 3] ICP18 handshake (actor [2:7577789472783963000:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.108757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789468488995690:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.108790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789468488995690:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:20.112687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789468488995690:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:19.893360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:19.895676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789467983175480:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.896776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789467983175560:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.908844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789467983175562:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.916607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789467983175524:2066] [node 6] ICP18 handshake (actor [5:7577789467983175560:2117]) is too old
2025-11-28T14:53:19.924737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789467983175501:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.956502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789467983175561:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-11-28T14:53:19.978198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789467983175480:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789467983175559:2116] held: no
2025-11-28T14:53:19.978848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789467983175532:2096] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:19.998090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789467983175570:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-11-28T14:53:19.998794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789467983175481:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:19.998844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:19.998891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789467983175481:2050] [node 3] ICP22 created new session: [5:7577789467983175572:2050]
2025-11-28T14:53:19.998940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789467983175572:2050] [node 3] ICS09 handshake done sender: [5:7577789467983175532:2096] self: [5:7577789467983175557:0] peer: [3:7577789466852804513:0] socket: 353 qp: -1
2025-11-28T14:53:19.998974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789467983175572:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:19.992976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789470746891504:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:20.160679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789468488995676:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.160791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:20.160852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789468488995676:2064] [node 4] ICP22 created new session: [2:7577789472783963007:2064]
2025-11-28T14:53:20.160895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789472783963007:2064] [node 4] ICS09 handshake done sender: [2:7577789468488995690:2114] self: [2:7577789468488995689:0] peer: [4:7577789474627807639:0] socket: 361 qp: -1
2025-11-28T14:53:20.160933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789472783963007:2064] [node 4] ICS10 traffic start
2025-11-28T14:53:20.160992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:20.092735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:19.984764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789466852804589:2067] [node 4] ICS09 handshake done sender: [3:7577789466852804571:2110] self: [3:7577789466852804570:0] peer: [4:7577789470332840327:0] socket: 357 qp: -1
2025-11-28T14:53:20.037341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789468180970756:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.092792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789470332840248:2050] [node 3] ICP22 created new session: [4:7577789474627807637:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.017187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789470746891489:2074] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.104387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789472475938061:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.999012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:20.092839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789474627807637:2050] [node 3] ICS09 handshake done sender: [4:7577789470332840314:2112] self: [4:7577789470332840327:0] peer: [3:7577789466852804570:0] socket: 364 qp: -1
2025-11-28T14:53:19.984798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789466852804589:2067] [node 4] ICS10 traffic start
2025-11-28T14:53:20.001675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789467983175571:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.206507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789472783963009:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10874
2025-11-28T14:53:20.017264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:20.128721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789468180970746:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.092880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789474627807637:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:20.092927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:20.095458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789470332840247:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:19.984842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:20.171503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789474627807640:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13860
2025-11-28T14:53:20.184694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789474627807641:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13860
2025-11-28T14:53:20.017313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789470746891489:2074] [node 4] ICP22 created new session: [6:7577789475041858817:2074]
2025-11-28T14:53:20.017940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789467983175570:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:19.986240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789466852804577:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.128815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:20.233324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789472783963009:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.017363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789475041858817:2074] [node 4] ICS09 handshake done sender: [6:7577789470746891504:2115] self: [6:7577789470746891516:0] peer: [4:7577789470332840315:0] socket: 379 qp: -1
2025-11-28T14:53:20.017400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789475041858817:2074] [node 4] ICS10 traffic start
2025-11-28T14:53:20.017439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:20.018953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789470746891429:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.018997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:20.019039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789470746891429:2049] [node 2] ICP22 created new session: [6:7577789475041858819:2049]
2025-11-28T14:53:20.019090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789475041858819:2049] [node 2] ICS09 handshake done sender: [6:7577789470746891501:2114] self: [6:7577789470746891515:0] peer: [2:7577789468488995682:0] socket: 369 qp: -1
2025-11-28T14:53:20.019121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789475041858819:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:20.019152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:20.176581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789470746891489:2074] [node 4] ICP35 event processing took too much time PendingConnection:279(133387462ns) {HandleHandshakeStatus:329(133381090ns) {DropIncomingHandshake:442(603ns) DropOutgoingHandshake:457(5335ns) SwitchToState:179(266ns) ProcessPendingSessionEvents:495(132981763ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 125074055ns max# 17532481ns}}}
2025-11-28T14:53:20.181131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789470746891520:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:20.199131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789470746891430:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7875) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789470746891514:2119] held: no
2025-11-28T14:53:20.200260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789470746891429:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:20.235283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789470746891514:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:20.018080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789467983175479:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:20.028888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789467983175569:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-11-28T14:53:20.045462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789467983175559:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:20.064767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789467983175562:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:20.072706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789467983175480:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.072771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:20.072830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789467983175480:2049] [node 2] ICP22 created new session: [5:7577789472278142872:2049]
2025-11-28T14:53:20.072881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789472278142872:2049] [node 2] ICS09 handshake done sender: [5:7577789467983175559:2116] self: [5:7577789467983175566:0] peer: [2:7577789468488995684:0] socket: 368 qp: -1
2025-11-28T14:53:20.072917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789472278142872:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:20.072965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:20.082865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789467983175501:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.097629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:20.104499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789467983175501:2063] [node 4] ICP22 created new session: [5:7577789472278142874:2063]
2025-11-28T14:53:20.124770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789472278142871:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-11-28T14:53:20.128608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789472278142874:2063] [node 4] ICS09 handshake done sender: [5:7577789467983175562:2119] self: [5:7577789467983175568:0] peer: [4:7577789470332840319:0] socket: 380 qp: -1
2025-11-28T14:53:20.128661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789472278142874:2063] [node 4] ICS10 traffic start
2025-11-28T14:53:20.128742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:20.130525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789467983175501:2063] [node 4] ICP35 event processing took too much time PendingConnection:279(46680315ns) {HandleHandshakeStatus:329(46677385ns) {DropIncomingHandshake:442(1087ns) DropOutgoingHandshake:457(5267ns) SwitchToState:179(922ns) ProcessPendingSessionEvents:495(600184ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 539092ns max# 59551ns}}}
2025-11-28T14:53:20.130604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789467983175501:2063] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:20.171860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789467983175479:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789467983175570:2121] held: no
2025-11-28T14:53:20.198373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789467983175570:2121] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.246381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789464194028317:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:20.237152Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789470332840247:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789470332840330:2119] held: no
2025-11-28T14:53:19.986282Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789466852804577:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:20.259960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789470746891430:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.260022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:20.260079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789470746891430:2050] [node 3] ICP22 created new session: [6:7577789475041858823:2050]
2025-11-28T14:53:20.260124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789475041858823:2050] [node 3] ICS09 handshake done sender: [6:7577789470746891514:2119] self: [6:7577789470746891519:0] peer: [3:7577789466852804576:0] socket: 389 qp: -1
2025-11-28T14:53:20.260161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789475041858823:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:20.260203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:20.276730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789475041858821:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:20.140746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789468180970749:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.259689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789470332840330:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.140777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789468180970749:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:19.987499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789466852804591:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49950
2025-11-28T14:53:20.293905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789475041858821:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.265703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789470332840307:2062] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.324862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789472783963011:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10874
2025-11-28T14:53:19.994559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789466852804577:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:20.033811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789466852804568:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.033902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:20.033951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789466852804568:2069] [node 6] ICP22 created new session: [3:7577789471147771890:2069]
2025-11-28T14:53:20.034003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789471147771890:2069] [node 6] ICS09 handshake done sender: [3:7577789466852804577:2114] self: [3:7577789466852804576:0] peer: [6:7577789470746891519:0] socket: 388 qp: -1
2025-11-28T14:53:20.034048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789471147771890:2069] [node 6] ICS10 traffic start
2025-11-28T14:53:20.034085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:20.046106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789466852804578:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:20.068855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789466852804501:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.069663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789466852804588:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49950
2025-11-28T14:53:20.104309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:20.110370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789466852804588:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.110574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789466852804500:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:20.120063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789466852804501:2049] [node 2] ICP22 created new session: [3:7577789471147771892:2049]
2025-11-28T14:53:20.128509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789471147771892:2049] [node 2] ICS09 handshake done sender: [3:7577789466852804578:2115] self: [3:7577789466852804586:0] peer: [2:7577789468488995678:0] socket: 372 qp: -1
2025-11-28T14:53:20.152561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789471147771892:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:20.157107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:20.245522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789466852804500:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789466852804588:2117] held: no
2025-11-28T14:53:20.292718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789466852804501:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(210173932ns) {HandleHandshakeStatus:329(210171254ns) {DropIncomingHandshake:442(1117ns) DropOutgoingHandshake:457(4903ns) SwitchToState:179(167ns) ProcessPendingSessionEvents:495(86192874ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 85716099ns max# 20174815ns}}}
2025-11-28T14:53:20.316416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789471147771895:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49950
2025-11-28T14:53:20.328235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789466852804588:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:20.313177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789470746891428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:20.282213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789470332840329:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:20.148543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789468180970751:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.289780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789470332840247:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.289847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:20.289895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789470332840247:2049] [node 2] ICP22 created new session: [4:7577789474627807642:2049]
2025-11-28T14:53:20.289944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789474627807642:2049] [node 2] ICS09 handshake done sender: [4:7577789470332840330:2119] self: [4:7577789474627807639:0] peer: [2:7577789468488995689:0] socket: 403 qp: -1
2025-11-28T14:53:20.289980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789474627807642:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:20.290029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:20.297815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789470332840246:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.297901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:20.297954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789470332840246:2048] [node 1] ICP22 created new session: [4:7577789474627807644:2048]
2025-11-28T14:53:20.298001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789474627807644:2048] [node 1] ICS09 handshake done sender: [4:7577789470332840329:2118] self: [4:7577789474627807634:0] peer: [1:7577789468180970750:0] socket: 396 qp: -1
2025-11-28T14:53:20.298036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789474627807644:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:20.298098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:20.326491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789470332840246:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27767000ns) {HandleHandshakeStatus:329(27763473ns) {DropIncomingHandshake:442(1734ns) DropOutgoingHandshake:457(4965ns) SwitchToState:179(904ns) ProcessPendingSessionEvents:495(24089890ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13668073ns max# 13226596ns}}}
2025-11-28T14:53:20.341804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789466852804500:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.148600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789468180970751:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:20.347503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:20.347580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789466852804500:2048] [node 1] ICP22 created new session: [3:7577789471147771896:2048]
2025-11-28T14:53:20.347645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789471147771896:2048] [node 1] ICS09 handshake done sender: [3:7577789466852804588:2117] self: [3:7577789471147771893:0] peer: [1:7577789468180970748:0] socket: 401 qp: -1
2025-11-28T14:53:20.347687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789471147771896:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:20.347749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:20.363763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789475041858826:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23891
2025-11-28T14:53:20.332221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789464194028317:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789472783963009:2119] held: no
2025-11-28T14:53:20.149750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789472475938062:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15960
2025-11-28T14:53:20.150297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789468180970749:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:20.152580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789468180970746:2079] [node 5] ICP22 created new session: [1:7577789472475938063:2079]
2025-11-28T14:53:20.156967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789468180970751:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:20.157070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789472475938063:2079] [node 5] ICS09 handshake done sender: [1:7577789468180970756:2115] self: [1:7577789468180970755:0] peer: [5:7577789472278142870:0] socket: 395 qp: -1
2025-11-28T14:53:20.161177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789463886003386:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.161251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:20.161292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789463886003386:2049] [node 3] ICP22 created new session: [1:7577789472475938068:2049]
2025-11-28T14:53:20.171725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789472475938063:2079] [node 5] ICS10 traffic start
2025-11-28T14:53:20.180577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789472475938068:2049] [node 3] ICS09 handshake done sender: [1:7577789468180970749:2111] self: [1:7577789468180970748:0] peer: [3:7577789471147771893:0] socket: 393 qp: -1
2025-11-28T14:53:20.186745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789472475938068:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:20.189159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789468180970739:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.189224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:20.189277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789468180970739:2065] [node 4] ICP22 created new session: [1:7577789472475938070:2065]
2025-11-28T14:53:20.189333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789472475938070:2065] [node 4] ICS09 handshake done sender: [1:7577789468180970751:2112] self: [1:7577789468180970750:0] peer: [4:7577789474627807634:0] socket: 394 qp: -1
2025-11-28T14:53:20.189370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789472475938070:2065] [node 4] ICS10 traffic start
2025-11-28T14:53:20.189397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:20.192355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:20.196439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:20.217121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789468180970746:2079] [node 5] ICP35 event processing took too much time PendingConnection:279(87310809ns) {HandleHandshakeStatus:329(87307551ns) {DropIncomingHandshake:442(14866ns) DropOutgoingHandshake:457(848ns) SwitchToState:179(1041ns) ProcessPendingSessionEvents:495(603142ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 533894ns max# 101892ns}}}
2025-11-28T14:53:20.256986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789472475938062:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.279908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789463886003386:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(117664980ns) {HandleHandshakeStatus:329(117663002ns) {DropIncomingHandshake:442(11963ns) DropOutgoingHandshake:457(286ns) SwitchToState:179(264ns) ProcessPendingSessionEvents:495(41858470ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41765075ns max# 15138322ns}}}
2025-11-28T14:53:20.280068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789468180970762:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.280103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789468180970762:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:20.299594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789472475938074:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15960
2025-11-28T14:53:20.299746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789468180970762:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:20.311970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789468180970739:2065] [node 4] ICP18 handshake (actor [1:7577789472475938062:2118]) is too old
2025-11-28T14:53:20.316572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [1:7577789468180970739:2065] [node 4] ICP35 event processing took too much time StateWork:294(27513108ns) {IncomingHandshake:261(23476216ns)}
2025-11-28T14:53:20.317088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789463886003385:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.317176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:20.317221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789463886003385:2048] [node 2] ICP22 created new session: [1:7577789472475938076:2048]
2025-11-28T14:53:20.317274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789472475938076:2048] [node 2] ICS09 handshake done sender: [1:7577789468180970762:2116] self: [1:7577789468180970761:0] peer: [2:7577789472783963010:0] socket: 350 qp: -1
2025-11-28T14:53:20.317308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789472475938076:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:20.317347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:20.340561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789468180970753:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.340601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789468180970753:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:20.369760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789472475938074:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:20.340201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789472783963009:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:20.352759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789464194028317:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.352823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:20.352881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789464194028317:2048] [node 1] ICP22 created new session: [2:7577789472783963012:2048]
2025-11-28T14:53:20.352936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789472783963012:2048] [node 1] ICS09 handshake done sender: [2:7577789472783963009:2119] self: [2:7577789472783963010:0] peer: [1:7577789468180970761:0] socket: 362 qp: -1
2025-11-28T14:53:20.352972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789472783963012:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:20.353007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:20.373925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789467983175479:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(119768615ns) {HandleHandshakeStatus:395(119764990ns) {LogHandshakeFail:472(3110870ns) {UpdateErrorStateLog:545(8826ns)} DropOutgoingHandshake:457(1083ns) ProcessPendingSessionEvents:495(116595041ns) {unordered DropSessionEvent:512 num# 100 sum# 116515545ns max# 15543988ns}}}
2025-11-28T14:53:20.373997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789467983175479:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.374046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:20.374096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789467983175479:2048] [node 1] ICP22 created new session: [5:7577789472278142876:2048]
2025-11-28T14:53:20.374139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789472278142876:2048] [node 1] ICS09 handshake done sender: [5:7577789467983175570:2121] self: [5:7577789472278142870:0] peer: [1:7577789468180970755:0] socket: 397 qp: -1
2025-11-28T14:53:20.374175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789472278142876:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:20.374224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:20.375276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789468180970753:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:20.400001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789475041858821:2123] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:20.392343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789468180970745:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:20.401115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789470746891428:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:20.401175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:20.401228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789470746891428:2048] [node 1] ICP22 created new session: [6:7577789475041858827:2048]
2025-11-28T14:53:20.401276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789475041858827:2048] [node 1] ICS09 handshake done sender: [6:7577789475041858821:2123] self: [6:7577789475041858825:0] peer: [1:7577789468180970752:0] socket: 370 qp: -1
2025-11-28T14:53:20.401310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789475041858827:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:20.401360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:20.413165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:20.414966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789468180970745:2075] [node 6] ICP22 created new session: [1:7577789472475938080:2075]
2025-11-28T14:53:20.415018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789472475938080:2075] [node 6] ICS09 handshake done sender: [1:7577789468180970753:2113] self: [1:7577789468180970752:0] peer: [6:7577789475041858825:0] socket: 418 qp: -1
2025-11-28T14:53:20.415058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789472475938080:2075] [node 6] ICS10 traffic start
2025-11-28T14:53:20.415131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:20.416773Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789468180970745:2075] [node 6] ICP35 event processing took too much time PendingConnection:279(37457237ns) {HandleHandshakeStatus:329(37453304ns) {DropIncomingHandshake:442(28401ns) DropOutgoingHandshake:457(1096ns) SwitchToState:179(855ns) ProcessPendingSessionEvents:495(532598ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 470169ns max# 62881ns}}}
2025-11-28T14:53:20.579943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:20.580191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789472783963012:2048] [node 1] ICS01 socket: 362 reason# EndOfStream
2025-11-28T14:53:20.580238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789464194028317:2048] [node 1] ICP30 unregister session Session# [2:7577789472783963012:2048] VirtualId# [2:7577789472783963010:0]
2025-11-28T14:53:20.580272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789472783963012:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.580382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:20.580428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:20.589235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:20.589474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789471147771896:2048] [node 1] ICS01 socket: 401 reason# EndOfStream
2025-11-28T14:53:20.589522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789466852804500:2048] [node 1] ICP30 unregister session Session# [3:7577789471147771896:2048] VirtualId# [3:7577789471147771893:0]
2025-11-28T14:53:20.589560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789471147771896:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.589663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:20.589709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:20.592712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:20.592940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789474627807644:2048] [node 1] ICS07 socket disconnect 396 reason# EndOfStream
2025-11-28T14:53:20.592970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789474627807644:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.593047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:20.593083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789474627807644:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:20.598392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:20.600638Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:20.612794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789472278142876:2048] [node 1] ICS01 socket: 397 reason# EndOfStream
2025-11-28T14:53:20.612852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789467983175479:2048] [node 1] ICP30 unregister session Session# [5:7577789472278142876:2048] VirtualId# [5:7577789472278142870:0]
2025-11-28T14:53:20.612895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789472278142876:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.613039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:20.613095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:20.620950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789475041858827:2048] [node 1] ICS01 socket: 370 reason# EndOfStream
2025-11-28T14:53:20.621015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789470746891428:2048] [node 1] ICP30 unregister session Session# [6:7577789475041858827:2048] VirtualId# [6:7577789475041858825:0]
2025-11-28T14:53:20.628783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789475041858827:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.632536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:20.633160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:20.645228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789470332840246:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22590) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:20.645308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789474627807644:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:20.645350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789470332840246:2048] [node 1] ICP30 unregister session Session# [4:7577789474627807644:2048] VirtualId# [4:7577789474627807634:0]
2025-11-28T14:53:20.645468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:20.645506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789470332840246:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22590) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.645522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22590) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.656618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789470332840246:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:20.704592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:20.704824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789471147771892:2049] [node 2] ICS07 socket disconnect 372 reason# EndOfStream
2025-11-28T14:53:20.704854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789471147771892:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.704970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:20.705012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789471147771892:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:20.713683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789466852804501:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31274) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:20.713728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789471147771892:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:20.714774Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789466852804501:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31274) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:20.714827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789471147771892:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:20.714867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789466852804501:2049] [node 2] ICP30 unregister session Session# [3:7577789471147771892:2049] VirtualId# [3:7577789466852804586:0]
2025-11-28T14:53:20.714994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:20.715026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789466852804501:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.715042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.719904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:20.720148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789472278142872:2049] [node 2] ICS01 socket: 368 reason# EndOfStream
2025-11-28T14:53:20.720192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789467983175480:2049] [node 2] ICP30 unregister session Session# [5:7577789472278142872:2049] VirtualId# [5:7577789467983175566:0]
2025-11-28T14:53:20.720227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789472278142872:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.720299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:20.720337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:20.720568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:20.720766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789475041858819:2049] [node 2] ICS07 socket disconnect 369 reason# EndOfStream
2025-11-28T14:53:20.720793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789475041858819:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.720861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:20.720900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789475041858819:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:20.723120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789470746891429:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31274) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:20.723186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789475041858819:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:20.723226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789470746891429:2049] [node 2] ICP30 unregister session Session# [6:7577789475041858819:2049] VirtualId# [6:7577789470746891515:0]
2025-11-28T14:53:20.723263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:20.723297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789470746891429:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.723312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.721391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:20.721580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789474627807642:2049] [node 2] ICS01 socket: 403 reason# EndOfStream
2025-11-28T14:53:20.721621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789470332840247:2049] [node 2] ICP30 unregister session Session# [4:7577789474627807642:2049] VirtualId# [4:7577789474627807639:0]
2025-11-28T14:53:20.721653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789474627807642:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.721706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:20.721747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:20.732619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789466852804501:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:20.736480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789470746891429:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:20.928591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:20.928865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789467983175572:2050] [node 3] ICS07 socket disconnect 353 reason# EndOfStream
2025-11-28T14:53:20.928890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789467983175572:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.928928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:20.928982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789467983175572:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:20.928591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:20.928862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789474627807637:2050] [node 3] ICS07 socket disconnect 364 reason# EndOfStream
2025-11-28T14:53:20.928890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789474627807637:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.928960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:20.928994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789474627807637:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:20.930879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789470332840248:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7875) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:20.930937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789474627807637:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:20.930971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789470332840248:2050] [node 3] ICP30 unregister session Session# [4:7577789474627807637:2050] VirtualId# [4:7577789470332840327:0]
2025-11-28T14:53:20.931022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:20.931061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789470332840248:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7875) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.931075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7875) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.936606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:20.936835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789475041858823:2050] [node 3] ICS01 socket: 389 reason# EndOfStream
2025-11-28T14:53:20.936875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789470746891430:2050] [node 3] ICP30 unregister session Session# [6:7577789475041858823:2050] VirtualId# [6:7577789470746891519:0]
2025-11-28T14:53:20.936913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789475041858823:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:20.937015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:20.937062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:20.937755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789467983175481:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7875) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:20.937823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789467983175572:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:20.937858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789467983175481:2050] [node 3] ICP30 unregister session Session# [5:7577789467983175572:2050] VirtualId# [5:7577789467983175557:0]
2025-11-28T14:53:20.937904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:20.937933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789467983175481:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7875) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.937947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7875) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:20.952487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789467983175481:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:21.048674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:21.048907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789475041858817:2074] [node 4] ICS07 socket disconnect 379 reason# EndOfStream
2025-11-28T14:53:21.048939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789475041858817:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:21.049029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:21.049072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789475041858817:2074] [node 4] ICS15 start handshake
2025-11-28T14:53:21.049619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:21.049821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789472278142874:2063] [node 4] ICS07 socket disconnect 380 reason# EndOfStream
2025-11-28T14:53:21.049849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789472278142874:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:21.049913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:21.049951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789472278142874:2063] [node 4] ICS15 start handshake
2025-11-28T14:53:21.051607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789467983175501:2063] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9270) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:21.051676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789472278142874:2063] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:21.051711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789467983175501:2063] [node 4] ICP30 unregister session Session# [5:7577789472278142874:2063] VirtualId# [5:7577789467983175568:0]
2025-11-28T14:53:21.051820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:21.051851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789467983175501:2063] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9270) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:21.051863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9270) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:21.052786Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789470746891489:2074] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9270) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:21.052842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789475041858817:2074] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:21.052881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789470746891489:2074] [node 4] ICP30 unregister session Session# [6:7577789475041858817:2074] VirtualId# [6:7577789470746891516:0]
2025-11-28T14:53:21.052973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:21.053001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789470746891489:2074] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9270) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:21.053018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9270) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:21.064538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789470746891489:2074] [node 4] ICP33 wake up from error state
2025-11-28T14:53:21.064538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789467983175501:2063] [node 4] ICP33 wake up from error state
2025-11-28T14:53:21.276597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:21.276850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789470746891511:2069] [node 5] ICS01 socket: 360 reason# EndOfStream
2025-11-28T14:53:21.276895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789470746891488:2069] [node 5] ICP30 unregister session Session# [6:7577789470746891511:2069] VirtualId# [6:7577789470746891505:0]
2025-11-28T14:53:21.276934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789470746891511:2069] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:21.277065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:21.277114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:21.658284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789478652504715:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:21.658335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789478652504716:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:21.752695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789479035365834:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:21.752746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789479035365835:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:21.864817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789479271947810:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:21.864891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789479271947811:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:21.940651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789477514241382:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:21.940713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789477514241383:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:21.940747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789477514241384:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:22.004917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789479529680568:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:22.004976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789479529680569:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:22.005008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789479529680570:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:22.064749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789482432687626:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:22.064797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789482432687627:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:22.064829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789482432687628:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:22.255369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789483566915128:2066] [node 5] ICP01 ready to work
2025-11-28T14:53:22.264555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789483330333160:2071] [node 4] ICP01 ready to work
2025-11-28T14:53:22.276563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789482947472045:2077] [node 4] ICP01 ready to work
2025-11-28T14:53:22.360457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789481809208698:2063] [node 5] ICP01 ready to work
2025-11-28T14:53:22.312548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789482947472050:2079] [node 5] ICP01 ready to work
2025-11-28T14:53:22.357375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789483824647884:2063] [node 6] ICP01 ready to work
2025-11-28T14:53:22.377067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789483330333169:2075] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.381028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789482947472063:2081] [node 6] ICP01 ready to work
2025-11-28T14:53:22.392475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789481809208708:2064] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.468632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789482432687658:2060] [node 4] ICP01 ready to work
2025-11-28T14:53:22.395666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789479529680569:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(70790840ns) {Configure:104(70783596ns) {ProcessConfigured:135(81039ns) {StartInitialHandshake:157(78188ns) {DropHandshakes:470(3887ns) {DropIncomingHandshake:442(1012ns) DropOutgoingHandshake:457(726ns)} GenerateSessionVirtualId:988(894ns) PrepareNewSessionHandshake:477(2663ns) {SwitchToState:179(989ns)}}}}}
2025-11-28T14:53:22.492644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789483566915159:2064] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.545294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789483824647934:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37722
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.409222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789483330333172:2073] [node 6] ICP01 ready to work
2025-11-28T14:53:22.548851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789483566915165:2065] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.563049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789483824647933:2062] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.643641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789483824647934:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.684754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789481809208708:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(136646068ns) {Configure:104(136641045ns) {ProcessConfigured:135(23718200ns) {StartInitialHandshake:157(23714505ns) {DropHandshakes:470(959ns) {DropIncomingHandshake:442(222ns) DropOutgoingHandshake:457(105ns)} GenerateSessionVirtualId:988(347ns) PrepareNewSessionHandshake:477(2368ns) {SwitchToState:179(990ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.643993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789483824647937:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37722
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.652917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789483824647932:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.652939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789483824647932:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.661448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789483824647932:2109] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.664448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789483824647937:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.664791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789479529680570:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.676870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789483824647942:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37722
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.705408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789483824647937:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.706408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789479529680570:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.562980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789483330333189:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23375
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.587612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789483566915166:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5478
2025-11-28T14:53:22.608756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789483566915128:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(352850145ns) {Configure:104(352844217ns) {ProcessConfigured:135(26756062ns) {StartInitialHandshake:157(26750735ns) {DropHandshakes:470(14698732ns) {DropIncomingHandshake:442(1836ns) DropOutgoingHandshake:457(441ns)} GenerateSessionVirtualId:988(2078ns) PrepareNewSessionHandshake:477(4734ns) {SwitchToState:179(2339ns)}}}}}
2025-11-28T14:53:22.621904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789479271947810:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(100642664ns) {Configure:104(100637204ns) {ProcessConfigured:135(24548558ns) {StartInitialHandshake:157(24544424ns) {DropHandshakes:470(1594ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:477(2336ns) {SwitchToState:179(1350ns)}}}}}
2025-11-28T14:53:22.624691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789483566915166:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.666269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789483566915176:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.666306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789483566915176:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.668650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789483566915176:2108] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:22.702839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789483566915181:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5478
2025-11-28T14:53:22.709098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789483566915128:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.709158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:22.709208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789483566915128:2066] [node 5] ICP22 created new session: [3:7577789483566915183:2066]
2025-11-28T14:53:22.709261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789483566915183:2066] [node 5] ICS09 handshake done sender: [3:7577789483566915176:2108] self: [3:7577789483566915173:0] peer: [5:7577789483824647941:0] socket: 379 qp: -1
2025-11-28T14:53:22.709300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789483566915183:2066] [node 5] ICS10 traffic start
2025-11-28T14:53:22.709360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:22.737799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789483566915185:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5478
2025-11-28T14:53:22.790943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789483566915185:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.802785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789479271947810:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.858018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789483566915165:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(56909398ns) {Configure:104(56901054ns) {ProcessConfigured:135(92953ns) {StartInitialHandshake:157(89169ns) {DropHandshakes:470(3968ns) {DropIncomingHandshake:442(991ns) DropOutgoingHandshake:457(672ns)} GenerateSessionVirtualId:988(1403ns) PrepareNewSessionHandshake:477(2501ns) {SwitchToState:179(1216ns)}}}}}
2025-11-28T14:53:22.888839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789483566915189:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5478
2025-11-28T14:53:22.891255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789483566915185:2111] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:22.546647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789482432687698:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:22.546816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789482432687699:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:22.580825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789482432687628:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(236820032ns) {Configure:104(236809644ns) {ProcessConfigured:135(47183284ns) {StartInitialHandshake:157(47179893ns) {DropHandshakes:470(2534ns) {DropIncomingHandshake:442(689ns) DropOutgoingHandshake:457(547ns)} GenerateSessionVirtualId:988(1002ns) PrepareNewSessionHandshake:477(3899ns) {SwitchToState:179(1786ns)}}}}}
2025-11-28T14:53:22.599291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789482432687698:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.621416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789482432687626:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.632545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789482432687702:2063] [node 5] ICP01 ready to work
2025-11-28T14:53:22.641396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789482432687704:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:22.641479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789482432687705:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:22.651513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789482432687698:2112] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:22.652427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789482432687699:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.652604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789482432687702:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.668712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789482432687626:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.668763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:22.668814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789482432687626:2048] [node 1] ICP22 created new session: [6:7577789482432687710:2048]
2025-11-28T14:53:22.668859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789482432687710:2048] [node 1] ICS09 handshake done sender: [6:7577789482432687698:2112] self: [6:7577789482432687703:0] peer: [1:7577789482947472073:0] socket: 370 qp: -1
2025-11-28T14:53:22.668893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789482432687710:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:22.668925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:22.670951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789482432687709:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:22.712008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789482432687699:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:22.712238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789482432687713:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:22.724913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789482432687702:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.729527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:22.749101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789482432687702:2063] [node 5] ICP22 created new session: [6:7577789482432687714:2063]
2025-11-28T14:53:22.757106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789482432687714:2063] [node 5] ICS09 handshake done sender: [6:7577789482432687699:2113] self: [6:7577789482432687708:0] peer: [5:7577789483824647931:0] socket: 374 qp: -1
2025-11-28T14:53:22.762865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789482432687713:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.773146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789482432687714:2063] [node 5] ICS10 traffic start
2025-11-28T14:53:22.773212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:22.775196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789482432687705:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.776243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789482432687658:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.778264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789482432687627:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.780824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789482432687718:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:22.791469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789482432687719:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:22.792051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789482432687713:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:22.796136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789482432687705:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:22.796469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789482432687658:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.796548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:22.796597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789482432687658:2060] [node 4] ICP22 created new session: [6:7577789482432687720:2060]
2025-11-28T14:53:22.796640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789482432687720:2060] [node 4] ICS09 handshake done sender: [6:7577789482432687713:2119] self: [6:7577789482432687716:0] peer: [4:7577789481809208732:0] socket: 406 qp: -1
2025-11-28T14:53:22.796682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789482432687720:2060] [node 4] ICS10 traffic start
2025-11-28T14:53:22.796714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:22.799237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789482432687627:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.799305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:22.799358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789482432687627:2049] [node 2] ICP22 created new session: [6:7577789482432687722:2049]
2025-11-28T14:53:22.799409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789482432687722:2049] [node 2] ICS09 handshake done sender: [6:7577789482432687705:2116] self: [6:7577789482432687717:0] peer: [2:7577789483330333191:0] socket: 388 qp: -1
2025-11-28T14:53:22.799444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789482432687722:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:22.799488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:22.877070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789482432687702:2063] [node 5] ICP35 event processing took too much time PendingConnection:279(90691495ns) {HandleHandshakeStatus:329(90688626ns) {DropIncomingHandshake:442(885ns) DropOutgoingHandshake:457(13090ns) SwitchToState:179(615ns) ProcessPendingSessionEvents:495(33187722ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 33118133ns max# 32703771ns}}}
2025-11-28T14:53:22.686379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789481809208721:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58941
2025-11-28T14:53:22.686809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789481809208734:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58941
2025-11-28T14:53:22.686957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789481809208735:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58941
2025-11-28T14:53:22.687060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789481809208736:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58941
2025-11-28T14:53:22.713676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789481809208734:2099] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.719673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789477514241382:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(397260121ns) {Configure:104(397251547ns) {ProcessConfigured:135(10800959ns) {StartInitialHandshake:157(10796898ns) {DropHandshakes:470(1015ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(641ns) PrepareNewSessionHandshake:477(2019ns) {SwitchToState:179(103ns)}}}}}
2025-11-28T14:53:22.719840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789481809208721:2088] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.732605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789481809208736:2101] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.750063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789477514241382:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.750170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789481809208743:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58941
2025-11-28T14:53:22.773303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789481809208743:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.778859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789481809208733:2098] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.778883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789481809208733:2098] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.779360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789481809208735:2100] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.779858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789481809208747:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58941
2025-11-28T14:53:22.781687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789477514241383:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(253778051ns) {Configure:104(253768661ns) {ProcessConfigured:135(21715789ns) {StartInitialHandshake:157(21712307ns) {DropHandshakes:470(4188ns) {DropIncomingHandshake:442(1170ns) DropOutgoingHandshake:457(782ns)} GenerateSessionVirtualId:988(943ns) PrepareNewSessionHandshake:477(11077ns) {SwitchToState:179(6916ns)}}}}}
2025-11-28T14:53:22.782162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789481809208736:2101] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:22.791116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789481809208733:2098] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:22.792422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789481809208723:2089] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.792449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789481809208723:2089] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.793076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789477514241382:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.793142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:22.793184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789477514241382:2048] [node 1] ICP22 created new session: [4:7577789481809208750:2048]
2025-11-28T14:53:22.793235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789481809208750:2048] [node 1] ICS09 handshake done sender: [4:7577789481809208736:2101] self: [4:7577789481809208745:0] peer: [1:7577789482947472082:0] socket: 397 qp: -1
2025-11-28T14:53:22.793267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789481809208750:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:22.793306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:22.801180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789481809208708:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.814756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789477514241383:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.816847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:22.819728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789481809208708:2064] [node 6] ICP22 created new session: [4:7577789481809208754:2064]
2025-11-28T14:53:22.852098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789481809208754:2064] [node 6] ICS09 handshake done sender: [4:7577789481809208733:2098] self: [4:7577789481809208732:0] peer: [6:7577789482432687716:0] socket: 399 qp: -1
2025-11-28T14:53:22.853089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789481809208754:2064] [node 6] ICS10 traffic start
2025-11-28T14:53:22.853242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789481809208723:2089] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:22.873886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:22.892796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789481809208698:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.632436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789483330333190:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23375
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.523056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789482947472086:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22545
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.706465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:22.637128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789483330333189:2101] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.935738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789477514241384:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(558279450ns) {Configure:104(558272821ns) {ProcessConfigured:135(52981ns) {StartInitialHandshake:157(51049ns) {DropHandshakes:470(3120ns) {DropIncomingHandshake:442(1142ns) DropOutgoingHandshake:457(334ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:477(2119ns) {SwitchToState:179(994ns)}}}}}
2025-11-28T14:53:22.910399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789483566915181:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.706523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789479529680570:2050] [node 3] ICP22 created new session: [5:7577789483824647945:2050]
2025-11-28T14:53:22.952661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:22.952880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789481809208708:2064] [node 6] ICP35 event processing took too much time PendingConnection:279(90707554ns) {HandleHandshakeStatus:329(90704392ns) {DropIncomingHandshake:442(13828ns) DropOutgoingHandshake:457(3546ns) SwitchToState:179(417ns) ProcessPendingSessionEvents:495(2581223ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 2522852ns max# 2162440ns}}}
2025-11-28T14:53:22.625872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789482947472088:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22545
2025-11-28T14:53:22.940924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789482432687725:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:22.916854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789479271947810:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789483566915185:2111] held: no
2025-11-28T14:53:22.638067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789479035365834:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.626503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472074:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.706568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789483824647945:2050] [node 3] ICS09 handshake done sender: [5:7577789483824647937:2112] self: [5:7577789483824647941:0] peer: [3:7577789483566915173:0] socket: 381 qp: -1
2025-11-28T14:53:22.954443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789477514241384:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.626533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789482947472074:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.632734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472086:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.924025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789483566915194:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5478
2025-11-28T14:53:22.960034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789481809208698:2063] [node 5] ICP22 created new session: [4:7577789481809208772:2063]
2025-11-28T14:53:22.638186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789483330333206:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23375
2025-11-28T14:53:22.706602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789483824647945:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:22.641748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789482947472074:2105] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:22.651740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472076:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.651781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789482947472076:2106] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.665552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789478652504716:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(281531378ns) {Configure:104(281527035ns) {ProcessConfigured:135(20989092ns) {StartInitialHandshake:157(20985186ns) {DropHandshakes:470(3374ns) {DropIncomingHandshake:442(808ns) DropOutgoingHandshake:457(591ns)} GenerateSessionVirtualId:988(830ns) PrepareNewSessionHandshake:477(2797ns) {SwitchToState:179(1604ns)}}}}}
2025-11-28T14:53:22.667848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789482947472063:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.667922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:22.678674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789482947472063:2081] [node 6] ICP22 created new session: [1:7577789482947472094:2081]
2025-11-28T14:53:22.690476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789482947472076:2106] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:22.711433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789482947472096:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22545
2025-11-28T14:53:22.711491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789482947472097:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22545
2025-11-28T14:53:22.712928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789478652504715:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.716570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789482947472094:2081] [node 6] ICS09 handshake done sender: [1:7577789482947472074:2105] self: [1:7577789482947472073:0] peer: [6:7577789482432687703:0] socket: 360 qp: -1
2025-11-28T14:53:22.722155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:22.722273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789482947472094:2081] [node 6] ICS10 traffic start
2025-11-28T14:53:22.722319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:22.728735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789478652504715:2048] [node 2] ICP22 created new session: [1:7577789482947472098:2048]
2025-11-28T14:53:22.732262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789482947472098:2048] [node 2] ICS09 handshake done sender: [1:7577789482947472076:2106] self: [1:7577789482947472075:0] peer: [2:7577789483330333210:0] socket: 361 qp: -1
2025-11-28T14:53:22.736467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472065:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.736528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789482947472065:2097] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.748560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789482947472098:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:22.748637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:22.769770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472083:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.769799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789482947472083:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.770482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472088:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.772262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789482947472102:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22545
2025-11-28T14:53:22.773432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472097:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.775281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789482947472065:2097] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:22.776402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789482947472083:2109] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:22.786085Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789478652504715:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(72290301ns) {HandleHandshakeStatus:329(72287997ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(770ns) SwitchToState:179(593ns) ProcessPendingSessionEvents:495(9605282ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 9532996ns max# 5333508ns}}}
2025-11-28T14:53:22.786349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789482947472045:2077] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.786417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:22.786470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789482947472045:2077] [node 4] ICP22 created new session: [1:7577789482947472105:2077]
2025-11-28T14:53:22.786520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789482947472105:2077] [node 4] ICS09 handshake done sender: [1:7577789482947472083:2109] self: [1:7577789482947472082:0] peer: [4:7577789481809208745:0] socket: 390 qp: -1
2025-11-28T14:53:22.786553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789482947472105:2077] [node 4] ICS10 traffic start
2025-11-28T14:53:22.786586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:22.787661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789482947472050:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.787699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:22.787748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789482947472050:2079] [node 5] ICP22 created new session: [1:7577789482947472107:2079]
2025-11-28T14:53:22.787790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789482947472107:2079] [node 5] ICS09 handshake done sender: [1:7577789482947472065:2097] self: [1:7577789482947472064:0] peer: [5:7577789483824647950:0] socket: 359 qp: -1
2025-11-28T14:53:22.787817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789482947472107:2079] [node 5] ICS10 traffic start
2025-11-28T14:53:22.787849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:22.789224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789478652504715:2048] [node 2] ICP18 handshake (actor [1:7577789482947472088:2114]) is too old
2025-11-28T14:53:22.790931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472096:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.812885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472102:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.813154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789482947472092:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.813179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789482947472092:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.819569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789482947472092:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:22.849652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789482947472050:2079] [node 5] ICP35 event processing took too much time PendingConnection:279(36910452ns) {HandleHandshakeStatus:329(36907911ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(492ns) SwitchToState:179(253ns) ProcessPendingSessionEvents:495(36620767ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36534354ns max# 14738985ns}}}
2025-11-28T14:53:22.850223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789482947472050:2079] [node 5] ICP18 handshake (actor [1:7577789482947472097:2117]) is too old
2025-11-28T14:53:22.871788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789478652504716:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.871876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:22.871921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789478652504716:2049] [node 3] ICP22 created new session: [1:7577789482947472111:2049]
2025-11-28T14:53:22.871969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789482947472111:2049] [node 3] ICS09 handshake done sender: [1:7577789482947472092:2115] self: [1:7577789482947472091:0] peer: [3:7577789483566915186:0] socket: 407 qp: -1
2025-11-28T14:53:22.872006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789482947472111:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:22.872043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:22.973985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789482947472045:2077] [node 4] ICP18 handshake (actor [1:7577789482947472102:2118]) is too old
2025-11-28T14:53:22.643654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789483330333206:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.965477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789483566915194:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.706656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:22.967806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789481809208772:2063] [node 5] ICS09 handshake done sender: [4:7577789481809208723:2089] self: [4:7577789481809208722:0] peer: [5:7577789483824647954:0] socket: 398 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:22.668038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789483330333173:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23375
2025-11-28T14:53:22.965698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789479271947811:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.972747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789481809208772:2063] [node 5] ICS10 traffic start
2025-11-28T14:53:22.708093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789483824647943:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37722
2025-11-28T14:53:23.021983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789482432687725:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:23.022126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789482432687628:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.972808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:22.992020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789483566915199:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5478
2025-11-28T14:53:22.689586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789483330333190:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.708565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789483824647884:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.708906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789479529680568:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(364743805ns) {Configure:104(364738337ns) {ProcessConfigured:135(16205896ns) {StartInitialHandshake:157(16201708ns) {DropHandshakes:470(545ns) {DropIncomingHandshake:442(131ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(181ns) PrepareNewSessionHandshake:477(5227ns) {SwitchToState:179(2759ns)}}}}}
2025-11-28T14:53:22.713033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:22.716753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789483824647884:2063] [node 6] ICP22 created new session: [5:7577789483824647947:2063]
2025-11-28T14:53:22.716806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789483824647947:2063] [node 6] ICS09 handshake done sender: [5:7577789483824647932:2109] self: [5:7577789483824647931:0] peer: [6:7577789482432687708:0] socket: 373 qp: -1
2025-11-28T14:53:22.716839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789483824647947:2063] [node 6] ICS10 traffic start
2025-11-28T14:53:22.716885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:22.718512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789483824647884:2063] [node 6] ICP35 event processing took too much time PendingConnection:279(19795383ns) {HandleHandshakeStatus:329(19791775ns) {DropIncomingHandshake:442(1314ns) DropOutgoingHandshake:457(978ns) SwitchToState:179(1124ns) ProcessPendingSessionEvents:495(606623ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 543344ns max# 85995ns}}}
2025-11-28T14:53:22.718780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789479529680568:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.730302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789483824647933:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31478249ns) {Configure:104(31473275ns) {ProcessConfigured:135(16505819ns) {StartInitialHandshake:157(16502340ns) {DropHandshakes:470(1768ns) {DropIncomingHandshake:442(602ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:477(3111ns) {SwitchToState:179(1270ns)}}}}}
2025-11-28T14:53:22.771300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789483824647953:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37722
2025-11-28T14:53:22.778505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789483824647943:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.781380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789483824647934:2110] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:22.781806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789483824647933:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.782711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789479529680568:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.782762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:22.782813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789479529680568:2048] [node 1] ICP22 created new session: [5:7577789483824647955:2048]
2025-11-28T14:53:22.782859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789483824647955:2048] [node 1] ICS09 handshake done sender: [5:7577789483824647934:2110] self: [5:7577789483824647950:0] peer: [1:7577789482947472064:0] socket: 371 qp: -1
2025-11-28T14:53:22.782898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789483824647955:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:22.782960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:22.837050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789483824647957:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37722
2025-11-28T14:53:22.837233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789483824647958:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37722
2025-11-28T14:53:22.870073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789479529680568:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(86487047ns) {HandleHandshakeStatus:329(86483946ns) {DropIncomingHandshake:442(587ns) DropOutgoingHandshake:457(6973ns) SwitchToState:179(223ns) ProcessPendingSessionEvents:495(84763231ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 84676869ns max# 15510361ns}}}
2025-11-28T14:53:22.881193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789483824647943:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:22.885582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789483824647957:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.885630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789483824647933:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.885698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:22.885750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789483824647933:2062] [node 4] ICP22 created new session: [5:7577789483824647959:2062]
2025-11-28T14:53:22.885800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789483824647959:2062] [node 4] ICS09 handshake done sender: [5:7577789483824647943:2115] self: [5:7577789483824647954:0] peer: [4:7577789481809208722:0] socket: 402 qp: -1
2025-11-28T14:53:22.885839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789483824647959:2062] [node 4] ICS10 traffic start
2025-11-28T14:53:22.885877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:22.887967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789479529680569:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:22.913343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789483824647963:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37722
2025-11-28T14:53:22.933557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789479529680569:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20315) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789483824647957:2118] held: no
2025-11-28T14:53:22.953042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789483824647957:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:22.958041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789479529680569:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.958108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:22.958163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789479529680569:2049] [node 2] ICP22 created new session: [5:7577789483824647965:2049]
2025-11-28T14:53:22.958210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789483824647965:2049] [node 2] ICS09 handshake done sender: [5:7577789483824647957:2118] self: [5:7577789483824647961:0] peer: [2:7577789483330333207:0] socket: 376 qp: -1
2025-11-28T14:53:22.958242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789483824647965:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:22.958278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:23.027850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789482947472063:2081] [node 6] ICP35 event processing took too much time PendingConnection:279(377507777ns) {HandleHandshakeStatus:329(377504994ns) {DropIncomingHandshake:442(16820ns) DropOutgoingHandshake:457(831ns) SwitchToState:179(737ns) ProcessPendingSessionEvents:495(277933217ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 256604641ns max# 21046537ns}}}
2025-11-28T14:53:22.993489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789483566915172:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.993529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789483566915172:2107] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:23.020785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789479271947810:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(119079510ns) {HandleHandshakeStatus:395(119075817ns) {LogHandshakeFail:472(13743ns) {UpdateErrorStateLog:545(5321ns)} DropOutgoingHandshake:457(3701ns) ProcessPendingSessionEvents:495(94855177ns) {unordered DropSessionEvent:512 num# 100 sum# 94775340ns max# 17044872ns}}}
2025-11-28T14:53:23.020902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789479271947811:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20315) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789483566915194:2119] held: no
2025-11-28T14:53:22.689685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789483330333173:2087] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.690102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789483330333212:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23375
2025-11-28T14:53:22.769168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789483330333189:2101] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:22.775365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789479035365834:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:22.781743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:22.781907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789483330333192:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.781943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789483330333192:2103] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.782443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789479035365834:2048] [node 1] ICP22 created new session: [2:7577789483330333214:2048]
2025-11-28T14:53:22.784795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789483330333192:2103] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:22.801019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789483330333214:2048] [node 1] ICS09 handshake done sender: [2:7577789483330333189:2101] self: [2:7577789483330333210:0] peer: [1:7577789482947472075:0] socket: 366 qp: -1
2025-11-28T14:53:22.816310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789483330333172:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.819344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789483330333214:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:22.820448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:22.829160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789483330333188:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.829189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789483330333188:2100] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.829780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:22.841154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789483330333172:2073] [node 6] ICP22 created new session: [2:7577789483330333217:2073]
2025-11-28T14:53:22.860278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789483330333217:2073] [node 6] ICS09 handshake done sender: [2:7577789483330333192:2103] self: [2:7577789483330333191:0] peer: [6:7577789482432687717:0] socket: 385 qp: -1
2025-11-28T14:53:22.880540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789483330333217:2073] [node 6] ICS10 traffic start
2025-11-28T14:53:22.882781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789479035365834:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(107527918ns) {HandleHandshakeStatus:329(107524910ns) {DropIncomingHandshake:442(2554ns) DropOutgoingHandshake:457(1578191ns) SwitchToState:179(971ns) ProcessPendingSessionEvents:495(13082516ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13012415ns max# 12577333ns}}}
2025-11-28T14:53:22.889923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789483330333208:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.889952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789483330333208:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.892952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:22.905642Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789479035365835:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(500596190ns) {Configure:104(500589890ns) {ProcessConfigured:135(93402ns) {StartInitialHandshake:157(89756ns) {DropHandshakes:470(3512ns) {DropIncomingHandshake:442(767ns) DropOutgoingHandshake:457(706ns)} GenerateSessionVirtualId:988(1023ns) PrepareNewSessionHandshake:477(3083ns) {SwitchToState:179(1275ns)}}}}}
2025-11-28T14:53:22.905888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789483330333188:2100] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:22.912506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789483330333219:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23375
2025-11-28T14:53:22.920790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789483330333208:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:22.921210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789483330333169:2075] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.921276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:22.921324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789483330333169:2075] [node 5] ICP22 created new session: [2:7577789483330333225:2075]
2025-11-28T14:53:22.921377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789483330333225:2075] [node 5] ICS09 handshake done sender: [2:7577789483330333208:2116] self: [2:7577789483330333207:0] peer: [5:7577789483824647961:0] socket: 358 qp: -1
2025-11-28T14:53:22.921410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789483330333225:2075] [node 5] ICS10 traffic start
2025-11-28T14:53:22.921455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:22.928717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789483330333160:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:22.948027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789483330333172:2073] [node 6] ICP35 event processing took too much time PendingConnection:279(130712284ns) {HandleHandshakeStatus:329(130709455ns) {DropIncomingHandshake:442(11153ns) DropOutgoingHandshake:457(1161ns) SwitchToState:179(490ns) ProcessPendingSessionEvents:495(14410483ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14347530ns max# 13885277ns}}}
2025-11-28T14:53:22.952595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:22.956737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789483330333160:2071] [node 4] ICP22 created new session: [2:7577789483330333229:2071]
2025-11-28T14:53:22.959525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789483330333219:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.966075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789483330333224:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:22.966101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789483330333224:2120] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:22.967037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789483330333224:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:22.967795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789483330333229:2071] [node 4] ICS09 handshake done sender: [2:7577789483330333188:2100] self: [2:7577789483330333187:0] peer: [4:7577789481809208753:0] socket: 384 qp: -1
2025-11-28T14:53:22.973175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789483330333229:2071] [node 4] ICS10 traffic start
2025-11-28T14:53:22.984644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:22.992568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789479035365835:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:23.002949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:23.008114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789483330333160:2071] [node 4] ICP35 event processing took too much time PendingConnection:279(78507199ns) {HandleHandshakeStatus:329(78501294ns) {DropIncomingHandshake:442(747ns) DropOutgoingHandshake:457(1068ns) SwitchToState:179(731ns) ProcessPendingSessionEvents:495(4362723ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4295469ns max# 3886873ns}}}
2025-11-28T14:53:23.008213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789483330333160:2071] [node 4] ICP18 handshake (actor [2:7577789483330333219:2118]) is too old
2025-11-28T14:53:23.009381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789479035365835:2049] [node 3] ICP22 created new session: [2:7577789487625300529:2049]
2025-11-28T14:53:23.013560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789487625300529:2049] [node 3] ICS09 handshake done sender: [2:7577789483330333224:2120] self: [2:7577789483330333223:0] peer: [3:7577789483566915198:0] socket: 356 qp: -1
2025-11-28T14:53:23.015357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789487625300529:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:23.015414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:23.010526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789481809208698:2063] [node 5] ICP35 event processing took too much time PendingConnection:279(117182700ns) {HandleHandshakeStatus:329(117179808ns) {DropIncomingHandshake:442(18826496ns) DropOutgoingHandshake:457(1707ns) SwitchToState:179(552ns) ProcessPendingSessionEvents:495(7399056ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 7363189ns max# 7158129ns}}}
2025-11-28T14:53:23.020097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789481809208769:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58941
2025-11-28T14:53:23.023211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789477514241383:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20315) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789481809208735:2100] held: no
2025-11-28T14:53:23.037578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789479271947810:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:23.037663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:23.037721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789479271947810:2048] [node 1] ICP22 created new session: [3:7577789487861882498:2048]
2025-11-28T14:53:23.037780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789487861882498:2048] [node 1] ICS09 handshake done sender: [3:7577789483566915185:2111] self: [3:7577789483566915186:0] peer: [1:7577789482947472091:0] socket: 408 qp: -1
2025-11-28T14:53:23.037812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789487861882498:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:23.037850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:23.038854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789483566915194:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:23.035339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789479035365835:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(41872892ns) {HandleHandshakeStatus:329(41869801ns) {DropIncomingHandshake:442(34583ns) DropOutgoingHandshake:457(746ns) SwitchToState:179(817ns) ProcessPendingSessionEvents:495(18119837ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18035084ns max# 7231867ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:23.041057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789483566915188:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:23.041094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789483566915188:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:23.042413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789479271947811:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:23.042492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:23.042545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789479271947811:2049] [node 2] ICP22 created new session: [3:7577789487861882500:2049]
2025-11-28T14:53:23.042590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789487861882500:2049] [node 2] ICS09 handshake done sender: [3:7577789483566915194:2119] self: [3:7577789483566915198:0] peer: [2:7577789483330333223:0] socket: 421 qp: -1
2025-11-28T14:53:23.042626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789487861882500:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:23.042664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:23.043806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789487861882496:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5478
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:23.046294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789486727655023:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33541
2025-11-28T14:53:23.048053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789481809208735:2100] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:23.049043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789486104176074:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58941
2025-11-28T14:53:23.047433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789483566915172:2107] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:23.048236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789483566915188:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:23.049914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789487861882496:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:23.050313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789483566915165:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:23.050398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:23.050444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789483566915165:2065] [node 6] ICP22 created new session: [3:7577789487861882504:2065]
2025-11-28T14:53:23.050498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789487861882504:2065] [node 6] ICS09 handshake done sender: [3:7577789483566915188:2112] self: [3:7577789483566915187:0] peer: [6:7577789486727655022:0] socket: 354 qp: -1
2025-11-28T14:53:23.050531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789487861882504:2065] [node 6] ICS10 traffic start
2025-11-28T14:53:23.050567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:23.052198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789483566915159:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:23.052262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:23.052309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789483566915159:2064] [node 4] ICP22 created new session: [3:7577789487861882506:2064]
2025-11-28T14:53:23.052358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789487861882506:2064] [node 4] ICS09 handshake done sender: [3:7577789483566915172:2107] self: [3:7577789483566915171:0] peer: [4:7577789481809208775:0] socket: 378 qp: -1
2025-11-28T14:53:23.052402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789487861882506:2064] [node 4] ICS10 traffic start
2025-11-28T14:53:23.052436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:23.054393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789482432687725:2123] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:23.054184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789477514241383:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:23.054245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:23.054309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789477514241383:2049] [node 2] ICP22 created new session: [4:7577789486104176075:2049]
2025-11-28T14:53:23.054353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789486104176075:2049] [node 2] ICS09 handshake done sender: [4:7577789481809208735:2100] self: [4:7577789481809208753:0] peer: [2:7577789483330333187:0] socket: 396 qp: -1
2025-11-28T14:53:23.054385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789486104176075:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:23.054427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:23.057465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789477514241384:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26133) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789481809208734:2099] held: no
2025-11-28T14:53:23.057666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789481809208734:2099] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:23.057597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789482432687628:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26133) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789482432687725:2123] held: no
2025-11-28T14:53:23.060035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789482432687628:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:23.060114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:23.060163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789482432687628:2050] [node 3] ICP22 created new session: [6:7577789486727655024:2050]
2025-11-28T14:53:23.060213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789486727655024:2050] [node 3] ICS09 handshake done sender: [6:7577789482432687725:2123] self: [6:7577789486727655022:0] peer: [3:7577789483566915187:0] socket: 357 qp: -1
2025-11-28T14:53:23.060244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789486727655024:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:23.060280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:23.062693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789477514241384:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:23.062761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:23.062804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789477514241384:2050] [node 3] ICP22 created new session: [4:7577789486104176077:2050]
2025-11-28T14:53:23.062849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789486104176077:2050] [node 3] ICS09 handshake done sender: [4:7577789481809208734:2099] self: [4:7577789481809208775:0] peer: [3:7577789483566915171:0] socket: 365 qp: -1
2025-11-28T14:53:23.062884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789486104176077:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:23.062914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:23.112260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:23.112529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789483330333214:2048] [node 1] ICS01 socket: 366 reason# EndOfStream
2025-11-28T14:53:23.112588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789479035365834:2048] [node 1] ICP30 unregister session Session# [2:7577789483330333214:2048] VirtualId# [2:7577789483330333210:0]
2025-11-28T14:53:23.112622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789483330333214:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.112706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:23.112753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:23.113507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:23.113716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789487861882498:2048] [node 1] ICS01 socket: 408 reason# EndOfStream
2025-11-28T14:53:23.113754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789479271947810:2048] [node 1] ICP30 unregister session Session# [3:7577789487861882498:2048] VirtualId# [3:7577789483566915186:0]
2025-11-28T14:53:23.113782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789487861882498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.113843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:23.113877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:23.120573Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:23.120814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789481809208750:2048] [node 1] ICS07 socket disconnect 397 reason# EndOfStream
2025-11-28T14:53:23.120870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789481809208750:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.120937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:23.120975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789481809208750:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:23.123155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789477514241382:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4440) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:23.123245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789481809208750:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:23.123282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789477514241382:2048] [node 1] ICP30 unregister session Session# [4:7577789481809208750:2048] VirtualId# [4:7577789481809208745:0]
2025-11-28T14:53:23.123401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:23.123445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789477514241382:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:23.123460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:23.124921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:23.125165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789483824647955:2048] [node 1] ICS07 socket disconnect 371 reason# EndOfStream
2025-11-28T14:53:23.125196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789483824647955:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.125284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:23.125333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789483824647955:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:23.126055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789479529680568:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4440) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:23.126224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789483824647955:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:23.126269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789479529680568:2048] [node 1] ICP30 unregister session Session# [5:7577789483824647955:2048] VirtualId# [5:7577789483824647950:0]
2025-11-28T14:53:23.126540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:23.126583Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789479529680568:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:23.126597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:23.128527Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:23.128686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789482432687710:2048] [node 1] ICS01 socket: 370 reason# EndOfStream
2025-11-28T14:53:23.128718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789482432687626:2048] [node 1] ICP30 unregister session Session# [6:7577789482432687710:2048] VirtualId# [6:7577789482432687703:0]
2025-11-28T14:53:23.128744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789482432687710:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.128819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:23.128875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:23.133107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789477514241382:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:23.136194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789479529680568:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:23.214972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:23.215201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789487861882500:2049] [node 2] ICS01 socket: 421 reason# EndOfStream
2025-11-28T14:53:23.215255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789479271947811:2049] [node 2] ICP30 unregister session Session# [3:7577789487861882500:2049] VirtualId# [3:7577789483566915198:0]
2025-11-28T14:53:23.215288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789487861882500:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.215363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:23.215401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:23.220638Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:23.220846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789486104176075:2049] [node 2] ICS01 socket: 396 reason# EndOfStream
2025-11-28T14:53:23.220893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789477514241383:2049] [node 2] ICP30 unregister session Session# [4:7577789486104176075:2049] VirtualId# [4:7577789481809208753:0]
2025-11-28T14:53:23.220929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789486104176075:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.221027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:23.221081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:23.221182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:23.221352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789482432687722:2049] [node 2] ICS01 socket: 388 reason# EndOfStream
2025-11-28T14:53:23.221387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789482432687627:2049] [node 2] ICP30 unregister session Session# [6:7577789482432687722:2049] VirtualId# [6:7577789482432687717:0]
2025-11-28T14:53:23.221414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789482432687722:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.221470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:23.221515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:23.221603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:23.221777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789483824647965:2049] [node 2] ICS01 socket: 376 reason# EndOfStream
2025-11-28T14:53:23.221802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789479529680569:2049] [node 2] ICP30 unregister session Session# [5:7577789483824647965:2049] VirtualId# [5:7577789483824647961:0]
2025-11-28T14:53:23.221820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789483824647965:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.221854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:23.221874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:23.292555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:23.292780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789486104176077:2050] [node 3] ICS01 socket: 365 reason# EndOfStream
2025-11-28T14:53:23.292826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789477514241384:2050] [node 3] ICP30 unregister session Session# [4:7577789486104176077:2050] VirtualId# [4:7577789481809208775:0]
2025-11-28T14:53:23.292864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789486104176077:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.292938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:23.292986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:23.296631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:23.296960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789486727655024:2050] [node 3] ICS01 socket: 357 reason# EndOfStream
2025-11-28T14:53:23.297003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789482432687628:2050] [node 3] ICP30 unregister session Session# [6:7577789486727655024:2050] VirtualId# [6:7577789486727655022:0]
2025-11-28T14:53:23.297068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789486727655024:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.297208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:23.297291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:23.297505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:23.297818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789483824647945:2050] [node 3] ICS01 socket: 381 reason# EndOfStream
2025-11-28T14:53:23.297937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789479529680570:2050] [node 3] ICP30 unregister session Session# [5:7577789483824647945:2050] VirtualId# [5:7577789483824647941:0]
2025-11-28T14:53:23.297991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789483824647945:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.298083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:23.298145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:23.367765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:23.368009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789483824647959:2062] [node 4] ICS01 socket: 402 reason# EndOfStream
2025-11-28T14:53:23.368056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789483824647933:2062] [node 4] ICP30 unregister session Session# [5:7577789483824647959:2062] VirtualId# [5:7577789483824647954:0]
2025-11-28T14:53:23.368089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789483824647959:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.368166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:23.368218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:23.369049Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:23.369244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789482432687720:2060] [node 4] ICS01 socket: 406 reason# EndOfStream
2025-11-28T14:53:23.369287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789482432687658:2060] [node 4] ICP30 unregister session Session# [6:7577789482432687720:2060] VirtualId# [6:7577789482432687716:0]
2025-11-28T14:53:23.369314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789482432687720:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.369386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:23.369421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:23.432802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:23.433068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789482432687714:2063] [node 5] ICS01 socket: 374 reason# EndOfStream
2025-11-28T14:53:23.433120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789482432687702:2063] [node 5] ICP30 unregister session Session# [6:7577789482432687714:2063] VirtualId# [6:7577789482432687708:0]
2025-11-28T14:53:23.433160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789482432687714:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:23.433261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:23.433312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:23.944697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789484743917414:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:23.952542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789484743917413:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:23.993492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789484215593113:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:23.993494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789484215593114:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:24.064766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789490276870238:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:24.064818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789490276870239:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:24.143175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789490958768873:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:24.143231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789490958768874:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:24.145858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789490958768872:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:24.182485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789490706412160:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:24.182543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789490706412162:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:24.182577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789490706412161:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:24.208678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789490698608191:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:24.208735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789490698608192:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:24.208768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789490698608193:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:24.603685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789490698608214:2069] [node 5] ICP01 ready to work
2025-11-28T14:53:24.670203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789490958768934:2108] [node 6] ICP01 ready to work
2025-11-28T14:53:24.777806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789490706412223:2100] [node 6] ICP01 ready to work
2025-11-28T14:53:24.777033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789490958768934:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(106494363ns) {Configure:104(106490700ns) {ProcessConfigured:135(24081391ns) {StartInitialHandshake:157(24077342ns) {DropHandshakes:470(3550ns) {DropIncomingHandshake:442(748ns) DropOutgoingHandshake:457(537ns)} GenerateSessionVirtualId:988(817ns) PrepareNewSessionHandshake:477(5020ns) {SwitchToState:179(1963ns)}}}}}
2025-11-28T14:53:24.797140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789488510560475:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.781362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789489038884785:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:24.800852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789488510560476:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56608
2025-11-28T14:53:24.801813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789488510560472:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:24.804807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789484215593114:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(192339486ns) {Configure:104(192314575ns) {ProcessConfigured:135(82944ns) {StartInitialHandshake:157(79928ns) {DropHandshakes:470(4171ns) {DropIncomingHandshake:442(933ns) DropOutgoingHandshake:457(1076ns)} GenerateSessionVirtualId:988(1257ns) PrepareNewSessionHandshake:477(4190ns) {SwitchToState:179(635ns)}}}}}
2025-11-28T14:53:24.783796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789490706412224:2104] [node 4] ICP01 ready to work
2025-11-28T14:53:24.798285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789490958768938:2109] [node 5] ICP01 ready to work
2025-11-28T14:53:24.793544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789489038884784:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:24.783781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789490698608254:2066] [node 4] ICP01 ready to work
2025-11-28T14:53:24.798351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789490276870319:2126] [node 6] ICP01 ready to work
2025-11-28T14:53:24.806333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789490276870321:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.797728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789489038884786:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:24.825919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789490698608214:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(221891109ns) {Configure:104(221885804ns) {ProcessConfigured:135(47990ns) {StartInitialHandshake:157(46615ns) {DropHandshakes:470(1370ns) {DropIncomingHandshake:442(415ns) DropOutgoingHandshake:457(185ns)} GenerateSessionVirtualId:988(315ns) PrepareNewSessionHandshake:477(1066ns) {SwitchToState:179(743ns)}}}}}
2025-11-28T14:53:24.825296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789490276870322:2107] [node 5] ICP01 ready to work
2025-11-28T14:53:24.823964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789490958768874:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(155104284ns) {Configure:104(155097843ns) {ProcessConfigured:135(49535ns) {StartInitialHandshake:157(47236ns) {DropHandshakes:470(4898ns) {DropIncomingHandshake:442(743ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(1514ns) {SwitchToState:179(600ns)}}}}}
2025-11-28T14:53:24.822885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789488510560477:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:24.838830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789490706412225:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26966
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.836026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789490698608191:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(155073975ns) {Configure:104(155070286ns) {ProcessConfigured:135(84499ns) {StartInitialHandshake:157(82187ns) {DropHandshakes:470(3055ns) {DropIncomingHandshake:442(579ns) DropOutgoingHandshake:457(482ns)} GenerateSessionVirtualId:988(720ns) PrepareNewSessionHandshake:477(2429ns) {SwitchToState:179(1116ns)}}}}}
2025-11-28T14:53:24.867934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789490698608269:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
2025-11-28T14:53:24.867998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789490698608270:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.877258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789490958768943:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:597
2025-11-28T14:53:24.856862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789490706412225:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.836601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789488510560482:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56608
2025-11-28T14:53:24.883769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789490706412235:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26966
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.895808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789489038884790:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-28T14:53:24.890344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789488510560476:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.893139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789484215593113:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(212182041ns) {Configure:104(212173653ns) {ProcessConfigured:135(72112ns) {StartInitialHandshake:157(66878ns) {DropHandshakes:470(2493ns) {DropIncomingHandshake:442(422ns) DropOutgoingHandshake:457(552ns)} GenerateSessionVirtualId:988(476ns) PrepareNewSessionHandshake:477(2502ns) {SwitchToState:179(1199ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.906700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789490706412235:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.877923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789490698608193:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(96269623ns) {Configure:104(96260471ns) {ProcessConfigured:135(9401545ns) {StartInitialHandshake:157(9399428ns) {DropHandshakes:470(3436ns) {DropIncomingHandshake:442(951ns) DropOutgoingHandshake:457(724ns)} GenerateSessionVirtualId:988(1731ns) PrepareNewSessionHandshake:477(2525ns) {SwitchToState:179(907ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.884256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789490276870320:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43054
2025-11-28T14:53:24.900569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789490276870323:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43054
2025-11-28T14:53:24.900723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789490276870325:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43054
2025-11-28T14:53:24.916770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789490276870325:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.917130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789490276870320:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.921364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789490276870323:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.923452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789490276870238:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:24.923512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789490276870239:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:24.907117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789490698608266:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
2025-11-28T14:53:24.913051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789489038884791:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-28T14:53:24.941004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789490958768943:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.925449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789490698608269:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.928176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789488510560482:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.934543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789490698608266:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.923860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789489038884782:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.924944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789490706412239:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26966
2025-11-28T14:53:24.944680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789490698608214:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:24.950178Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789490276870321:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26671651ns) {Configure:104(26667755ns) {ProcessConfigured:135(24833834ns) {StartInitialHandshake:157(24831419ns) {DropHandshakes:470(2115ns) {DropIncomingHandshake:442(213ns) DropOutgoingHandshake:457(408ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:477(2951ns) {SwitchToState:179(1268ns)}}}}}
2025-11-28T14:53:24.962646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789490276870342:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43054
2025-11-28T14:53:24.980428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789490706412233:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.980497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789490706412233:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.984770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789490706412233:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:24.985210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789490706412223:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:24.985297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:24.985364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789490706412223:2100] [node 6] ICP22 created new session: [5:7577789490706412243:2100]
2025-11-28T14:53:24.985410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789490706412243:2100] [node 6] ICS09 handshake done sender: [5:7577789490706412233:2113] self: [5:7577789490706412232:0] peer: [6:7577789490698608274:0] socket: 380 qp: -1
2025-11-28T14:53:24.985446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789490706412243:2100] [node 6] ICS10 traffic start
2025-11-28T14:53:24.985480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:24.923885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789489038884782:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:24.936424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789488510560474:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.963351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789490276870343:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43054
2025-11-28T14:53:25.008489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789490958768873:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(213207863ns) {Configure:104(213194064ns) {ProcessConfigured:135(8388923ns) {StartInitialHandshake:157(8386397ns) {DropHandshakes:470(1867ns) {DropIncomingHandshake:442(180ns) DropOutgoingHandshake:457(536ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:477(2794ns) {SwitchToState:179(992ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.936471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789488510560474:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:24.935453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789489038884782:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.012451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789490958768937:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.948692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789490698608192:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:24.973016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789488510560479:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.009216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789490706412160:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(240079267ns) {Configure:104(240070487ns) {ProcessConfigured:135(43036916ns) {StartInitialHandshake:157(43033602ns) {DropHandshakes:470(12688138ns) {DropIncomingHandshake:442(2897ns) DropOutgoingHandshake:457(569ns)} GenerateSessionVirtualId:988(2313ns) PrepareNewSessionHandshake:477(3307ns) {SwitchToState:179(1858ns)}}}}}
2025-11-28T14:53:25.013359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789495001379542:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26966
2025-11-28T14:53:25.012529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789490958768937:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.008649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789490276870324:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43054
2025-11-28T14:53:24.960583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789490698608270:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.992944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789484743917414:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.025881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789490706412239:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.037627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789490276870335:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:24.993036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:24.973065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789488510560479:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.960897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789490698608254:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.029367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789495253736247:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:597
2025-11-28T14:53:24.992716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789488510560474:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.037657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789490276870335:2135] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.026048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789490706412162:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:25.032652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789490958768937:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:24.981839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789490698608276:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
2025-11-28T14:53:24.993094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789484743917414:2049] [node 3] ICP22 created new session: [1:7577789489038884795:2049]
2025-11-28T14:53:25.038371Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789490276870320:2128] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:25.046288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789495001379542:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.049425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789484215593114:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:24.993153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789489038884795:2049] [node 3] ICS09 handshake done sender: [1:7577789489038884782:2119] self: [1:7577789489038884781:0] peer: [3:7577789490276870339:0] socket: 361 qp: -1
2025-11-28T14:53:24.998582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789490698608277:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
2025-11-28T14:53:25.042737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789490958768872:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(236075385ns) {Configure:104(236064906ns) {ProcessConfigured:135(13950459ns) {StartInitialHandshake:157(13946691ns) {DropHandshakes:470(3804ns) {DropIncomingHandshake:442(931ns) DropOutgoingHandshake:457(714ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:477(17857ns) {SwitchToState:179(696ns)}}}}}
2025-11-28T14:53:25.049594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789488510560479:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:25.058104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789490706412224:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.998648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789490698608278:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
2025-11-28T14:53:25.077726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789495253736250:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:597
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:24.993195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789489038884795:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:25.046501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789490276870324:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.073870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789488510560475:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.077793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789495253736251:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:597
2025-11-28T14:53:25.080214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789490958768934:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.088196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789490958768945:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.088235Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789490958768945:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.092477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789495253736247:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.092868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789490958768872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:25.106314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:25.106920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789490958768945:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:25.107659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789490958768934:2108] [node 6] ICP22 created new session: [4:7577789495253736254:2108]
2025-11-28T14:53:25.110022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789495253736250:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.110071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789490958768938:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.110158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:25.110209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789490958768938:2109] [node 5] ICP22 created new session: [4:7577789495253736256:2109]
2025-11-28T14:53:25.110256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789495253736256:2109] [node 5] ICS09 handshake done sender: [4:7577789490958768945:2113] self: [4:7577789490958768944:0] peer: [5:7577789495001379544:0] socket: 408 qp: -1
2025-11-28T14:53:25.110285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789495253736256:2109] [node 5] ICS10 traffic start
2025-11-28T14:53:25.117444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789495253736254:2108] [node 6] ICS09 handshake done sender: [4:7577789490958768937:2110] self: [4:7577789490958768936:0] peer: [6:7577789490698608275:0] socket: 383 qp: -1
2025-11-28T14:53:25.118575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789495253736251:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.118719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789490958768874:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:25.124533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789495253736254:2108] [node 6] ICS10 traffic start
2025-11-28T14:53:25.132010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789495253736260:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:597
2025-11-28T14:53:25.138995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:25.140578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:25.085822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789495001379545:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26966
2025-11-28T14:53:25.106098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789495001379547:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26966
2025-11-28T14:53:25.106488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789490706412239:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:25.106640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789490706412160:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:25.127312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789490706412224:2104] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21762) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789495001379542:2118] held: no
2025-11-28T14:53:25.128032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789495001379542:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:25.128541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789490706412162:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.128619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:25.128667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789490706412162:2050] [node 3] ICP22 created new session: [5:7577789495001379549:2050]
2025-11-28T14:53:25.128704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789495001379549:2050] [node 3] ICS09 handshake done sender: [5:7577789490706412239:2116] self: [5:7577789495001379543:0] peer: [3:7577789490276870334:0] socket: 397 qp: -1
2025-11-28T14:53:25.128738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789495001379549:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:25.128781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:25.129861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789495001379548:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26966
2025-11-28T14:53:25.046565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789490276870238:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:24.993239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.009275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789490698608278:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.079131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:25.002378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789489038884786:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(70724028ns) {Configure:104(70720459ns) {ProcessConfigured:135(19037951ns) {StartInitialHandshake:157(19034822ns) {DropHandshakes:470(1960ns) {DropIncomingHandshake:442(445ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(399ns) PrepareNewSessionHandshake:477(3157ns) {SwitchToState:179(1388ns)}}}}}
2025-11-28T14:53:25.161363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789495253736261:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:597
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.020562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789490698608214:2069] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22121) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789490698608269:2116] held: no
2025-11-28T14:53:25.167821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789490706412224:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.052777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789490276870323:2129] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:25.088737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789484215593114:2049] [node 3] ICP22 created new session: [2:7577789492805527785:2049]
2025-11-28T14:53:25.088804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789492805527785:2049] [node 3] ICS09 handshake done sender: [2:7577789488510560474:2110] self: [2:7577789488510560473:0] peer: [3:7577789490276870340:0] socket: 367 qp: -1
2025-11-28T14:53:25.088851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789492805527785:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:25.088898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:25.091581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:25.100613Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789488510560477:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(67631523ns) {Configure:104(67623321ns) {ProcessConfigured:135(71109ns) {StartInitialHandshake:157(68241ns) {DropHandshakes:470(2795ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(541ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(1856ns) {SwitchToState:179(869ns)}}}}}
2025-11-28T14:53:25.105393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789492805527790:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56608
2025-11-28T14:53:25.107536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789492805527791:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56608
2025-11-28T14:53:25.108997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789488510560475:2109] [node 6] ICP22 created new session: [2:7577789492805527787:2109]
2025-11-28T14:53:25.115165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789492805527787:2109] [node 6] ICS09 handshake done sender: [2:7577789488510560479:2112] self: [2:7577789488510560478:0] peer: [6:7577789490698608273:0] socket: 368 qp: -1
2025-11-28T14:53:25.117652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789492805527787:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:25.143260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789492805527791:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.143540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:25.172542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789492805527790:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.174463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789484215593113:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:25.205892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789488510560475:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(130980592ns) {HandleHandshakeStatus:329(130977552ns) {DropIncomingHandshake:442(15253ns) DropOutgoingHandshake:457(562ns) SwitchToState:179(884ns) ProcessPendingSessionEvents:495(17898919ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17827704ns max# 17383477ns}}}
2025-11-28T14:53:25.207947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789484215593114:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(40153283ns) {HandleHandshakeStatus:329(40150347ns) {DropIncomingHandshake:442(741ns) DropOutgoingHandshake:457(925ns) SwitchToState:179(1073ns) ProcessPendingSessionEvents:495(528518ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 462367ns max# 63156ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.213672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789492805527797:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56608
2025-11-28T14:53:25.167444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789495253736247:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:25.009111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789489038884790:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.168164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789490706412225:2110] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:25.053308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789490276870335:2135] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:25.217503Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789488510560472:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(207865778ns) {Configure:104(207858177ns) {ProcessConfigured:135(6911810ns) {StartInitialHandshake:157(6907770ns) {DropHandshakes:470(3435ns) {DropIncomingHandshake:442(723ns) DropOutgoingHandshake:457(605ns)} GenerateSessionVirtualId:988(841ns) PrepareNewSessionHandshake:477(3007ns) {SwitchToState:179(1425ns)}}}}}
2025-11-28T14:53:25.225319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789492805527790:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:25.021064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789490698608191:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:25.236617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789492805527789:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.236666Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789492805527789:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.237671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789492805527789:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:25.237990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789488510560477:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.238085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:25.238133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789488510560477:2107] [node 4] ICP22 created new session: [2:7577789492805527802:2107]
2025-11-28T14:53:25.238179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789492805527802:2107] [node 4] ICS09 handshake done sender: [2:7577789492805527789:2115] self: [2:7577789492805527788:0] peer: [4:7577789495253736268:0] socket: 428 qp: -1
2025-11-28T14:53:25.238208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789492805527802:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:25.238258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.168148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789490958768872:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.184662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789490706412160:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.240680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789484215593113:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.040831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789489038884798:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.062163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789490276870239:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.168238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:25.042575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789490698608272:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.184758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:25.062258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:25.062313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789490276870239:2049] [node 2] ICP22 created new session: [3:7577789494571837642:2049]
2025-11-28T14:53:25.062356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789494571837642:2049] [node 2] ICS09 handshake done sender: [3:7577789490276870323:2129] self: [3:7577789490276870340:0] peer: [2:7577789488510560473:0] socket: 370 qp: -1
2025-11-28T14:53:25.040878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789489038884798:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.062408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789494571837642:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:25.069511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789494571837640:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43054
2025-11-28T14:53:25.079147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:25.079247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789494571837640:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.088025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789490276870322:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.088540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:25.090703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:25.090769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789490276870322:2107] [node 5] ICP22 created new session: [3:7577789494571837648:2107]
2025-11-28T14:53:25.091312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789490276870327:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.091346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789490276870327:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.099012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789494571837648:2107] [node 5] ICS09 handshake done sender: [3:7577789490276870335:2135] self: [3:7577789490276870334:0] peer: [5:7577789495001379543:0] socket: 395 qp: -1
2025-11-28T14:53:25.106574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789490276870238:2048] [node 1] ICP22 created new session: [3:7577789494571837647:2048]
2025-11-28T14:53:25.106661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789494571837648:2107] [node 5] ICS10 traffic start
2025-11-28T14:53:25.107654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789494571837647:2048] [node 1] ICS09 handshake done sender: [3:7577789490276870320:2128] self: [3:7577789490276870339:0] peer: [1:7577789489038884781:0] socket: 363 qp: -1
2025-11-28T14:53:25.113518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789490276870239:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(50429992ns) {HandleHandshakeStatus:329(50427703ns) {DropIncomingHandshake:442(1533ns) DropOutgoingHandshake:457(5139ns) SwitchToState:179(695ns) ProcessPendingSessionEvents:495(15250781ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15182274ns max# 14743823ns}}}
2025-11-28T14:53:25.113683Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789490276870327:2132] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:25.114142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789494571837647:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:25.114186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:25.114522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:25.141856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789490276870333:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.141889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789490276870333:2134] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.151024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789490276870319:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.169199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789490276870333:2134] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:25.181680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789490276870322:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(101349462ns) {HandleHandshakeStatus:329(101346758ns) {DropIncomingHandshake:442(2274ns) DropOutgoingHandshake:457(1210ns) SwitchToState:179(1354ns) ProcessPendingSessionEvents:495(10305687ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10226947ns max# 9768391ns}}}
2025-11-28T14:53:25.181779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577789490276870322:2107] [node 5] ICP18 handshake (actor [3:7577789494571837640:2139]) is too old
2025-11-28T14:53:25.186657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:25.186712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789490276870319:2126] [node 6] ICP22 created new session: [3:7577789494571837653:2126]
2025-11-28T14:53:25.217087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789490276870321:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.220611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789494571837653:2126] [node 6] ICS09 handshake done sender: [3:7577789490276870327:2132] self: [3:7577789490276870326:0] peer: [6:7577789494993575579:0] socket: 391 qp: -1
2025-11-28T14:53:25.232981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:25.234384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789494571837653:2126] [node 6] ICS10 traffic start
2025-11-28T14:53:25.234791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789490276870321:2108] [node 4] ICP22 created new session: [3:7577789494571837654:2108]
2025-11-28T14:53:25.253751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:25.256834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789494571837654:2108] [node 4] ICS09 handshake done sender: [3:7577789490276870333:2134] self: [3:7577789490276870328:0] peer: [4:7577789495253736258:0] socket: 393 qp: -1
2025-11-28T14:53:25.256887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789494571837654:2108] [node 4] ICS10 traffic start
2025-11-28T14:53:25.256977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:25.258682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789490276870321:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(40657501ns) {HandleHandshakeStatus:329(40653322ns) {DropIncomingHandshake:442(28119ns) DropOutgoingHandshake:457(756ns) SwitchToState:179(835ns) ProcessPendingSessionEvents:495(575718ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 496160ns max# 53353ns}}}
2025-11-28T14:53:25.168294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789490958768872:2048] [node 1] ICP22 created new session: [4:7577789495253736263:2048]
2025-11-28T14:53:25.058175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789490698608272:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.277497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:25.184815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789490706412160:2048] [node 1] ICP22 created new session: [5:7577789495001379552:2048]
2025-11-28T14:53:25.041993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789484743917413:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(189092090ns) {Configure:104(189081124ns) {ProcessConfigured:135(107184ns) {StartInitialHandshake:157(104386ns) {DropHandshakes:470(3679ns) {DropIncomingHandshake:442(736ns) DropOutgoingHandshake:457(472ns)} GenerateSessionVirtualId:988(910ns) PrepareNewSessionHandshake:477(2701ns) {SwitchToState:179(863ns)}}}}}
2025-11-28T14:53:25.168346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789495253736263:2048] [node 1] ICS09 handshake done sender: [4:7577789495253736247:2115] self: [4:7577789495253736253:0] peer: [1:7577789489038884793:0] socket: 411 qp: -1
2025-11-28T14:53:25.184868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789495001379552:2048] [node 1] ICS09 handshake done sender: [5:7577789490706412225:2110] self: [5:7577789495001379546:0] peer: [1:7577789489038884787:0] socket: 365 qp: -1
2025-11-28T14:53:25.048464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789489038884798:2124] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:25.049695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789489038884791:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.049838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789484743917414:2049] [node 3] ICP18 handshake (actor [1:7577789489038884791:2122]) is too old
2025-11-28T14:53:25.058604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789489038884784:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.091343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:25.093237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789489038884794:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.093261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789489038884794:2123] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.106597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789489038884784:2109] [node 6] ICP22 created new session: [1:7577789493333852102:2109]
2025-11-28T14:53:25.106821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789489038884788:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.106836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789489038884788:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.107827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789489038884794:2123] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:25.114594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789493333852102:2109] [node 6] ICS09 handshake done sender: [1:7577789489038884798:2124] self: [1:7577789489038884797:0] peer: [6:7577789494993575575:0] socket: 405 qp: -1
2025-11-28T14:53:25.119044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789493333852102:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:25.119556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789489038884788:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:25.143602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:25.165253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789489038884785:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.165336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:25.165403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789489038884785:2108] [node 5] ICP22 created new session: [1:7577789493333852108:2108]
2025-11-28T14:53:25.165459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789493333852108:2108] [node 5] ICS09 handshake done sender: [1:7577789489038884788:2120] self: [1:7577789489038884787:0] peer: [5:7577789495001379546:0] socket: 364 qp: -1
2025-11-28T14:53:25.165493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789493333852108:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:25.165556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:25.161307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789489038884786:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.177089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789493333852107:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-28T14:53:25.178331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789493333852098:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.178359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789493333852098:2125] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.192593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789493333852098:2125] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:25.202572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789489038884784:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(142898068ns) {HandleHandshakeStatus:329(142894560ns) {DropIncomingHandshake:442(742ns) DropOutgoingHandshake:457(1033ns) SwitchToState:179(255ns) ProcessPendingSessionEvents:495(11804607ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11741726ns max# 11309719ns}}}
2025-11-28T14:53:25.206361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:25.061041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789490698608214:2069] [node 5] ICP35 event processing took too much time PendingConnection:279(57738508ns) {HandleHandshakeStatus:395(57734982ns) {LogHandshakeFail:472(14359ns) {UpdateErrorStateLog:545(5307ns)} DropOutgoingHandshake:457(2508ns) ProcessPendingSessionEvents:495(39591891ns) {unordered DropSessionEvent:512 num# 100 sum# 39536169ns max# 11360086ns}}}
2025-11-28T14:53:25.206414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789489038884786:2107] [node 4] ICP22 created new session: [1:7577789493333852111:2107]
2025-11-28T14:53:25.214781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789493333852111:2107] [node 4] ICS09 handshake done sender: [1:7577789489038884794:2123] self: [1:7577789489038884793:0] peer: [4:7577789495253736253:0] socket: 410 qp: -1
2025-11-28T14:53:25.227396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789484743917413:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.232681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789493333852111:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:25.232750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:25.259547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:25.259601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789484743917413:2048] [node 2] ICP22 created new session: [1:7577789493333852113:2048]
2025-11-28T14:53:25.270620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789493333852107:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.271101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789493333852106:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33357
2025-11-28T14:53:25.285644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789493333852113:2048] [node 2] ICS09 handshake done sender: [1:7577789493333852098:2125] self: [1:7577789493333852097:0] peer: [2:7577789492805527795:0] socket: 420 qp: -1
2025-11-28T14:53:25.289094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789493333852113:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:25.293443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:25.279289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789484215593113:2048] [node 1] ICP22 created new session: [2:7577789492805527804:2048]
2025-11-28T14:53:25.061498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789494993575576:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
2025-11-28T14:53:25.168396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789495253736263:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:25.292704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789492805527804:2048] [node 1] ICS09 handshake done sender: [2:7577789492805527790:2116] self: [2:7577789492805527795:0] peer: [1:7577789493333852097:0] socket: 423 qp: -1
2025-11-28T14:53:25.184904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789495001379552:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:25.168450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:25.062570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789490698608269:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:25.322702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789489038884786:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(171054731ns) {HandleHandshakeStatus:329(158479796ns) {DropIncomingHandshake:442(1179ns) DropOutgoingHandshake:457(1762ns) SwitchToState:179(468ns) ProcessPendingSessionEvents:495(42457452ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 42332339ns max# 15256739ns}}}
2025-11-28T14:53:25.322937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789489038884786:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:25.200800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789495253736265:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:597
2025-11-28T14:53:25.220472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789490958768938:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(73705850ns) {HandleHandshakeStatus:329(73702957ns) {DropIncomingHandshake:442(13202ns) DropOutgoingHandshake:457(912ns) SwitchToState:179(857ns) ProcessPendingSessionEvents:495(12722253ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12656394ns max# 12255862ns}}}
2025-11-28T14:53:25.224290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789490958768934:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(143165358ns) {HandleHandshakeStatus:329(143163027ns) {DropIncomingHandshake:442(180ns) DropOutgoingHandshake:457(414ns) SwitchToState:179(108ns) ProcessPendingSessionEvents:495(12707789ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6971172ns max# 6564449ns}}}
2025-11-28T14:53:25.224926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789495253736251:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:25.225760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789495253736261:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.229179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789490958768934:2108] [node 6] ICP18 handshake (actor [4:7577789495253736250:2117]) is too old
2025-11-28T14:53:25.229829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789490958768874:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789495253736251:2118] held: no
2025-11-28T14:53:25.235543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789490958768874:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.235635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:25.235687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789490958768874:2050] [node 3] ICP22 created new session: [4:7577789495253736266:2050]
2025-11-28T14:53:25.235729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789495253736266:2050] [node 3] ICS09 handshake done sender: [4:7577789495253736251:2118] self: [4:7577789495253736258:0] peer: [3:7577789490276870328:0] socket: 419 qp: -1
2025-11-28T14:53:25.235765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789495253736266:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:25.235798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:25.236069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789490958768873:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:25.240184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789495253736269:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:597
2025-11-28T14:53:25.245927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789495253736261:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:25.246135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789490958768873:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8413) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789495253736261:2120] held: no
2025-11-28T14:53:25.248846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789490958768873:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.279155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:25.285674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789490958768873:2049] [node 2] ICP22 created new session: [4:7577789495253736270:2049]
2025-11-28T14:53:25.293508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789495253736270:2049] [node 2] ICS09 handshake done sender: [4:7577789495253736261:2120] self: [4:7577789495253736268:0] peer: [2:7577789492805527788:0] socket: 433 qp: -1
2025-11-28T14:53:25.293868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789495253736270:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:25.302046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:25.068383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789490698608193:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:25.068923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789494993575577:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
2025-11-28T14:53:25.080122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789490698608214:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.082672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789490698608278:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:25.090076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789490698608266:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:25.295362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789492805527804:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:25.328642Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789489038884785:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(131389288ns) {HandleHandshakeStatus:329(131386528ns) {DropIncomingHandshake:442(1238ns) DropOutgoingHandshake:457(936ns) SwitchToState:179(1163ns) ProcessPendingSessionEvents:495(130858540ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 129266632ns max# 12063101ns}}}
2025-11-28T14:53:25.184956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:25.091088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:25.330831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789489038884785:2108] [node 5] ICP18 handshake (actor [1:7577789493333852107:2127]) is too old
2025-11-28T14:53:25.200245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:25.343404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789490958768873:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(83486916ns) {HandleHandshakeStatus:329(83484849ns) {DropIncomingHandshake:442(1258ns) DropOutgoingHandshake:457(2114ns) SwitchToState:179(818ns) ProcessPendingSessionEvents:495(702ns)}}
2025-11-28T14:53:25.296652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789492805527798:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.091216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789490698608270:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:25.200321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789490706412224:2104] [node 4] ICP22 created new session: [5:7577789495001379554:2104]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:25.095200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789490698608191:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.296682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789492805527798:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:25.206990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789495001379554:2104] [node 4] ICS09 handshake done sender: [5:7577789495001379542:2118] self: [5:7577789495001379544:0] peer: [4:7577789490958768944:0] socket: 409 qp: -1
2025-11-28T14:53:25.358314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789490958768872:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(163413828ns) {HandleHandshakeStatus:329(163411921ns) {DropIncomingHandshake:442(242ns) DropOutgoingHandshake:457(4562ns) SwitchToState:179(389ns) ProcessPendingSessionEvents:495(162947762ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 148243213ns max# 16930822ns}}}
2025-11-28T14:53:25.211429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789495001379554:2104] [node 4] ICS10 traffic start
2025-11-28T14:53:25.214529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:25.219656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789490706412160:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(34011004ns) {HandleHandshakeStatus:329(34008581ns) {DropIncomingHandshake:442(1327ns) DropOutgoingHandshake:457(5629ns) SwitchToState:179(1037ns) ProcessPendingSessionEvents:495(7924946ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7856994ns max# 7425555ns}}}
2025-11-28T14:53:25.259369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789495001379556:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26966
2025-11-28T14:53:25.270665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789490706412224:2104] [node 4] ICP35 event processing took too much time PendingConnection:279(102704298ns) {HandleHandshakeStatus:329(102701015ns) {DropIncomingHandshake:442(1434ns) DropOutgoingHandshake:457(666ns) SwitchToState:179(772ns) ProcessPendingSessionEvents:495(1726ns)}}
2025-11-28T14:53:25.277566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789495001379556:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:25.294466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789490706412161:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:25.352115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789490706412161:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8413) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789495001379556:2122] held: no
2025-11-28T14:53:25.356259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789495001379558:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26966
2025-11-28T14:53:25.095296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:25.095350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789490698608191:2048] [node 1] ICP22 created new session: [6:7577789494993575581:2048]
2025-11-28T14:53:25.095400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789494993575581:2048] [node 1] ICS09 handshake done sender: [6:7577789490698608278:2121] self: [6:7577789494993575575:0] peer: [1:7577789489038884797:0] socket: 407 qp: -1
2025-11-28T14:53:25.095431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789494993575581:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:25.095482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:25.106435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789490698608214:2069] [node 5] ICP22 created new session: [6:7577789494993575580:2069]
2025-11-28T14:53:25.117507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789494993575580:2069] [node 5] ICS09 handshake done sender: [6:7577789490698608269:2116] self: [6:7577789490698608274:0] peer: [5:7577789490706412232:0] socket: 381 qp: -1
2025-11-28T14:53:25.118995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789490698608254:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.125005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789490698608192:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.125106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:25.125155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789490698608192:2049] [node 2] ICP22 created new session: [6:7577789494993575583:2049]
2025-11-28T14:53:25.125205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789494993575583:2049] [node 2] ICS09 handshake done sender: [6:7577789490698608266:2113] self: [6:7577789490698608273:0] peer: [2:7577789488510560478:0] socket: 369 qp: -1
2025-11-28T14:53:25.125237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789494993575583:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:25.125295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:25.126122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789490698608192:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:25.126710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789494993575585:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17739
2025-11-28T14:53:25.131072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789490698608272:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:25.131314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789490698608193:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.131376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:25.131420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789490698608193:2050] [node 3] ICP22 created new session: [6:7577789494993575586:2050]
2025-11-28T14:53:25.131461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789494993575586:2050] [node 3] ICS09 handshake done sender: [6:7577789490698608272:2118] self: [6:7577789494993575579:0] peer: [3:7577789490276870326:0] socket: 394 qp: -1
2025-11-28T14:53:25.131493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789494993575586:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:25.131519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:25.144005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789494993575580:2069] [node 5] ICS10 traffic start
2025-11-28T14:53:25.146312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:25.173735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:25.189218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789490698608254:2066] [node 4] ICP22 created new session: [6:7577789494993575589:2066]
2025-11-28T14:53:25.193095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789490698608214:2069] [node 5] ICP35 event processing took too much time PendingConnection:279(105156020ns) {HandleHandshakeStatus:329(105153676ns) {DropIncomingHandshake:442(1134ns) DropOutgoingHandshake:457(523ns) SwitchToState:179(907ns) ProcessPendingSessionEvents:495(1474ns)}}
2025-11-28T14:53:25.214981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789494993575589:2066] [node 4] ICS09 handshake done sender: [6:7577789490698608270:2117] self: [6:7577789490698608275:0] peer: [4:7577789490958768936:0] socket: 385 qp: -1
2025-11-28T14:53:25.221331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789494993575589:2066] [node 4] ICS10 traffic start
2025-11-28T14:53:25.232894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:25.314932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789490698608254:2066] [node 4] ICP35 event processing took too much time PendingConnection:279(194888260ns) {HandleHandshakeStatus:329(194885059ns) {DropIncomingHandshake:442(1219ns) DropOutgoingHandshake:457(39089ns) SwitchToState:179(849ns) ProcessPendingSessionEvents:495(24675086ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17007313ns max# 7785080ns}}}
2025-11-28T14:53:25.301886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:25.324726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789492805527798:2119] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:25.327771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789488510560472:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:25.342820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:25.350027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789488510560472:2108] [node 5] ICP22 created new session: [2:7577789492805527807:2108]
2025-11-28T14:53:25.350064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789492805527807:2108] [node 5] ICS09 handshake done sender: [2:7577789492805527798:2119] self: [2:7577789492805527796:0] peer: [5:7577789495001379557:0] socket: 392 qp: -1
2025-11-28T14:53:25.350093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789492805527807:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:25.350128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:25.369143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789495001379556:2122] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:25.370069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789488510560472:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(38877089ns) {HandleHandshakeStatus:329(38873631ns) {DropIncomingHandshake:442(17699ns) DropOutgoingHandshake:457(660ns) SwitchToState:179(2605ns) ProcessPendingSessionEvents:495(1542375ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1471464ns max# 1022617ns}}}
2025-11-28T14:53:25.364022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789490698608193:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(231687277ns) {HandleHandshakeStatus:329(231685313ns) {DropIncomingHandshake:442(508ns) DropOutgoingHandshake:457(4111ns) SwitchToState:179(821ns) ProcessPendingSessionEvents:495(231353055ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 231143714ns max# 16431772ns}}}
2025-11-28T14:53:25.372272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789490276870319:2126] [node 6] ICP35 event processing took too much time PendingConnection:279(218656994ns) {HandleHandshakeStatus:329(218654417ns) {DropIncomingHandshake:442(21075ns) DropOutgoingHandshake:457(1001ns) SwitchToState:179(756ns) ProcessPendingSessionEvents:495(72123992ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 70254371ns max# 9123427ns}}}
2025-11-28T14:53:25.376911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789490276870238:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(329344165ns) {HandleHandshakeStatus:329(329341199ns) {DropIncomingHandshake:442(1320ns) DropOutgoingHandshake:457(5964ns) SwitchToState:179(939ns) ProcessPendingSessionEvents:495(161409077ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 161255106ns max# 33591261ns}}}
2025-11-28T14:53:25.379908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789484743917413:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(164369986ns) {HandleHandshakeStatus:329(158077705ns) {DropIncomingHandshake:442(28634ns) DropOutgoingHandshake:457(780ns) SwitchToState:179(1314ns) ProcessPendingSessionEvents:495(61425026ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56682504ns max# 6043689ns}}}
2025-11-28T14:53:25.381082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789490706412161:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:25.381501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:25.381552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789490706412161:2049] [node 2] ICP22 created new session: [5:7577789495001379559:2049]
2025-11-28T14:53:25.381594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789495001379559:2049] [node 2] ICS09 handshake done sender: [5:7577789495001379556:2122] self: [5:7577789495001379557:0] peer: [2:7577789492805527796:0] socket: 404 qp: -1
2025-11-28T14:53:25.381628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789495001379559:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:25.381696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:25.388981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789484215593113:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(153470642ns) {HandleHandshakeStatus:329(153466732ns) {DropIncomingHandshake:442(1932ns) DropOutgoingHandshake:457(5083ns) SwitchToState:179(1256ns) ProcessPendingSessionEvents:495(58331858ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 58241303ns max# 13363376ns}}}
2025-11-28T14:53:25.419389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:25.419623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789492805527804:2048] [node 1] ICS07 socket disconnect 423 reason# EndOfStream
2025-11-28T14:53:25.419650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789492805527804:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.419723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:25.419765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789492805527804:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:25.422686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:25.422926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789494571837647:2048] [node 1] ICS07 socket disconnect 363 reason# EndOfStream
2025-11-28T14:53:25.422954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789494571837647:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.423027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:25.423066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789494571837647:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:25.425098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789484215593113:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:25.425137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789492805527804:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:25.425852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789484215593113:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:25.425895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789492805527804:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:25.425929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789484215593113:2048] [node 1] ICP30 unregister session Session# [2:7577789492805527804:2048] VirtualId# [2:7577789492805527795:0]
2025-11-28T14:53:25.426033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:25.426062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789484215593113:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:25.426076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:25.424825Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789490276870238:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:25.424901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789494571837647:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:25.424948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789490276870238:2048] [node 1] ICP30 unregister session Session# [3:7577789494571837647:2048] VirtualId# [3:7577789490276870339:0]
2025-11-28T14:53:25.433535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:25.433735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789495001379552:2048] [node 1] ICS01 socket: 365 reason# EndOfStream
2025-11-28T14:53:25.433776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789490706412160:2048] [node 1] ICP30 unregister session Session# [5:7577789495001379552:2048] VirtualId# [5:7577789495001379546:0]
2025-11-28T14:53:25.433811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789495001379552:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.433874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:25.433921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:25.427031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:25.427075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789490276870238:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:25.427089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:25.432607Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:25.432866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789495253736263:2048] [node 1] ICS07 socket disconnect 411 reason# EndOfStream
2025-11-28T14:53:25.432894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789495253736263:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.432979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:25.433026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789495253736263:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:25.435708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789490958768872:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:25.435756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789495253736263:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:25.435785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789490958768872:2048] [node 1] ICP30 unregister session Session# [4:7577789495253736263:2048] VirtualId# [4:7577789495253736253:0]
2025-11-28T14:53:25.435896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:25.435928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789490958768872:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:25.435946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19842) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:25.434005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:25.434200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789494993575581:2048] [node 1] ICS01 socket: 407 reason# EndOfStream
2025-11-28T14:53:25.434231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789490698608191:2048] [node 1] ICP30 unregister session Session# [6:7577789494993575581:2048] VirtualId# [6:7577789494993575575:0]
2025-11-28T14:53:25.434256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789494993575581:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.434313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:25.434354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:25.437304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789484215593113:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:25.437352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789490276870238:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:25.447165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789490958768872:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:25.528576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:25.528807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789494571837642:2049] [node 2] ICS01 socket: 370 reason# EndOfStream
2025-11-28T14:53:25.528846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789490276870239:2049] [node 2] ICP30 unregister session Session# [3:7577789494571837642:2049] VirtualId# [3:7577789490276870340:0]
2025-11-28T14:53:25.528959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789494571837642:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.529039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:25.529090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:25.536609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:25.536830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789495253736270:2049] [node 2] ICS01 socket: 433 reason# EndOfStream
2025-11-28T14:53:25.536879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789490958768873:2049] [node 2] ICP30 unregister session Session# [4:7577789495253736270:2049] VirtualId# [4:7577789495253736268:0]
2025-11-28T14:53:25.536914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789495253736270:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.536997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:25.537047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:25.537134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:25.537318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789495001379559:2049] [node 2] ICS01 socket: 404 reason# EndOfStream
2025-11-28T14:53:25.537356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789490706412161:2049] [node 2] ICP30 unregister session Session# [5:7577789495001379559:2049] VirtualId# [5:7577789495001379557:0]
2025-11-28T14:53:25.537381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789495001379559:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.537432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:25.537474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:25.537556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:25.537714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789494993575583:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-11-28T14:53:25.537745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789490698608192:2049] [node 2] ICP30 unregister session Session# [6:7577789494993575583:2049] VirtualId# [6:7577789490698608273:0]
2025-11-28T14:53:25.537767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789494993575583:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.537813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:25.537852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:25.656637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:25.656883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789494993575586:2050] [node 3] ICS01 socket: 394 reason# EndOfStream
2025-11-28T14:53:25.656927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789490698608193:2050] [node 3] ICP30 unregister session Session# [6:7577789494993575586:2050] VirtualId# [6:7577789494993575579:0]
2025-11-28T14:53:25.656959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789494993575586:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.657050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:25.657112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:25.656622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:25.656882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789495001379549:2050] [node 3] ICS01 socket: 397 reason# EndOfStream
2025-11-28T14:53:25.656924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789490706412162:2050] [node 3] ICP30 unregister session Session# [5:7577789495001379549:2050] VirtualId# [5:7577789495001379543:0]
2025-11-28T14:53:25.656960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789495001379549:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.657040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:25.657088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:25.657299Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:25.657514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789495253736266:2050] [node 3] ICS01 socket: 419 reason# EndOfStream
2025-11-28T14:53:25.657549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789490958768874:2050] [node 3] ICP30 unregister session Session# [4:7577789495253736266:2050] VirtualId# [4:7577789495253736258:0]
2025-11-28T14:53:25.657591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789495253736266:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.657653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:25.657692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:25.849580Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:25.849801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789495001379554:2104] [node 4] ICS01 socket: 409 reason# EndOfStream
2025-11-28T14:53:25.849841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789490706412224:2104] [node 4] ICP30 unregister session Session# [5:7577789495001379554:2104] VirtualId# [5:7577789495001379544:0]
2025-11-28T14:53:25.849878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789495001379554:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.849941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:25.849984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:25.848597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:25.848845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789494993575589:2066] [node 4] ICS07 socket disconnect 385 reason# EndOfStream
2025-11-28T14:53:25.848873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789494993575589:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.848975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:25.849023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789494993575589:2066] [node 4] ICS15 start handshake
2025-11-28T14:53:25.850840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789490698608254:2066] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21762) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:25.850886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789494993575589:2066] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:25.850929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789490698608254:2066] [node 4] ICP30 unregister session Session# [6:7577789494993575589:2066] VirtualId# [6:7577789490698608275:0]
2025-11-28T14:53:25.851036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:25.851063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789490698608254:2066] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:25.851078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:25.860562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789490698608254:2066] [node 4] ICP33 wake up from error state
2025-11-28T14:53:25.992703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:25.992962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789494993575580:2069] [node 5] ICS01 socket: 381 reason# EndOfStream
2025-11-28T14:53:25.993010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789490698608214:2069] [node 5] ICP30 unregister session Session# [6:7577789494993575580:2069] VirtualId# [6:7577789490698608274:0]
2025-11-28T14:53:25.993045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789494993575580:2069] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:25.993158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:25.993202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:26.216646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789497570651636:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:26.216708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789497570651637:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:26.220739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789499595262337:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:26.220798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789499595262338:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:26.241081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789500873606110:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:26.241148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789500873606111:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:26.280785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789497263799445:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:26.280847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789497263799446:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:26.280881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789497263799447:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:26.305261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789500888776766:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:26.305331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789500888776767:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:26.305364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789500888776768:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:26.314937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789498233451766:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:26.314982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789498233451767:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:26.315012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789498233451768:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:26.704691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789500888776785:2062] [node 4] ICP01 ready to work
2025-11-28T14:53:26.723658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789498233451799:2063] [node 5] ICP01 ready to work
2025-11-28T14:53:26.764697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789500888776826:2064] [node 6] ICP01 ready to work
2025-11-28T14:53:26.740542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789499595262399:2087] [node 5] ICP01 ready to work
2025-11-28T14:53:26.776971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789499595262402:2088] [node 4] ICP01 ready to work
2025-11-28T14:53:26.777604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789499595262403:2089] [node 6] ICP01 ready to work
2025-11-28T14:53:26.772616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789500873606172:2069] [node 5] ICP01 ready to work
2025-11-28T14:53:26.792938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789497263799508:2066] [node 6] ICP01 ready to work
2025-11-28T14:53:26.836956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789500873606174:2070] [node 6] ICP01 ready to work
2025-11-28T14:53:26.796449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789497570651699:2086] [node 4] ICP01 ready to work
2025-11-28T14:53:26.848953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789500888776768:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(81352821ns) {Configure:104(81348181ns) {ProcessConfigured:135(104217ns) {StartInitialHandshake:157(100549ns) {DropHandshakes:470(3642ns) {DropIncomingHandshake:442(910ns) DropOutgoingHandshake:457(788ns)} GenerateSessionVirtualId:988(1393ns) PrepareNewSessionHandshake:477(19995ns) {SwitchToState:179(1201ns)}}}}}
2025-11-28T14:53:26.852848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789497263799509:2068] [node 5] ICP01 ready to work
2025-11-28T14:53:26.853007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789497570651700:2087] [node 5] ICP01 ready to work
2025-11-28T14:53:26.836540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789498233451830:2068] [node 4] ICP01 ready to work
2025-11-28T14:53:26.865638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789497570651701:2090] [node 6] ICP01 ready to work
2025-11-28T14:53:26.837328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789500873606175:2071] [node 4] ICP01 ready to work
2025-11-28T14:53:26.869734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789499595262402:2088] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(77621378ns) {Configure:104(77616433ns) {ProcessConfigured:135(82770ns) {StartInitialHandshake:157(79702ns) {DropHandshakes:470(2827ns) {DropIncomingHandshake:442(812ns) DropOutgoingHandshake:457(392ns)} GenerateSessionVirtualId:988(693ns) PrepareNewSessionHandshake:477(1756ns) {SwitchToState:179(1044ns)}}}}}
2025-11-28T14:53:26.889034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789500873606172:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50474334ns) {Configure:104(50468311ns) {ProcessConfigured:135(16152821ns) {StartInitialHandshake:157(16149868ns) {DropHandshakes:470(1241ns) {DropIncomingHandshake:442(270ns) DropOutgoingHandshake:457(137ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:477(3841ns) {SwitchToState:179(2151ns)}}}}}
2025-11-28T14:53:26.896579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789498233451799:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(54597720ns) {Configure:104(54591561ns) {ProcessConfigured:135(77596ns) {StartInitialHandshake:157(74540ns) {DropHandshakes:470(3079ns) {DropIncomingHandshake:442(706ns) DropOutgoingHandshake:457(516ns)} GenerateSessionVirtualId:988(1334ns) PrepareNewSessionHandshake:477(1891ns) {SwitchToState:179(842ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.907623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789500873606186:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15421
2025-11-28T14:53:26.907706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789500873606187:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15421
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.913117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789497570651637:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29740035ns) {Configure:104(29733961ns) {ProcessConfigured:135(85626ns) {StartInitialHandshake:157(82046ns) {DropHandshakes:470(3466ns) {DropIncomingHandshake:442(817ns) DropOutgoingHandshake:457(603ns)} GenerateSessionVirtualId:988(832ns) PrepareNewSessionHandshake:477(2101ns) {SwitchToState:179(984ns)}}}}}
2025-11-28T14:53:26.933197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789498233451830:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(33840680ns) {Configure:104(33833473ns) {ProcessConfigured:135(69760ns) {StartInitialHandshake:157(67151ns) {DropHandshakes:470(3954ns) {DropIncomingHandshake:442(799ns) DropOutgoingHandshake:457(847ns)} GenerateSessionVirtualId:988(1181ns) PrepareNewSessionHandshake:477(2314ns) {SwitchToState:179(933ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.938081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789498233451841:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
2025-11-28T14:53:26.919675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789497570651706:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22644
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.941572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789500873606191:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15421
2025-11-28T14:53:26.919753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789497570651707:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22644
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.953604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789500888776844:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18536
2025-11-28T14:53:26.958057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789497570651717:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22644
2025-11-28T14:53:26.953707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789500888776845:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18536
2025-11-28T14:53:26.953689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789498233451841:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.960430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789498233451768:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:26.981129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789497570651717:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.973244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789499595262417:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9971
2025-11-28T14:53:26.979663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789500888776845:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.979230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789500873606179:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.982952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789497570651709:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.984154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789499595262417:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.985693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789498233451846:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
2025-11-28T14:53:26.982979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789497570651709:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:26.984281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789499595262407:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.991116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789500888776844:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.991220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789500888776846:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18536
2025-11-28T14:53:26.991349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789500888776847:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18536
2025-11-28T14:53:26.991539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789500888776768:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:26.991733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789500888776767:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:26.979288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789500873606179:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.993748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789497263799510:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33815
2025-11-28T14:53:26.993912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789497263799511:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33815
2025-11-28T14:53:26.994052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789497263799512:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33815
2025-11-28T14:53:26.994478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789497263799523:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33815
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.982146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789500873606187:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.984302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789499595262407:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:26.992839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789498233451841:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.984247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789497570651709:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:26.982231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789500873606191:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.984951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789499595262337:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:26.998540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789497570651707:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.998639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789497570651703:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.998659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789497570651703:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:26.999070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789497570651637:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:26.999130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:26.999176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789497570651637:2049] [node 3] ICP22 created new session: [1:7577789497570651720:2049]
2025-11-28T14:53:26.999249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789497570651720:2049] [node 3] ICS09 handshake done sender: [1:7577789497570651709:2114] self: [1:7577789497570651708:0] peer: [3:7577789500873606194:0] socket: 385 qp: -1
2025-11-28T14:53:26.999286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789497570651720:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:26.999335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:27.002172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789501865619018:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22644
2025-11-28T14:53:27.002312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789497570651703:2110] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:27.002759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789497263799512:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.009753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789500888776847:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.009919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789500888776846:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.010075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789500888776766:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:26.982501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789500873606111:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:26.985295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789499595262407:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:26.986535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789499595262338:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:26.986628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:26.986681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789499595262338:2049] [node 3] ICP22 created new session: [2:7577789499595262420:2049]
2025-11-28T14:53:26.986957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789499595262420:2049] [node 3] ICS09 handshake done sender: [2:7577789499595262407:2112] self: [2:7577789499595262406:0] peer: [3:7577789500873606193:0] socket: 370 qp: -1
2025-11-28T14:53:26.986992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789499595262420:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:26.987058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:27.009561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789499595262405:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.009588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789499595262405:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:26.996082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789498233451768:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:26.982555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789500873606110:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:26.983501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789500873606179:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:27.009091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789500873606174:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.009136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:27.009192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789500873606174:2070] [node 6] ICP22 created new session: [3:7577789505168573494:2070]
2025-11-28T14:53:26.996170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:27.012712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789497570651706:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:26.996223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789498233451768:2050] [node 3] ICP22 created new session: [6:7577789498233451848:2050]
2025-11-28T14:53:27.012854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789497570651705:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.009236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789505168573494:2070] [node 6] ICS09 handshake done sender: [3:7577789500873606179:2111] self: [3:7577789500873606178:0] peer: [6:7577789498233451845:0] socket: 378 qp: -1
2025-11-28T14:53:26.996278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789498233451848:2050] [node 3] ICS09 handshake done sender: [6:7577789498233451841:2115] self: [6:7577789498233451845:0] peer: [3:7577789500873606178:0] socket: 386 qp: -1
2025-11-28T14:53:26.996312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789498233451848:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:27.012876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789497570651705:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:27.009273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789505168573494:2070] [node 6] ICS10 traffic start
2025-11-28T14:53:26.996380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:27.009319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:26.997942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789498233451847:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
2025-11-28T14:53:27.019536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789500873606177:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.019554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789500873606177:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:27.013334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789497570651636:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.023715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789505168573491:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15421
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.025088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789505168573492:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15421
2025-11-28T14:53:27.025481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789505183744148:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18536
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.025956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789505168573493:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15421
2025-11-28T14:53:27.026175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789500873606186:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.026502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789500873606191:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.026999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789505183744149:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18536
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.031389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789505183744150:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18536
2025-11-28T14:53:27.013440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:27.003660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789502528419147:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
2025-11-28T14:53:27.024163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789497263799510:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.033293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789500888776846:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.033849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789503890229719:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9971
2025-11-28T14:53:27.033999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789503890229720:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9971
2025-11-28T14:53:27.013495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789497570651636:2048] [node 2] ICP22 created new session: [1:7577789501865619022:2048]
2025-11-28T14:53:27.013544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789501865619022:2048] [node 2] ICS09 handshake done sender: [1:7577789497570651703:2110] self: [1:7577789497570651702:0] peer: [2:7577789499595262419:0] socket: 382 qp: -1
2025-11-28T14:53:27.013585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789501865619022:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:27.013624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:27.030989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789501865619025:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22644
2025-11-28T14:53:27.032647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789497570651705:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:27.024292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789497263799511:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.032435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789497263799447:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:27.035195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789500873606187:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:27.035468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789500888776766:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.035572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:27.035621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789500888776766:2048] [node 1] ICP22 created new session: [5:7577789505183744151:2048]
2025-11-28T14:53:27.035670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789505183744151:2048] [node 1] ICS09 handshake done sender: [5:7577789500888776846:2117] self: [5:7577789505183744147:0] peer: [1:7577789497570651704:0] socket: 393 qp: -1
2025-11-28T14:53:27.035702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789505183744151:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:27.035758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:27.036982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789498233451847:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.037133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789498233451830:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:27.037618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789500873606177:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.038094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789501558766826:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33815
2025-11-28T14:53:27.038632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789499595262405:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:27.039112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789500873606110:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.054437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789501865619018:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.051804Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789499595262337:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789499595262417:2116] held: no
2025-11-28T14:53:27.041170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789497263799518:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.039177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:27.053300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789503890229719:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.060314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789497570651700:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.041208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789497263799518:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:27.054369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789499595262399:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.057410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789502528419147:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.039222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789500873606110:2048] [node 1] ICP22 created new session: [3:7577789505168573498:2048]
2025-11-28T14:53:27.061094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:27.059252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789497263799523:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.060803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789497263799446:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:27.061262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789497263799518:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:27.039267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789505168573498:2048] [node 1] ICS09 handshake done sender: [3:7577789500873606191:2117] self: [3:7577789500873606194:0] peer: [1:7577789497570651708:0] socket: 389 qp: -1
2025-11-28T14:53:27.039305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789505168573498:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:27.039337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:27.041359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789505168573493:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.041483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789500873606181:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.041505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789500873606181:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:27.042002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789500873606111:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.042036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:27.042068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789500873606111:2049] [node 2] ICP22 created new session: [3:7577789505168573500:2049]
2025-11-28T14:53:27.042097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789505168573500:2049] [node 2] ICS09 handshake done sender: [3:7577789500873606187:2116] self: [3:7577789500873606193:0] peer: [2:7577789499595262406:0] socket: 376 qp: -1
2025-11-28T14:53:27.042124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789505168573500:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:27.042147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:27.043586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789500873606172:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.043639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:27.043670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789500873606172:2069] [node 5] ICP22 created new session: [3:7577789505168573503:2069]
2025-11-28T14:53:27.043702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789505168573503:2069] [node 5] ICS09 handshake done sender: [3:7577789500873606177:2110] self: [3:7577789500873606176:0] peer: [5:7577789500888776848:0] socket: 377 qp: -1
2025-11-28T14:53:27.043727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789505168573503:2069] [node 5] ICS10 traffic start
2025-11-28T14:53:27.061156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789497570651700:2087] [node 5] ICP22 created new session: [1:7577789501865619028:2087]
2025-11-28T14:53:27.043751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:27.061216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789501865619028:2087] [node 5] ICS09 handshake done sender: [1:7577789497570651705:2111] self: [1:7577789497570651704:0] peer: [5:7577789505183744147:0] socket: 383 qp: -1
2025-11-28T14:53:27.060494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789498233451766:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:27.061250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789501865619028:2087] [node 5] ICS10 traffic start
2025-11-28T14:53:27.062195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789502528419150:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
2025-11-28T14:53:27.061320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:27.055855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:27.062847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789500888776766:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16045067ns) {HandleHandshakeStatus:329(16041221ns) {DropIncomingHandshake:442(1270ns) DropOutgoingHandshake:457(6341ns) SwitchToState:179(698ns) ProcessPendingSessionEvents:495(15616742ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15549660ns max# 6453144ns}}}
2025-11-28T14:53:27.055917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789499595262399:2087] [node 5] ICP22 created new session: [2:7577789503890229722:2087]
2025-11-28T14:53:27.063038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789497263799508:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.063686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789498233451847:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:27.063621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789500873606181:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:27.055962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789503890229722:2087] [node 5] ICS09 handshake done sender: [2:7577789499595262405:2111] self: [2:7577789499595262404:0] peer: [5:7577789500888776849:0] socket: 369 qp: -1
2025-11-28T14:53:27.063103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:27.064034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789497263799508:2066] [node 6] ICP22 created new session: [4:7577789501558766829:2066]
2025-11-28T14:53:27.064760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789501558766829:2066] [node 6] ICS09 handshake done sender: [4:7577789497263799518:2115] self: [4:7577789497263799517:0] peer: [6:7577789502528419148:0] socket: 409 qp: -1
2025-11-28T14:53:27.064793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789501558766829:2066] [node 6] ICS10 traffic start
2025-11-28T14:53:27.065880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:27.065499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789500873606175:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.064696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789498233451830:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.056007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789503890229722:2087] [node 5] ICS10 traffic start
2025-11-28T14:53:27.065467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:27.065546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:27.056063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:27.059393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789499595262417:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:27.062872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789499595262337:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.063045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:27.063634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789499595262337:2048] [node 1] ICP22 created new session: [2:7577789503890229725:2048]
2025-11-28T14:53:27.063686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789503890229725:2048] [node 1] ICS09 handshake done sender: [2:7577789499595262417:2116] self: [2:7577789499595262419:0] peer: [1:7577789497570651702:0] socket: 387 qp: -1
2025-11-28T14:53:27.063722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789503890229725:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:27.063776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:27.065595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789500873606175:2071] [node 4] ICP22 created new session: [3:7577789505168573507:2071]
2025-11-28T14:53:27.065514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789498233451830:2068] [node 4] ICP22 created new session: [6:7577789502528419151:2068]
2025-11-28T14:53:27.065568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789502528419151:2068] [node 4] ICS09 handshake done sender: [6:7577789498233451847:2117] self: [6:7577789502528419148:0] peer: [4:7577789497263799517:0] socket: 410 qp: -1
2025-11-28T14:53:27.065604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789502528419151:2068] [node 4] ICS10 traffic start
2025-11-28T14:53:27.065642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.073255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789502528419153:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
2025-11-28T14:53:27.065641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789505168573507:2071] [node 4] ICS09 handshake done sender: [3:7577789500873606181:2112] self: [3:7577789500873606180:0] peer: [4:7577789501558766825:0] socket: 379 qp: -1
2025-11-28T14:53:27.065669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789505168573507:2071] [node 4] ICS10 traffic start
2025-11-28T14:53:27.065696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:27.070990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789499595262411:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.071033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789499595262411:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.076560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789501558766831:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33815
2025-11-28T14:53:27.077469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789497263799511:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:27.089652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789499595262411:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:27.092742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789497263799447:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.092807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:27.092862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789497263799447:2050] [node 3] ICP22 created new session: [4:7577789501558766833:2050]
2025-11-28T14:53:27.092909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789501558766833:2050] [node 3] ICS09 handshake done sender: [4:7577789497263799511:2111] self: [4:7577789501558766825:0] peer: [3:7577789500873606180:0] socket: 399 qp: -1
2025-11-28T14:53:27.092950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789501558766833:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:27.092986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:27.081706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789500888776768:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789500888776845:2116] held: no
2025-11-28T14:53:27.082820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789500888776845:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.088791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789497570651711:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.097755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789497263799447:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:27.088826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789497570651711:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:27.084957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789500888776844:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:27.085287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789500888776768:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.085354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.102137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789502528419154:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.097986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789501558766835:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33815
2025-11-28T14:53:27.085403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789500888776768:2050] [node 3] ICP22 created new session: [5:7577789505183744156:2050]
2025-11-28T14:53:27.085452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789505183744156:2050] [node 3] ICS09 handshake done sender: [5:7577789500888776845:2116] self: [5:7577789500888776848:0] peer: [3:7577789500873606176:0] socket: 380 qp: -1
2025-11-28T14:53:27.085492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789505183744156:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.105150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789502528419155:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
2025-11-28T14:53:27.101873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789497263799523:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:27.085545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:27.105647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789497263799446:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.105734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:27.105789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789497263799446:2049] [node 2] ICP22 created new session: [4:7577789501558766837:2049]
2025-11-28T14:53:27.105839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789501558766837:2049] [node 2] ICS09 handshake done sender: [4:7577789497263799523:2118] self: [4:7577789501558766828:0] peer: [2:7577789499595262410:0] socket: 406 qp: -1
2025-11-28T14:53:27.105873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789501558766837:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:27.105907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:27.107053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789497570651711:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:27.107345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789497570651701:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.107441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:27.107503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789497570651701:2090] [node 6] ICP22 created new session: [1:7577789501865619032:2090]
2025-11-28T14:53:27.107561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789501865619032:2090] [node 6] ICS09 handshake done sender: [1:7577789497570651711:2115] self: [1:7577789497570651710:0] peer: [6:7577789502528419149:0] socket: 414 qp: -1
2025-11-28T14:53:27.107597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789501865619032:2090] [node 6] ICS10 traffic start
2025-11-28T14:53:27.107678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:27.086541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789500888776767:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.086616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:27.086658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789500888776767:2049] [node 2] ICP22 created new session: [5:7577789505183744158:2049]
2025-11-28T14:53:27.086706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789505183744158:2049] [node 2] ICS09 handshake done sender: [5:7577789500888776844:2115] self: [5:7577789500888776849:0] peer: [2:7577789499595262404:0] socket: 374 qp: -1
2025-11-28T14:53:27.086734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789505183744158:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:27.086767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:27.102719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789505183744153:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18536
2025-11-28T14:53:27.111490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789502528419153:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.109398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789499595262402:2088] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.109492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:27.111595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789502528419147:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:27.113612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789498233451766:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.113688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:27.113735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789498233451766:2048] [node 1] ICP22 created new session: [6:7577789502528419156:2048]
2025-11-28T14:53:27.113783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789502528419156:2048] [node 1] ICS09 handshake done sender: [6:7577789502528419147:2118] self: [6:7577789502528419149:0] peer: [1:7577789497570651710:0] socket: 416 qp: -1
2025-11-28T14:53:27.113822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789502528419156:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:27.113856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:27.114545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789498233451766:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:27.115732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789502528419154:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.115809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789498233451767:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:27.116409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789498233451799:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:27.122063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789500888776838:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.109539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789499595262402:2088] [node 4] ICP22 created new session: [2:7577789503890229729:2088]
2025-11-28T14:53:27.109592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789503890229729:2088] [node 4] ICS09 handshake done sender: [2:7577789499595262411:2114] self: [2:7577789499595262410:0] peer: [4:7577789501558766828:0] socket: 405 qp: -1
2025-11-28T14:53:27.109623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789503890229729:2088] [node 4] ICS10 traffic start
2025-11-28T14:53:27.109657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:27.122101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789500888776838:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:27.125984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789501558766826:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.126809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789497263799445:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:27.128346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789505183744153:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.128694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789500888776785:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.130799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789502528419160:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
2025-11-28T14:53:27.131151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789499595262413:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.131203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789499595262413:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:27.131775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789500888776838:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:27.132148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789500888776826:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.132237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:27.132286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789500888776826:2064] [node 6] ICP22 created new session: [5:7577789505183744162:2064]
2025-11-28T14:53:27.132334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789505183744162:2064] [node 6] ICS09 handshake done sender: [5:7577789500888776838:2113] self: [5:7577789500888776837:0] peer: [6:7577789502528419159:0] socket: 407 qp: -1
2025-11-28T14:53:27.132386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789505183744162:2064] [node 6] ICS10 traffic start
2025-11-28T14:53:27.132434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:27.134081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789497570651713:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.135554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789497263799520:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:27.135586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789497263799520:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.138133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789505183744164:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18536
2025-11-28T14:53:27.136789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789501558766840:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33815
2025-11-28T14:53:27.136941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789497263799520:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:27.134126Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789497570651713:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:27.140095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789497263799509:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.137198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789497570651713:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:27.140180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:27.140701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789505183744153:2122] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:27.140230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789497263799509:2068] [node 5] ICP22 created new session: [4:7577789501558766842:2068]
2025-11-28T14:53:27.140271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789501558766842:2068] [node 5] ICS09 handshake done sender: [4:7577789497263799520:2116] self: [4:7577789497263799519:0] peer: [5:7577789505183744161:0] socket: 367 qp: -1
2025-11-28T14:53:27.142236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789500888776785:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.140310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789501558766842:2068] [node 5] ICS10 traffic start
2025-11-28T14:53:27.140347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:27.144636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789497570651699:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.144736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:27.144783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789497570651699:2086] [node 4] ICP22 created new session: [1:7577789501865619035:2086]
2025-11-28T14:53:27.144825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789501865619035:2086] [node 4] ICS09 handshake done sender: [1:7577789497570651713:2116] self: [1:7577789497570651712:0] peer: [4:7577789501558766839:0] socket: 428 qp: -1
2025-11-28T14:53:27.141894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789502528419154:2121] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:27.142139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789498233451799:2063] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26696) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789502528419154:2121] held: no
2025-11-28T14:53:27.143174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789498233451799:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.143235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:27.143282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789498233451799:2063] [node 5] ICP22 created new session: [6:7577789502528419161:2063]
2025-11-28T14:53:27.143327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789502528419161:2063] [node 5] ICS09 handshake done sender: [6:7577789502528419154:2121] self: [6:7577789502528419159:0] peer: [5:7577789500888776837:0] socket: 432 qp: -1
2025-11-28T14:53:27.143362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789502528419161:2063] [node 5] ICS10 traffic start
2025-11-28T14:53:27.143396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.147307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789501865619035:2086] [node 4] ICS10 traffic start
2025-11-28T14:53:27.147396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:27.147315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789503890229731:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9971
2025-11-28T14:53:27.148577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:27.148971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789500888776785:2062] [node 4] ICP22 created new session: [5:7577789505183744165:2062]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:27.150804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789503890229732:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9971
2025-11-28T14:53:27.150814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789502528419163:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3677
2025-11-28T14:53:27.151015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789505183744165:2062] [node 4] ICS09 handshake done sender: [5:7577789505183744153:2122] self: [5:7577789505183744161:0] peer: [4:7577789497263799519:0] socket: 427 qp: -1
2025-11-28T14:53:27.151070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789505183744165:2062] [node 4] ICS10 traffic start
2025-11-28T14:53:27.151108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:27.153937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789501558766826:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:27.156640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789499595262413:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:27.158746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789499595262403:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:27.157475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789497263799445:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.157552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:27.157594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789497263799445:2048] [node 1] ICP22 created new session: [4:7577789501558766844:2048]
2025-11-28T14:53:27.157648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789501558766844:2048] [node 1] ICS09 handshake done sender: [4:7577789501558766826:2119] self: [4:7577789501558766839:0] peer: [1:7577789497570651712:0] socket: 371 qp: -1
2025-11-28T14:53:27.157684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789501558766844:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:27.157721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:27.158228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789497263799445:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:27.158852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:27.158891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789499595262403:2089] [node 6] ICP22 created new session: [2:7577789503890229734:2089]
2025-11-28T14:53:27.158931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789503890229734:2089] [node 6] ICS09 handshake done sender: [2:7577789499595262413:2115] self: [2:7577789499595262412:0] peer: [6:7577789502528419158:0] socket: 366 qp: -1
2025-11-28T14:53:27.158960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789503890229734:2089] [node 6] ICS10 traffic start
2025-11-28T14:53:27.159043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:27.190278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789502528419153:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:27.190601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789498233451767:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62246) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789502528419153:2120] held: no
2025-11-28T14:53:27.191199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789498233451767:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:27.191258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:27.191317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789498233451767:2049] [node 2] ICP22 created new session: [6:7577789502528419164:2049]
2025-11-28T14:53:27.191364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789502528419164:2049] [node 2] ICS09 handshake done sender: [6:7577789502528419153:2120] self: [6:7577789502528419158:0] peer: [2:7577789499595262412:0] socket: 392 qp: -1
2025-11-28T14:53:27.191402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789502528419164:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:27.191461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:27.224045Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:27.224264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789503890229725:2048] [node 1] ICS01 socket: 387 reason# EndOfStream
2025-11-28T14:53:27.224307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789499595262337:2048] [node 1] ICP30 unregister session Session# [2:7577789503890229725:2048] VirtualId# [2:7577789499595262419:0]
2025-11-28T14:53:27.224343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789503890229725:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.224438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:27.224477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:27.226278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:27.226505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789505168573498:2048] [node 1] ICS07 socket disconnect 389 reason# EndOfStream
2025-11-28T14:53:27.226541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789505168573498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.226624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:27.226668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789505168573498:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:27.227600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789500873606110:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:27.227657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789505168573498:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:27.227691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789500873606110:2048] [node 1] ICP30 unregister session Session# [3:7577789505168573498:2048] VirtualId# [3:7577789500873606194:0]
2025-11-28T14:53:27.227733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:27.227775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789500873606110:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.227795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.232559Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:27.232556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:27.232801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789502528419156:2048] [node 1] ICS01 socket: 416 reason# EndOfStream
2025-11-28T14:53:27.232844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789498233451766:2048] [node 1] ICP30 unregister session Session# [6:7577789502528419156:2048] VirtualId# [6:7577789502528419149:0]
2025-11-28T14:53:27.232879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789502528419156:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.232958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:27.233008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:27.233275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789501558766844:2048] [node 1] ICS07 socket disconnect 371 reason# EndOfStream
2025-11-28T14:53:27.233310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789501558766844:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.233359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:27.233414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789501558766844:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:27.232636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:27.232843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789505183744151:2048] [node 1] ICS07 socket disconnect 393 reason# EndOfStream
2025-11-28T14:53:27.232873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789505183744151:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.232959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:27.232998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789505183744151:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:27.237717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789500873606110:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:27.261456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789500888776766:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:27.261543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789505183744151:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:27.261586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789500888776766:2048] [node 1] ICP30 unregister session Session# [5:7577789505183744151:2048] VirtualId# [5:7577789505183744147:0]
2025-11-28T14:53:27.261728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:27.261767Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789500888776766:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.261783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.264907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789497263799445:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:27.264974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789501558766844:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:27.265008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789497263799445:2048] [node 1] ICP30 unregister session Session# [4:7577789501558766844:2048] VirtualId# [4:7577789501558766839:0]
2025-11-28T14:53:27.265063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:27.265092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789497263799445:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.265108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.276498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789497263799445:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:27.280500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789500888776766:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:27.298436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:27.298732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789505168573500:2049] [node 2] ICS07 socket disconnect 376 reason# EndOfStream
2025-11-28T14:53:27.298766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789505168573500:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.298883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:27.298939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789505168573500:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:27.306168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789500873606111:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62246) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:27.306231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789505168573500:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:27.306270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789500873606111:2049] [node 2] ICP30 unregister session Session# [3:7577789505168573500:2049] VirtualId# [3:7577789500873606193:0]
2025-11-28T14:53:27.306310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:27.306344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789500873606111:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62246) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.306366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62246) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.305250Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:27.305477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789502528419164:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-11-28T14:53:27.305515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789498233451767:2049] [node 2] ICP30 unregister session Session# [6:7577789502528419164:2049] VirtualId# [6:7577789502528419158:0]
2025-11-28T14:53:27.305551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789502528419164:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.305631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:27.305677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:27.308545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:27.308856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789501558766837:2049] [node 2] ICS01 socket: 406 reason# EndOfStream
2025-11-28T14:53:27.308899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789497263799446:2049] [node 2] ICP30 unregister session Session# [4:7577789501558766837:2049] VirtualId# [4:7577789501558766828:0]
2025-11-28T14:53:27.308934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789501558766837:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.309030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:27.309079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:27.308480Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:27.308705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789505183744158:2049] [node 2] ICS07 socket disconnect 374 reason# EndOfStream
2025-11-28T14:53:27.308737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789505183744158:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.308868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:27.308920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789505183744158:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:27.316654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789500873606111:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:27.316968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789500888776767:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62246) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:27.317035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789505183744158:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:27.317070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789500888776767:2049] [node 2] ICP30 unregister session Session# [5:7577789505183744158:2049] VirtualId# [5:7577789500888776849:0]
2025-11-28T14:53:27.317175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:27.317239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789500888776767:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62246) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.317254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62246) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.330078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789500888776767:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:27.376605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:27.376850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789505183744156:2050] [node 3] ICS01 socket: 380 reason# EndOfStream
2025-11-28T14:53:27.376898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789500888776768:2050] [node 3] ICP30 unregister session Session# [5:7577789505183744156:2050] VirtualId# [5:7577789500888776848:0]
2025-11-28T14:53:27.376932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789505183744156:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.377017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:27.377059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:27.376605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:27.376849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789498233451848:2050] [node 3] ICS07 socket disconnect 386 reason# EndOfStream
2025-11-28T14:53:27.376883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789498233451848:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.376968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:27.377007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789498233451848:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:27.376839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:27.377069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789501558766833:2050] [node 3] ICS07 socket disconnect 399 reason# EndOfStream
2025-11-28T14:53:27.377101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789501558766833:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.377232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:27.377292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789501558766833:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:27.387453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789498233451768:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15676) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:27.387528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789498233451848:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:27.387566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789498233451768:2050] [node 3] ICP30 unregister session Session# [6:7577789498233451848:2050] VirtualId# [6:7577789498233451845:0]
2025-11-28T14:53:27.387618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:27.387651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789498233451768:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15676) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.387668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15676) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:27.396476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789498233451768:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:27.416106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:27.416348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789502528419151:2068] [node 4] ICS01 socket: 410 reason# EndOfStream
2025-11-28T14:53:27.416405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789498233451830:2068] [node 4] ICP30 unregister session Session# [6:7577789502528419151:2068] VirtualId# [6:7577789502528419148:0]
2025-11-28T14:53:27.416443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789502528419151:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.416552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:27.416609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:27.416708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:27.416896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789505183744165:2062] [node 4] ICS01 socket: 427 reason# EndOfStream
2025-11-28T14:53:27.416936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789500888776785:2062] [node 4] ICP30 unregister session Session# [5:7577789505183744165:2062] VirtualId# [5:7577789505183744161:0]
2025-11-28T14:53:27.416966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789505183744165:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.417018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:27.417054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:27.484581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:27.484832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789502528419161:2063] [node 5] ICS01 socket: 432 reason# EndOfStream
2025-11-28T14:53:27.484876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789498233451799:2063] [node 5] ICP30 unregister session Session# [6:7577789502528419161:2063] VirtualId# [6:7577789502528419159:0]
2025-11-28T14:53:27.484914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789502528419161:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:27.485026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:27.485073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:27.764738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789502970188029:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:27.764791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789502970188030:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:27.816642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789504683466555:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:27.816686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789504683466556:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:27.889927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789502156727155:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:27.889981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789502156727156:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:27.955774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789504572029810:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:27.955827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789504572029811:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:27.955861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789504572029812:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:28.004632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789503672262543:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:28.004688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789503672262544:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:28.004722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789503672262545:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:28.025585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789508995307210:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:28.025638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789508995307211:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:28.025684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789508995307212:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:28.385463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789508866997133:2073] [node 6] ICP01 ready to work
2025-11-28T14:53:28.498650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789508995307244:2072] [node 5] ICP01 ready to work
2025-11-28T14:53:28.558640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789506451694514:2097] [node 4] ICP01 ready to work
2025-11-28T14:53:28.561752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789508995307210:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20548445ns) {Configure:104(20542407ns) {ProcessConfigured:135(83260ns) {StartInitialHandshake:157(80419ns) {DropHandshakes:470(3296ns) {DropIncomingHandshake:442(998ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(1410ns) PrepareNewSessionHandshake:477(1460ns) {SwitchToState:179(633ns)}}}}}
2025-11-28T14:53:28.514211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789508978433914:2102] [node 6] ICP01 ready to work
2025-11-28T14:53:28.592754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789508995307278:2074] [node 4] ICP01 ready to work
2025-11-28T14:53:28.577207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789507967229902:2065] [node 4] ICP01 ready to work
2025-11-28T14:53:28.577620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789508978433915:2103] [node 4] ICP01 ready to work
2025-11-28T14:53:28.580864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789506451694515:2099] [node 5] ICP01 ready to work
2025-11-28T14:53:28.613657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789502970188030:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(237580654ns) {Configure:104(237574573ns) {ProcessConfigured:135(52279ns) {StartInitialHandshake:157(50213ns) {DropHandshakes:470(2970ns) {DropIncomingHandshake:442(659ns) DropOutgoingHandshake:457(570ns)} GenerateSessionVirtualId:988(1121ns) PrepareNewSessionHandshake:477(1483ns) {SwitchToState:179(769ns)}}}}}
2025-11-28T14:53:28.614118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789507265155390:2105] [node 6] ICP01 ready to work
2025-11-28T14:53:28.614710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789507265155391:2106] [node 4] ICP01 ready to work
2025-11-28T14:53:28.615237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789507265155392:2107] [node 5] ICP01 ready to work
2025-11-28T14:53:28.564657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789504572029810:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(178868748ns) {Configure:104(178864489ns) {ProcessConfigured:135(15549721ns) {StartInitialHandshake:157(15546207ns) {DropHandshakes:470(2730ns) {DropIncomingHandshake:442(1138ns) DropOutgoingHandshake:457(339ns)} GenerateSessionVirtualId:988(902ns) PrepareNewSessionHandshake:477(3055ns) {SwitchToState:179(301ns)}}}}}
2025-11-28T14:53:28.612770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789507967229903:2063] [node 6] ICP01 ready to work
2025-11-28T14:53:28.600748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789508978433916:2104] [node 5] ICP01 ready to work
2025-11-28T14:53:28.565477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789508866997172:2084] [node 5] ICP01 ready to work
2025-11-28T14:53:28.665943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789507265155390:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(49652428ns) {Configure:104(49647850ns) {ProcessConfigured:135(65905ns) {StartInitialHandshake:157(63185ns) {DropHandshakes:470(3014ns) {DropIncomingHandshake:442(834ns) DropOutgoingHandshake:457(436ns)} GenerateSessionVirtualId:988(893ns) PrepareNewSessionHandshake:477(1559ns) {SwitchToState:179(904ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.688353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789508978433919:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58192
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.713189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789508978433919:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.716964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789508978433915:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55968337ns) {Configure:104(55963046ns) {ProcessConfigured:135(7251439ns) {StartInitialHandshake:157(7247703ns) {DropHandshakes:470(2717ns) {DropIncomingHandshake:442(558ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(1124ns) PrepareNewSessionHandshake:477(2096ns) {SwitchToState:179(592ns)}}}}}
2025-11-28T14:53:28.676773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789506451694527:2102] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.709386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789508866997133:2073] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(282633757ns) {Configure:104(282619077ns) {ProcessConfigured:135(49538218ns) {StartInitialHandshake:157(49535810ns) {DropHandshakes:470(2566ns) {DropIncomingHandshake:442(647ns) DropOutgoingHandshake:457(504ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:477(3792ns) {SwitchToState:179(2022ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.727397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789507265155392:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(59534593ns) {Configure:104(59526891ns) {ProcessConfigured:135(51610ns) {StartInitialHandshake:157(49805ns) {DropHandshakes:470(2803ns) {DropIncomingHandshake:442(719ns) DropOutgoingHandshake:457(463ns)} GenerateSessionVirtualId:988(720ns) PrepareNewSessionHandshake:477(1493ns) {SwitchToState:179(772ns)}}}}}
2025-11-28T14:53:28.728064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789507265155393:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10535
2025-11-28T14:53:28.728881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789507265155402:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10535
2025-11-28T14:53:28.728942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789507265155403:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10535
2025-11-28T14:53:28.698259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789506451694530:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46896
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.722069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789508978433930:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58192
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.743726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789502970188029:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(341547785ns) {Configure:104(341543471ns) {ProcessConfigured:135(49531897ns) {StartInitialHandshake:157(49530233ns) {DropHandshakes:470(1389ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(406ns) PrepareNewSessionHandshake:477(4076ns) {SwitchToState:179(1918ns)}}}}}
2025-11-28T14:53:28.744309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789507265155407:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10535
2025-11-28T14:53:28.744799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789508978433933:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58192
2025-11-28T14:53:28.745279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155393:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.752942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155403:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.734939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789504572029812:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(158155302ns) {Configure:104(158150067ns) {ProcessConfigured:135(7849510ns) {StartInitialHandshake:157(7845727ns) {DropHandshakes:470(3326ns) {DropIncomingHandshake:442(669ns) DropOutgoingHandshake:457(584ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:477(4279ns) {SwitchToState:179(2347ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.758921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789503672262545:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(73737100ns) {Configure:104(73728565ns) {ProcessConfigured:135(55749ns) {StartInitialHandshake:157(53103ns) {DropHandshakes:470(2718ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(444ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:477(1567ns) {SwitchToState:179(671ns)}}}}}
2025-11-28T14:53:28.762031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789507967229904:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63987
2025-11-28T14:53:28.762095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789507967229905:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63987
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.766535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789507265155410:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10535
2025-11-28T14:53:28.736397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789508866997179:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11290
2025-11-28T14:53:28.777804Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789503672262544:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26277649ns) {Configure:104(26272737ns) {ProcessConfigured:135(97699ns) {StartInitialHandshake:157(94420ns) {DropHandshakes:470(2959ns) {DropIncomingHandshake:442(575ns) DropOutgoingHandshake:457(549ns)} GenerateSessionVirtualId:988(1109ns) PrepareNewSessionHandshake:477(1776ns) {SwitchToState:179(750ns)}}}}}
2025-11-28T14:53:28.778306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789507967229905:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.778666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789503672262544:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.764083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789506451694530:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.805843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789507967229920:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63987
2025-11-28T14:53:28.772543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789508978433918:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.736569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789508866997180:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11290
2025-11-28T14:53:28.821082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789507967229905:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:28.786814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155407:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.821774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789503672262544:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:28.772604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789508978433918:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:28.764269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789502156727156:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.738983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789508866997172:2084] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29316943ns) {Configure:104(29309155ns) {ProcessConfigured:135(71158ns) {StartInitialHandshake:157(68027ns) {DropHandshakes:470(2972ns) {DropIncomingHandshake:442(512ns) DropOutgoingHandshake:457(421ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:477(1269ns) {SwitchToState:179(469ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.861552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789508995307278:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(96457327ns) {Configure:104(96452423ns) {ProcessConfigured:135(18281875ns) {StartInitialHandshake:157(18277867ns) {DropHandshakes:470(3250ns) {DropIncomingHandshake:442(532ns) DropOutgoingHandshake:457(505ns)} GenerateSessionVirtualId:988(1162ns) PrepareNewSessionHandshake:477(4373ns) {SwitchToState:179(2864ns)}}}}}
2025-11-28T14:53:28.861979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789508995307286:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
2025-11-28T14:53:28.862147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789508995307287:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
2025-11-28T14:53:28.862935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789508995307288:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
2025-11-28T14:53:28.863062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789508995307291:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.779025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789508978433923:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.779054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789508978433923:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:28.780106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789508978433923:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:28.784750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789508978433918:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:28.828139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789504683466556:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:28.828224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:28.828283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789504683466556:2049] [node 3] ICP22 created new session: [2:7577789508978433936:2049]
2025-11-28T14:53:28.828335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789508978433936:2049] [node 3] ICS09 handshake done sender: [2:7577789508978433918:2110] self: [2:7577789508978433917:0] peer: [3:7577789506451694532:0] socket: 380 qp: -1
2025-11-28T14:53:28.828394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789508978433936:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:28.828435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:28.829630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789508978433916:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:28.829667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:28.829722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789508978433916:2104] [node 5] ICP22 created new session: [2:7577789508978433938:2104]
2025-11-28T14:53:28.829782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789508978433938:2104] [node 5] ICS09 handshake done sender: [2:7577789508978433923:2113] self: [2:7577789508978433922:0] peer: [5:7577789507967229919:0] socket: 382 qp: -1
2025-11-28T14:53:28.829828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789508978433938:2104] [node 5] ICS10 traffic start
2025-11-28T14:53:28.829868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:28.844829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789508978433933:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.821848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:28.783639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789508866997179:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.783264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789506451694531:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46896
2025-11-28T14:53:28.880654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789508978433930:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.885167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789508995307287:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.885330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789508995307291:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.885541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789508995307278:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:28.885628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789508995307211:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:28.864777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155410:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.784050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789504572029812:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:28.816926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789508866997180:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.840547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789504572029810:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:28.869217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789508866997188:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11290
2025-11-28T14:53:28.873421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789508866997189:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11290
2025-11-28T14:53:28.888691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789508866997189:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.821925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789503672262544:2049] [node 2] ICP22 created new session: [5:7577789507967229921:2049]
2025-11-28T14:53:28.893022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789508995307286:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.896551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789508995307288:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.801677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789506451694533:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46896
2025-11-28T14:53:28.908417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789508978433925:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.908515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789508978433925:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:28.874117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155397:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.839974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789506451694521:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.874164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789507265155397:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:28.821981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789507967229921:2049] [node 2] ICS09 handshake done sender: [5:7577789507967229905:2111] self: [5:7577789507967229919:0] peer: [2:7577789508978433922:0] socket: 403 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.924666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789508978433932:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.909064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789508995307210:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:28.926288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789508995307244:2072] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:28.912424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789508866997176:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.840029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789506451694521:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:28.852975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789506451694534:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46896
2025-11-28T14:53:28.853273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789506451694531:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.854548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789506451694530:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:28.854836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789502156727156:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20707) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789506451694530:2115] held: no
2025-11-28T14:53:28.855520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789502156727156:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:28.855586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:28.855664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789502156727156:2049] [node 2] ICP22 created new session: [3:7577789506451694535:2049]
2025-11-28T14:53:28.855771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789506451694535:2049] [node 2] ICS09 handshake done sender: [3:7577789506451694530:2115] self: [3:7577789506451694532:0] peer: [2:7577789508978433917:0] socket: 383 qp: -1
2025-11-28T14:53:28.855812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789506451694535:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:28.855852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:28.928755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789506451694534:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.928998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789502156727155:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:28.937307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789506451694521:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:28.912517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789508866997176:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.952839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789508995307244:2072] [node 5] ICP35 event processing took too much time PendingConnection:279(17637036ns) {IncomingHandshake:261(17634802ns) {DropIncomingHandshake:442(527ns) PrepareNewSessionHandshake:477(2153ns) {SwitchToState:179(880ns)} GenerateSessionVirtualId:988(826ns) IssueIncomingHandshakeReply:193(17601741ns) {SwitchToState:179(2458ns)}}}
2025-11-28T14:53:28.959048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789508995307211:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20707) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789508995307291:2118] held: no
2025-11-28T14:53:28.924717Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789508978433932:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:28.822023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789507967229921:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.890178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155402:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.937375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789508978433925:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:28.920729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789508866997188:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.957586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155389:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.822081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.940636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789508978433932:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:28.976608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789506451694514:2097] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:28.976691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:28.976767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789506451694514:2097] [node 4] ICP22 created new session: [3:7577789506451694541:2097]
2025-11-28T14:53:28.976828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789506451694541:2097] [node 4] ICS09 handshake done sender: [3:7577789506451694521:2112] self: [3:7577789506451694520:0] peer: [4:7577789508866997186:0] socket: 374 qp: -1
2025-11-28T14:53:28.976880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789506451694541:2097] [node 4] ICS10 traffic start
2025-11-28T14:53:28.976920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:28.981445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789506451694543:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46896
2025-11-28T14:53:28.920861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789504572029811:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:28.989446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789508995307297:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
2025-11-28T14:53:28.989720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789508995307298:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
2025-11-28T14:53:28.989824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789508995307296:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
2025-11-28T14:53:28.990305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789508995307291:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:28.992557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789508995307211:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:28.992604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:28.992659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789508995307211:2049] [node 2] ICP22 created new session: [6:7577789508995307300:2049]
2025-11-28T14:53:28.992711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789508995307300:2049] [node 2] ICS09 handshake done sender: [6:7577789508995307291:2118] self: [6:7577789508995307293:0] peer: [2:7577789508978433924:0] socket: 377 qp: -1
2025-11-28T14:53:28.992749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789508995307300:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:28.992787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:28.993988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789508995307288:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:28.994633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789508995307244:2072] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:28.994683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:28.994738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789508995307244:2072] [node 5] ICP22 created new session: [6:7577789508995307302:2072]
2025-11-28T14:53:28.994791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789508995307302:2072] [node 5] ICS09 handshake done sender: [6:7577789508995307288:2116] self: [6:7577789508995307294:0] peer: [5:7577789507967229908:0] socket: 417 qp: -1
2025-11-28T14:53:28.994822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789508995307302:2072] [node 5] ICS10 traffic start
2025-11-28T14:53:28.994860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:28.869571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789507967229904:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.957634Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789507265155389:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:28.936889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789508866997192:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11290
2025-11-28T14:53:28.966861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155395:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.966647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789507967229909:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.966695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789507967229909:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:28.970002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789507967229925:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63987
2025-11-28T14:53:28.980240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789507967229909:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:28.982404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789507967229925:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:28.982535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789503672262545:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:28.984568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789507967229903:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:28.991232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:28.996617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789507967229903:2063] [node 6] ICP22 created new session: [5:7577789507967229928:2063]
2025-11-28T14:53:29.025034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789507967229928:2063] [node 6] ICS09 handshake done sender: [5:7577789507967229909:2113] self: [5:7577789507967229908:0] peer: [6:7577789508995307294:0] socket: 410 qp: -1
2025-11-28T14:53:28.954856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789508978433914:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.017052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789506451694523:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.953395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789508866997193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11290
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.954932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:28.954994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789508978433914:2102] [node 6] ICP22 created new session: [2:7577789508978433946:2102]
2025-11-28T14:53:28.955048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789508978433946:2102] [node 6] ICS09 handshake done sender: [2:7577789508978433925:2114] self: [2:7577789508978433924:0] peer: [6:7577789508995307293:0] socket: 419 qp: -1
2025-11-28T14:53:28.955088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789508978433946:2102] [node 6] ICS10 traffic start
2025-11-28T14:53:28.955126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:28.956808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789508978433915:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:28.956886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:28.956932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789508978433915:2103] [node 4] ICP22 created new session: [2:7577789508978433948:2103]
2025-11-28T14:53:28.956986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789508978433948:2103] [node 4] ICS09 handshake done sender: [2:7577789508978433932:2116] self: [2:7577789508978433929:0] peer: [4:7577789508866997190:0] socket: 420 qp: -1
2025-11-28T14:53:28.957021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789508978433948:2103] [node 4] ICS10 traffic start
2025-11-28T14:53:28.957056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:28.977345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789508866997179:2114] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:29.017099Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789506451694523:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:29.000667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789508866997189:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:29.019184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789506451694531:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:29.000986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789504572029811:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20707) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789508866997189:2118] held: no
2025-11-28T14:53:28.966896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789507265155395:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:29.001690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789504572029811:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.001771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:28.977546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789507265155397:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:29.001832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789504572029811:2049] [node 2] ICP22 created new session: [4:7577789508866997194:2049]
2025-11-28T14:53:28.982250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789507265155389:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:29.001887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789508866997194:2049] [node 2] ICS09 handshake done sender: [4:7577789508866997189:2118] self: [4:7577789508866997190:0] peer: [2:7577789508978433929:0] socket: 372 qp: -1
2025-11-28T14:53:29.001923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789508866997194:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:29.001967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:29.003441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789504572029812:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.003525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:29.003573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789504572029812:2050] [node 3] ICP22 created new session: [4:7577789508866997198:2050]
2025-11-28T14:53:29.003650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789508866997198:2050] [node 3] ICS09 handshake done sender: [4:7577789508866997179:2114] self: [4:7577789508866997186:0] peer: [3:7577789506451694520:0] socket: 394 qp: -1
2025-11-28T14:53:29.003680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789508866997198:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:29.003712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:29.013171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789508866997196:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11290
2025-11-28T14:53:29.020697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789508866997176:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:29.046692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789502156727155:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.041370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789508866997180:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:29.033927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789507967229928:2063] [node 6] ICS10 traffic start
2025-11-28T14:53:29.056647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789513290274601:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
2025-11-28T14:53:29.065106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789507265155391:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.065192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:29.065254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789507265155391:2106] [node 4] ICP22 created new session: [1:7577789511560122713:2106]
2025-11-28T14:53:29.065307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789511560122713:2106] [node 4] ICS09 handshake done sender: [1:7577789507265155397:2113] self: [1:7577789507265155396:0] peer: [4:7577789508866997187:0] socket: 391 qp: -1
2025-11-28T14:53:29.065350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789511560122713:2106] [node 4] ICS10 traffic start
2025-11-28T14:53:29.065390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:29.067268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789502970188030:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.067326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:29.067373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789502970188030:2049] [node 3] ICP22 created new session: [1:7577789511560122715:2049]
2025-11-28T14:53:29.067420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789511560122715:2049] [node 3] ICS09 handshake done sender: [1:7577789507265155389:2110] self: [1:7577789507265155388:0] peer: [3:7577789506451694538:0] socket: 389 qp: -1
2025-11-28T14:53:29.067460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789511560122715:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:29.067495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:29.052321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789512262197225:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63987
2025-11-28T14:53:29.046746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:29.048735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789504572029810:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:29.046799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789502156727155:2048] [node 1] ICP22 created new session: [3:7577789510746661843:2048]
2025-11-28T14:53:29.048833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:29.096089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789508995307298:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:29.104906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789507265155395:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:29.052562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:29.096390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789513290274602:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
2025-11-28T14:53:29.096484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789508995307212:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:29.101587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789508995307287:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:29.102701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789508995307278:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.109001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:29.109087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789508995307278:2074] [node 4] ICP22 created new session: [6:7577789513290274604:2074]
2025-11-28T14:53:29.109159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789513290274604:2074] [node 4] ICS09 handshake done sender: [6:7577789508995307287:2115] self: [6:7577789508995307292:0] peer: [4:7577789508866997175:0] socket: 407 qp: -1
2025-11-28T14:53:29.109201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789513290274604:2074] [node 4] ICS10 traffic start
2025-11-28T14:53:29.109244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:29.048889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789504572029810:2048] [node 1] ICP22 created new session: [4:7577789513161964499:2048]
2025-11-28T14:53:29.090558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789507967229925:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:29.046871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789510746661843:2048] [node 1] ICS09 handshake done sender: [3:7577789506451694531:2116] self: [3:7577789506451694538:0] peer: [1:7577789507265155388:0] socket: 392 qp: -1
2025-11-28T14:53:29.046910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789510746661843:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:29.046951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:29.055765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789510746661841:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46896
2025-11-28T14:53:29.056068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789506451694523:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:29.085397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789510746661841:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:29.085451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789506451694515:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.085538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:29.085626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789506451694515:2099] [node 5] ICP22 created new session: [3:7577789510746661845:2099]
2025-11-28T14:53:29.085685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789510746661845:2099] [node 5] ICS09 handshake done sender: [3:7577789506451694523:2113] self: [3:7577789506451694522:0] peer: [5:7577789507967229927:0] socket: 424 qp: -1
2025-11-28T14:53:29.085722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789510746661845:2099] [node 5] ICS10 traffic start
2025-11-28T14:53:29.085759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:29.124022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789506451694529:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:29.124071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789506451694529:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:29.134085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789508995307286:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:29.134943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789507265155390:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.135023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:29.135071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789507265155390:2105] [node 6] ICP22 created new session: [1:7577789511560122719:2105]
2025-11-28T14:53:29.135134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789511560122719:2105] [node 6] ICS09 handshake done sender: [1:7577789507265155395:2112] self: [1:7577789507265155394:0] peer: [6:7577789508995307295:0] socket: 390 qp: -1
2025-11-28T14:53:29.135176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789511560122719:2105] [node 6] ICS10 traffic start
2025-11-28T14:53:29.135212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:29.048939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789513161964499:2048] [node 1] ICS09 handshake done sender: [4:7577789508866997180:2115] self: [4:7577789508866997187:0] peer: [1:7577789507265155396:0] socket: 395 qp: -1
2025-11-28T14:53:29.129111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789512262197227:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63987
2025-11-28T14:53:29.137150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789513273401246:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58192
2025-11-28T14:53:29.139546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789507967229903:2063] [node 6] ICP35 event processing took too much time PendingConnection:279(154046095ns) {HandleHandshakeStatus:329(154042676ns) {DropIncomingHandshake:442(25210ns) DropOutgoingHandshake:457(399ns) SwitchToState:179(635ns) ProcessPendingSessionEvents:495(57889170ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 57787078ns max# 17118035ns}}}
2025-11-28T14:53:29.139623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789503672262545:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.139680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:29.139745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789503672262545:2050] [node 3] ICP22 created new session: [5:7577789512262197228:2050]
2025-11-28T14:53:29.139798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789512262197228:2050] [node 3] ICS09 handshake done sender: [5:7577789507967229925:2118] self: [5:7577789507967229927:0] peer: [3:7577789506451694522:0] socket: 427 qp: -1
2025-11-28T14:53:29.139830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789512262197228:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:29.139864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:29.141222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789503672262545:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:29.142509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789512262197227:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:29.048973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789513161964499:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:29.146430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789507967229902:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:29.140603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789508995307210:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.161755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:29.161864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789508995307210:2048] [node 1] ICP22 created new session: [6:7577789513290274606:2048]
2025-11-28T14:53:29.161932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789513290274606:2048] [node 1] ICS09 handshake done sender: [6:7577789508995307286:2114] self: [6:7577789508995307295:0] peer: [1:7577789507265155394:0] socket: 404 qp: -1
2025-11-28T14:53:29.161976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789513290274606:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:29.162037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:29.049022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:29.146554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789512262197230:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63987
2025-11-28T14:53:29.168768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789512262197230:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:29.154280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789506451694529:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:29.174256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155399:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:29.174297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789507265155399:2114] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:29.177038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789512262197232:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63987
2025-11-28T14:53:29.050121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789508866997133:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.165661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789513273401246:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:29.178148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789508995307210:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22203598ns) {HandleHandshakeStatus:329(22200464ns) {DropIncomingHandshake:442(5832ns) DropOutgoingHandshake:457(10629ns) SwitchToState:179(727ns) ProcessPendingSessionEvents:495(609284ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 546814ns max# 58382ns}}}
2025-11-28T14:53:29.180931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789507265155399:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:29.180469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789503672262543:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:29.181488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789507265155401:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:29.181525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789507265155401:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:29.182117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789502970188029:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.182194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:29.182249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789502970188029:2048] [node 2] ICP22 created new session: [1:7577789511560122722:2048]
2025-11-28T14:53:29.182302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789511560122722:2048] [node 2] ICS09 handshake done sender: [1:7577789507265155399:2114] self: [1:7577789507265155398:0] peer: [2:7577789513273401247:0] socket: 375 qp: -1
2025-11-28T14:53:29.182345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789511560122722:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:29.182389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:29.163343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789506451694527:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.163408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:29.163484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789506451694527:2102] [node 6] ICP22 created new session: [3:7577789510746661849:2102]
2025-11-28T14:53:29.163539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789510746661849:2102] [node 6] ICS09 handshake done sender: [3:7577789506451694529:2114] self: [3:7577789506451694528:0] peer: [6:7577789513290274603:0] socket: 430 qp: -1
2025-11-28T14:53:29.163577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789510746661849:2102] [node 6] ICS10 traffic start
2025-11-28T14:53:29.163616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:29.050189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:29.050256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789508866997133:2073] [node 6] ICP22 created new session: [4:7577789513161964501:2073]
2025-11-28T14:53:29.050310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789513161964501:2073] [node 6] ICS09 handshake done sender: [4:7577789508866997176:2112] self: [4:7577789508866997175:0] peer: [6:7577789508995307292:0] socket: 398 qp: -1
2025-11-28T14:53:29.050346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789513161964501:2073] [node 6] ICS10 traffic start
2025-11-28T14:53:29.050379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:29.051473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789513161964497:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11290
2025-11-28T14:53:29.092632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789513161964497:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:29.101912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789508866997133:2073] [node 6] ICP18 handshake (actor [4:7577789513161964497:2122]) is too old
2025-11-28T14:53:29.173598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789508866997185:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:29.173645Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789508866997185:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:29.172740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789504683466555:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:29.175733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789513273401248:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58192
2025-11-28T14:53:29.185866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789513273401246:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:29.187094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789508866997185:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:29.187927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789512262197227:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:29.189344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789508866997172:2084] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.189419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:29.189466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789508866997172:2084] [node 5] ICP22 created new session: [4:7577789513161964504:2084]
2025-11-28T14:53:29.189519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789513161964504:2084] [node 5] ICS09 handshake done sender: [4:7577789508866997185:2116] self: [4:7577789508866997184:0] peer: [5:7577789512262197231:0] socket: 440 qp: -1
2025-11-28T14:53:29.189561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789513161964504:2084] [node 5] ICS10 traffic start
2025-11-28T14:53:29.189602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:29.189956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789507967229902:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.190007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:29.190060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789507967229902:2065] [node 4] ICP22 created new session: [5:7577789512262197235:2065]
2025-11-28T14:53:29.190103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789512262197235:2065] [node 4] ICS09 handshake done sender: [5:7577789512262197227:2120] self: [5:7577789512262197231:0] peer: [4:7577789508866997184:0] socket: 396 qp: -1
2025-11-28T14:53:29.190137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789512262197235:2065] [node 4] ICS10 traffic start
2025-11-28T14:53:29.190176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:29.187769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789504683466555:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.186368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789508995307210:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:29.200692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789508995307212:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12471) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789508995307298:2121] held: no
2025-11-28T14:53:29.201733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789513290274608:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29501
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:29.210310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789512262197234:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63987
2025-11-28T14:53:29.199318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789507265155401:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:29.187833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:29.199694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789507265155392:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:29.199761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:29.199818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789507265155392:2107] [node 5] ICP22 created new session: [1:7577789511560122725:2107]
2025-11-28T14:53:29.199870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789511560122725:2107] [node 5] ICS09 handshake done sender: [1:7577789507265155401:2115] self: [1:7577789507265155400:0] peer: [5:7577789512262197233:0] socket: 400 qp: -1
2025-11-28T14:53:29.199909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789511560122725:2107] [node 5] ICS10 traffic start
2025-11-28T14:53:29.199946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:29.220473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789503672262543:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10025) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789512262197230:2121] held: no
2025-11-28T14:53:29.208601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789508995307298:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:29.213634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789508995307212:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.220714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:29.220796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789508995307212:2050] [node 3] ICP22 created new session: [6:7577789513290274609:2050]
2025-11-28T14:53:29.220867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789513290274609:2050] [node 3] ICS09 handshake done sender: [6:7577789508995307298:2121] self: [6:7577789513290274603:0] peer: [3:7577789506451694528:0] socket: 432 qp: -1
2025-11-28T14:53:29.220916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789513290274609:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:29.220958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:29.187890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789504683466555:2048] [node 1] ICP22 created new session: [2:7577789513273401249:2048]
2025-11-28T14:53:29.231543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789503672262543:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16606477ns) {HandleHandshakeStatus:395(16604354ns) {LogHandshakeFail:472(12706ns) {UpdateErrorStateLog:545(5152ns)} DropOutgoingHandshake:457(2605ns) ProcessPendingSessionEvents:495(10091276ns) {unordered DropSessionEvent:512 num# 100 sum# 1199467ns max# 599071ns}}}
2025-11-28T14:53:29.231660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789512262197230:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:29.198490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789513273401249:2048] [node 1] ICS09 handshake done sender: [2:7577789513273401246:2118] self: [2:7577789513273401247:0] peer: [1:7577789507265155398:0] socket: 401 qp: -1
2025-11-28T14:53:29.208850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789513273401249:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:29.213430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:29.216088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789504683466555:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27283682ns) {HandleHandshakeStatus:329(27281091ns) {DropIncomingHandshake:442(986ns) DropOutgoingHandshake:457(5202ns) SwitchToState:179(1135ns) ProcessPendingSessionEvents:495(809481ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 749720ns max# 67031ns}}}
2025-11-28T14:53:29.299996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789503672262543:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:29.300095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:29.300179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789503672262543:2048] [node 1] ICP22 created new session: [5:7577789512262197237:2048]
2025-11-28T14:53:29.300243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789512262197237:2048] [node 1] ICS09 handshake done sender: [5:7577789512262197230:2121] self: [5:7577789512262197233:0] peer: [1:7577789507265155400:0] socket: 411 qp: -1
2025-11-28T14:53:29.300283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789512262197237:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:29.300332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:29.353555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:29.353762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789513273401249:2048] [node 1] ICS07 socket disconnect 401 reason# EndOfStream
2025-11-28T14:53:29.353786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789513273401249:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.353858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:29.353895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789513273401249:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:29.355875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:29.356059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789510746661843:2048] [node 1] ICS01 socket: 392 reason# EndOfStream
2025-11-28T14:53:29.356095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789502156727155:2048] [node 1] ICP30 unregister session Session# [3:7577789510746661843:2048] VirtualId# [3:7577789506451694538:0]
2025-11-28T14:53:29.356124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789510746661843:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.356177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:29.356211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:29.364595Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:29.364939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:29.365156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789512262197237:2048] [node 1] ICS01 socket: 411 reason# EndOfStream
2025-11-28T14:53:29.365194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789503672262543:2048] [node 1] ICP30 unregister session Session# [5:7577789512262197237:2048] VirtualId# [5:7577789512262197233:0]
2025-11-28T14:53:29.365230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789512262197237:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.365308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:29.365351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:29.364728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:29.389634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789513161964499:2048] [node 1] ICS07 socket disconnect 395 reason# EndOfStream
2025-11-28T14:53:29.389671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789513161964499:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.389734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:29.389775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789513161964499:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:29.385201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789513290274606:2048] [node 1] ICS07 socket disconnect 404 reason# EndOfStream
2025-11-28T14:53:29.388663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789513290274606:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.388819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:29.388865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789513290274606:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:29.391760Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789508995307210:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10025) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:29.391820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789513290274606:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:29.391854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789508995307210:2048] [node 1] ICP30 unregister session Session# [6:7577789513290274606:2048] VirtualId# [6:7577789508995307295:0]
2025-11-28T14:53:29.391975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:29.392009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789508995307210:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10025) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.392024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10025) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.404014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789504572029810:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10025) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:29.404098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789513161964499:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:29.404677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789508995307210:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:29.404140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789504572029810:2048] [node 1] ICP30 unregister session Session# [4:7577789513161964499:2048] VirtualId# [4:7577789508866997187:0]
2025-11-28T14:53:29.404193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:29.404232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789504572029810:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10025) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.404249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10025) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.408609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789504572029810:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:29.444734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:29.444965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789506451694535:2049] [node 2] ICS01 socket: 383 reason# EndOfStream
2025-11-28T14:53:29.445010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789502156727156:2049] [node 2] ICP30 unregister session Session# [3:7577789506451694535:2049] VirtualId# [3:7577789506451694532:0]
2025-11-28T14:53:29.445047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789506451694535:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.445137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:29.445178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:29.455758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:29.455986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789508866997194:2049] [node 2] ICS01 socket: 372 reason# EndOfStream
2025-11-28T14:53:29.456057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789504572029811:2049] [node 2] ICP30 unregister session Session# [4:7577789508866997194:2049] VirtualId# [4:7577789508866997190:0]
2025-11-28T14:53:29.456117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789508866997194:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.456260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:29.456315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:29.456680Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:29.456848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789508995307300:2049] [node 2] ICS01 socket: 377 reason# EndOfStream
2025-11-28T14:53:29.456884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789508995307211:2049] [node 2] ICP30 unregister session Session# [6:7577789508995307300:2049] VirtualId# [6:7577789508995307293:0]
2025-11-28T14:53:29.456913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789508995307300:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.456972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:29.457008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:29.456672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:29.456883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789507967229921:2049] [node 2] ICS07 socket disconnect 403 reason# EndOfStream
2025-11-28T14:53:29.456912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789507967229921:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.456997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:29.457039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789507967229921:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:29.458164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789503672262544:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20707) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:29.458221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789507967229921:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:29.458259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789503672262544:2049] [node 2] ICP30 unregister session Session# [5:7577789507967229921:2049] VirtualId# [5:7577789507967229919:0]
2025-11-28T14:53:29.458305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:29.458336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789503672262544:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20707) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.458351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20707) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.468595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789503672262544:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:29.682479Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:29.682744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789508866997198:2050] [node 3] ICS07 socket disconnect 394 reason# EndOfStream
2025-11-28T14:53:29.682777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789508866997198:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.682875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:29.682923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789508866997198:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:29.684583Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789504572029812:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12471) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:29.684641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789508866997198:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:29.684692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789504572029812:2050] [node 3] ICP30 unregister session Session# [4:7577789508866997198:2050] VirtualId# [4:7577789508866997186:0]
2025-11-28T14:53:29.684733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:29.684763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789504572029812:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.684784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.684689Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:29.684915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789513290274609:2050] [node 3] ICS01 socket: 432 reason# EndOfStream
2025-11-28T14:53:29.684962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789508995307212:2050] [node 3] ICP30 unregister session Session# [6:7577789513290274609:2050] VirtualId# [6:7577789513290274603:0]
2025-11-28T14:53:29.684996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789513290274609:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.685067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:29.685107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:29.684689Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:29.684919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789512262197228:2050] [node 3] ICS07 socket disconnect 427 reason# EndOfStream
2025-11-28T14:53:29.684950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789512262197228:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.685039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:29.685090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789512262197228:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:29.688701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789503672262545:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12471) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:29.688785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789512262197228:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:29.688826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789503672262545:2050] [node 3] ICP30 unregister session Session# [5:7577789512262197228:2050] VirtualId# [5:7577789507967229927:0]
2025-11-28T14:53:29.688942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:29.688986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789503672262545:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.689013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.704558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789503672262545:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:29.732811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:29.733054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789513290274604:2074] [node 4] ICS07 socket disconnect 407 reason# EndOfStream
2025-11-28T14:53:29.733088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789513290274604:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.733190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:29.733233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789513290274604:2074] [node 4] ICS15 start handshake
2025-11-28T14:53:29.733750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:29.733938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789512262197235:2065] [node 4] ICS01 socket: 396 reason# EndOfStream
2025-11-28T14:53:29.733980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789507967229902:2065] [node 4] ICP30 unregister session Session# [5:7577789512262197235:2065] VirtualId# [5:7577789512262197231:0]
2025-11-28T14:53:29.734019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789512262197235:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.734080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:29.734123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:29.735362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789508995307278:2074] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6700) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:29.735422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789513290274604:2074] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:29.735462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789508995307278:2074] [node 4] ICP30 unregister session Session# [6:7577789513290274604:2074] VirtualId# [6:7577789508995307292:0]
2025-11-28T14:53:29.735498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:29.735537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789508995307278:2074] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.735556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:29.746170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789508995307278:2074] [node 4] ICP33 wake up from error state
2025-11-28T14:53:29.912729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:29.912998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789508995307302:2072] [node 5] ICS01 socket: 417 reason# EndOfStream
2025-11-28T14:53:29.913050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789508995307244:2072] [node 5] ICP30 unregister session Session# [6:7577789508995307302:2072] VirtualId# [6:7577789508995307294:0]
2025-11-28T14:53:29.913095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789508995307302:2072] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:29.913224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:29.913273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:30.133081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789515132763628:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:30.133137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789515132763629:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:30.184758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789516682942923:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:30.184813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789516682942924:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:30.250399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789515961169786:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:30.250450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789515961169787:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:30.316810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789516166254124:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:30.316862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789516166254125:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:30.316930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789516166254126:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:30.353650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789517609974298:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:30.353697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789517609974299:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:30.353741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789517609974300:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:30.596969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789514356990848:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:30.597018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789514356990849:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:30.597047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789514356990850:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:30.599325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789516166254140:2063] [node 5] ICP01 ready to work
2025-11-28T14:53:30.613715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789515961169806:2063] [node 4] ICP01 ready to work
2025-11-28T14:53:30.721896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789517609974314:2062] [node 4] ICP01 ready to work
2025-11-28T14:53:30.673687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789515961169810:2067] [node 5] ICP01 ready to work
2025-11-28T14:53:30.654459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789515132763688:2102] [node 6] ICP01 ready to work
2025-11-28T14:53:30.736495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789516682942975:2069] [node 5] ICP01 ready to work
2025-11-28T14:53:30.777473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789514356990867:2061] [node 4] ICP01 ready to work
2025-11-28T14:53:30.754612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789515132763693:2103] [node 4] ICP01 ready to work
2025-11-28T14:53:30.728240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789515961169813:2065] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:30.763239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789515132763691:2104] [node 5] ICP01 ready to work
2025-11-28T14:53:30.903132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789516682942986:2070] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:30.877207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789516166254181:2090] [node 6] ICP01 ready to work
2025-11-28T14:53:30.946647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789516682942990:2074] [node 6] ICP01 ready to work
2025-11-28T14:53:30.916973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789515132763694:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16186
2025-11-28T14:53:30.922164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789517609974300:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(229660945ns) {Configure:104(229652174ns) {ProcessConfigured:135(78831ns) {StartInitialHandshake:157(75655ns) {DropHandshakes:470(4101ns) {DropIncomingHandshake:442(729ns) DropOutgoingHandshake:457(990ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:477(2697ns) {SwitchToState:179(1200ns)}}}}}
2025-11-28T14:53:30.966089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789516166254125:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(86072208ns) {Configure:104(86066954ns) {ProcessConfigured:135(76510ns) {StartInitialHandshake:157(73218ns) {DropHandshakes:470(4128ns) {DropIncomingHandshake:442(1062ns) DropOutgoingHandshake:457(722ns)} GenerateSessionVirtualId:988(1731ns) PrepareNewSessionHandshake:477(2173ns) {SwitchToState:179(925ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:30.981887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789516166254196:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36686
2025-11-28T14:53:30.982099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789516166254197:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36686
2025-11-28T14:53:30.952940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789516682942924:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(333324280ns) {Configure:104(333304884ns) {ProcessConfigured:135(25894616ns) {StartInitialHandshake:157(25890000ns) {DropHandshakes:470(4328ns) {DropIncomingHandshake:442(850ns) DropOutgoingHandshake:457(1092ns)} GenerateSessionVirtualId:988(1257ns) PrepareNewSessionHandshake:477(3601ns) {SwitchToState:179(540ns)}}}}}
2025-11-28T14:53:30.940168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789517609974369:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39277
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:30.983638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789516166254197:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:30.998348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789514356990850:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26331086ns) {Configure:104(26325788ns) {ProcessConfigured:135(77767ns) {StartInitialHandshake:157(74881ns) {DropHandshakes:470(3932ns) {DropIncomingHandshake:442(1207ns) DropOutgoingHandshake:457(759ns)} GenerateSessionVirtualId:988(1145ns) PrepareNewSessionHandshake:477(2629ns) {SwitchToState:179(1074ns)}}}}}
2025-11-28T14:53:31.002233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789518651958215:2075] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.032485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789521904941671:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39277
2025-11-28T14:53:31.032642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789521904941672:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39277
2025-11-28T14:53:31.032740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789521904941673:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39277
2025-11-28T14:53:31.034830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789517609974369:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.034950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789521904941671:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.035110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789517609974298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.035204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789517609974314:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.020001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789520977910291:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53806
2025-11-28T14:53:31.020083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789520977910292:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53806
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:30.985081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789516166254140:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(341181451ns) {Configure:104(341175570ns) {ProcessConfigured:135(37109405ns) {StartInitialHandshake:157(37106261ns) {DropHandshakes:470(2830ns) {DropIncomingHandshake:442(660ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(683ns) PrepareNewSessionHandshake:477(3842ns) {SwitchToState:179(1531ns)}}}}}
2025-11-28T14:53:31.044448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789517609974374:2072] [node 6] ICP01 ready to work
2025-11-28T14:53:31.004793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789515132763708:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16186
2025-11-28T14:53:31.020461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789515132763628:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(60813066ns) {Configure:104(60807017ns) {ProcessConfigured:135(58712319ns) {StartInitialHandshake:157(58709695ns) {DropHandshakes:470(8018778ns) {DropIncomingHandshake:442(2092ns) DropOutgoingHandshake:457(560ns)} GenerateSessionVirtualId:988(1425ns) PrepareNewSessionHandshake:477(3472ns) {SwitchToState:179(975ns)}}}}}
2025-11-28T14:53:31.033366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789520461221498:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36686
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.105211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789515961169787:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(388112464ns) {Configure:104(388104524ns) {ProcessConfigured:135(76523ns) {StartInitialHandshake:157(74492ns) {DropHandshakes:470(2934ns) {DropIncomingHandshake:442(664ns) DropOutgoingHandshake:457(377ns)} GenerateSessionVirtualId:988(1010ns) PrepareNewSessionHandshake:477(3543ns) {SwitchToState:179(516ns)}}}}}
2025-11-28T14:53:31.108110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789520256137148:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2932
2025-11-28T14:53:31.108304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789520256137149:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2932
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.088450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789521904941672:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.114051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789515961169806:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21687375ns) {Configure:104(21681998ns) {ProcessConfigured:135(14800014ns) {StartInitialHandshake:157(14795788ns) {DropHandshakes:470(2007ns) {DropIncomingHandshake:442(238ns) DropOutgoingHandshake:457(828ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:477(12337ns) {SwitchToState:179(1563ns)}}}}}
2025-11-28T14:53:31.116833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789520256137150:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2932
2025-11-28T14:53:31.117126Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789520256137149:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.034143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789515132763708:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.036079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789516166254191:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.096579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789516682942923:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(401085527ns) {Configure:104(401079384ns) {ProcessConfigured:135(33614330ns) {StartInitialHandshake:157(33609345ns) {DropHandshakes:470(1282810ns) {DropIncomingHandshake:442(1438ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(2257ns) PrepareNewSessionHandshake:477(7467ns) {SwitchToState:179(3692ns)}}}}}
2025-11-28T14:53:31.108945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789521904941676:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39277
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.125008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789520256137150:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.036105Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789516166254191:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.097182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789516682942990:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(139602737ns) {Configure:104(139598205ns) {ProcessConfigured:135(2936661ns) {StartInitialHandshake:157(2934219ns) {DropHandshakes:470(6795ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(369ns)} GenerateSessionVirtualId:988(791ns) PrepareNewSessionHandshake:477(3092ns) {SwitchToState:179(1704ns)}}}}}
2025-11-28T14:53:31.045531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789519427731006:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16186
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.109732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789521904941677:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39277
2025-11-28T14:53:31.148589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789514356990849:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(343742611ns) {Configure:104(343736625ns) {ProcessConfigured:135(23431436ns) {StartInitialHandshake:157(18874800ns) {DropHandshakes:470(4829ns) {DropIncomingHandshake:442(2056ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(2865ns) PrepareNewSessionHandshake:477(3064ns) {SwitchToState:179(1606ns)}}}}}
2025-11-28T14:53:31.073059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789516166254196:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.127129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789515961169787:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.150759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958222:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.097343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789520977910291:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.068631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789515132763701:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.110098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789521904941671:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:31.150255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789520256137164:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2932
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.090293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789516166254124:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(458965734ns) {Configure:104(458955903ns) {ProcessConfigured:135(5576047ns) {StartInitialHandshake:157(5572766ns) {DropHandshakes:470(18032ns) {DropIncomingHandshake:442(859ns) DropOutgoingHandshake:457(15264ns)} GenerateSessionVirtualId:988(1118ns) PrepareNewSessionHandshake:477(2262ns) {SwitchToState:179(1417ns)}}}}}
2025-11-28T14:53:31.150818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958223:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.068688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789515132763701:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.101467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789520977910292:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.110289Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789517609974369:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:31.090469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789520461221498:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.165221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958224:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.158485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789515961169786:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(239658916ns) {Configure:104(239650456ns) {ProcessConfigured:135(68182ns) {StartInitialHandshake:157(64734ns) {DropHandshakes:470(3548ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(657ns)} GenerateSessionVirtualId:988(1080ns) PrepareNewSessionHandshake:477(2462ns) {SwitchToState:179(944ns)}}}}}
2025-11-28T14:53:31.097144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789515132763701:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:31.110499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789517609974314:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.161738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789516682942993:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.192609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789520256137148:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.200776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789515961169786:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.091016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789516166254191:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:31.194467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958225:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.215675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789520256137154:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.161791Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789516682942993:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.163136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789516682942993:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:31.163468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789516682942975:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.163533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:31.163589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789516682942975:2069] [node 5] ICP22 created new session: [2:7577789520977910304:2069]
2025-11-28T14:53:31.163637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789520977910304:2069] [node 5] ICS09 handshake done sender: [2:7577789516682942993:2111] self: [2:7577789516682942992:0] peer: [5:7577789521904941685:0] socket: 402 qp: -1
2025-11-28T14:53:31.163672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789520977910304:2069] [node 5] ICS10 traffic start
2025-11-28T14:53:31.163709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:31.196970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789516682942989:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.197013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789516682942989:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.208265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789516682942989:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:31.225011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789516682942924:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.238695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:31.249036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789516682942924:2049] [node 3] ICP22 created new session: [2:7577789520977910308:2049]
2025-11-28T14:53:31.252512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789520977910302:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53806
2025-11-28T14:53:31.264789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789520977910294:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.264820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789520977910294:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.265802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789520977910309:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53806
2025-11-28T14:53:31.278771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789520977910294:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:31.281786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789520977910308:2049] [node 3] ICS09 handshake done sender: [2:7577789516682942989:2110] self: [2:7577789516682942988:0] peer: [3:7577789520256137162:0] socket: 401 qp: -1
2025-11-28T14:53:31.281839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789520977910308:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:31.097438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789515132763694:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.110590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:31.116666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789516166254140:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.112481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789515132763691:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.112576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:31.115498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789515132763691:2104] [node 5] ICP22 created new session: [1:7577789519427731009:2104]
2025-11-28T14:53:31.129016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789519427731009:2104] [node 5] ICS09 handshake done sender: [1:7577789515132763701:2113] self: [1:7577789515132763700:0] peer: [5:7577789521904941674:0] socket: 383 qp: -1
2025-11-28T14:53:31.129106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789519427731009:2104] [node 5] ICS10 traffic start
2025-11-28T14:53:31.139278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789519427731006:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.140609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:31.176951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789515132763698:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.176991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789515132763698:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.201483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789515132763698:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:31.212932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789515132763696:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.212962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789515132763696:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.228771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789515132763693:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.228839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:31.228889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789515132763693:2103] [node 4] ICP22 created new session: [1:7577789519427731015:2103]
2025-11-28T14:53:31.228950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789519427731015:2103] [node 4] ICS09 handshake done sender: [1:7577789515132763698:2112] self: [1:7577789515132763697:0] peer: [4:7577789520461221506:0] socket: 382 qp: -1
2025-11-28T14:53:31.228979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789519427731015:2103] [node 4] ICS10 traffic start
2025-11-28T14:53:31.229018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:31.240902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789515132763696:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:31.278581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789515132763629:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.215546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789518651958223:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.215717Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789520256137154:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.116742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:31.129400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789516166254140:2063] [node 5] ICP22 created new session: [4:7577789520461221504:2063]
2025-11-28T14:53:31.139647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789520461221504:2063] [node 5] ICS09 handshake done sender: [4:7577789516166254191:2110] self: [4:7577789516166254190:0] peer: [5:7577789521904941675:0] socket: 394 qp: -1
2025-11-28T14:53:31.142189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789516166254124:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.142652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789520461221505:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36686
2025-11-28T14:53:31.152671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789520461221504:2063] [node 5] ICS10 traffic start
2025-11-28T14:53:31.152747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:31.167268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789520461221505:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.179220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789516166254126:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.193577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789520461221510:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36686
2025-11-28T14:53:31.193651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789520461221511:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36686
2025-11-28T14:53:31.206700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789516166254140:2063] [node 5] ICP35 event processing took too much time PendingConnection:279(92360401ns) {HandleHandshakeStatus:329(92357686ns) {DropIncomingHandshake:442(20380ns) DropOutgoingHandshake:457(228ns) SwitchToState:179(523ns) ProcessPendingSessionEvents:495(697936ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 630406ns max# 85697ns}}}
2025-11-28T14:53:31.206939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789520461221511:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.218698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789516166254197:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:31.234457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789516166254125:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.234685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789516166254124:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.234732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:31.234779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789516166254124:2048] [node 1] ICP22 created new session: [4:7577789520461221513:2048]
2025-11-28T14:53:31.234827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789520461221513:2048] [node 1] ICS09 handshake done sender: [4:7577789516166254197:2114] self: [4:7577789520461221506:0] peer: [1:7577789515132763697:0] socket: 389 qp: -1
2025-11-28T14:53:31.234858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789520461221513:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:31.234894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:31.266353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789516166254195:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.266380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789516166254195:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.267551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789520461221515:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36686
2025-11-28T14:53:31.313142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789516166254195:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.110655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789517609974314:2062] [node 4] ICP22 created new session: [5:7577789521904941678:2062]
2025-11-28T14:53:31.308662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:31.294673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:31.221162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789520256137168:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2932
2025-11-28T14:53:31.220968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789518651958224:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.324028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789520461221516:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.302245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789520977910309:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.302286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789516682942986:2070] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.302334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:31.302374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789516682942986:2070] [node 4] ICP22 created new session: [2:7577789520977910313:2070]
2025-11-28T14:53:31.302407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789520977910313:2070] [node 4] ICS09 handshake done sender: [2:7577789520977910294:2115] self: [2:7577789520977910293:0] peer: [4:7577789520461221512:0] socket: 399 qp: -1
2025-11-28T14:53:31.302433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789520977910313:2070] [node 4] ICS10 traffic start
2025-11-28T14:53:31.302473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:31.304817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789516682942923:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.324668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789516682942924:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(70785424ns) {HandleHandshakeStatus:329(70781921ns) {DropIncomingHandshake:442(648ns) DropOutgoingHandshake:457(965ns) SwitchToState:179(826ns) ProcessPendingSessionEvents:495(838669ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 734730ns max# 124537ns}}}
2025-11-28T14:53:31.324749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789516682942924:2049] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.110710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789521904941678:2062] [node 4] ICS09 handshake done sender: [5:7577789521904941671:2115] self: [5:7577789521904941675:0] peer: [4:7577789516166254190:0] socket: 406 qp: -1
2025-11-28T14:53:31.317348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789515132763629:2049] [node 3] ICP22 created new session: [1:7577789519427731018:2049]
2025-11-28T14:53:31.221110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789518651958222:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.227244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789520256137164:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.334698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789520461221505:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:31.110752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789521904941678:2062] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.227361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789520256137150:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:31.238786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789514356990850:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.367331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789520977910316:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53806
2025-11-28T14:53:31.110804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:31.325876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789519427731018:2049] [node 3] ICS09 handshake done sender: [1:7577789515132763696:2111] self: [1:7577789515132763695:0] peer: [3:7577789520256137167:0] socket: 381 qp: -1
2025-11-28T14:53:31.335047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789516166254125:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11986) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789520461221511:2120] held: no
2025-11-28T14:53:31.328942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789519427731018:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:31.244149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789518651958225:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.227680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789515961169787:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.358638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789516166254181:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.227736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:31.245523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789514356990867:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.358714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:31.358779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789516166254181:2090] [node 6] ICP22 created new session: [4:7577789520461221518:2090]
2025-11-28T14:53:31.359380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789520461221518:2090] [node 6] ICS09 handshake done sender: [4:7577789516166254195:2112] self: [4:7577789516166254194:0] peer: [6:7577789518651958228:0] socket: 425 qp: -1
2025-11-28T14:53:31.359446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789520461221518:2090] [node 6] ICS10 traffic start
2025-11-28T14:53:31.359513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:31.369345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789520461221511:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:31.369678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789516166254126:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29707) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789520461221505:2118] held: no
2025-11-28T14:53:31.370312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789516166254126:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.370383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:31.370441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789516166254126:2050] [node 3] ICP22 created new session: [4:7577789520461221520:2050]
2025-11-28T14:53:31.370492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789520461221520:2050] [node 3] ICS09 handshake done sender: [4:7577789520461221505:2118] self: [4:7577789520461221509:0] peer: [3:7577789520256137153:0] socket: 380 qp: -1
2025-11-28T14:53:31.370540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789520461221520:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:31.370578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:31.377658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789516166254125:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.377741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:31.377814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789516166254125:2049] [node 2] ICP22 created new session: [4:7577789520461221522:2049]
2025-11-28T14:53:31.377873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789520461221522:2049] [node 2] ICS09 handshake done sender: [4:7577789520461221511:2120] self: [4:7577789520461221512:0] peer: [2:7577789520977910293:0] socket: 432 qp: -1
2025-11-28T14:53:31.377922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789520461221522:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:31.377960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:31.398290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789516166254181:2090] [node 6] ICP35 event processing took too much time PendingConnection:279(38721590ns) {HandleHandshakeStatus:329(38717603ns) {DropIncomingHandshake:442(17007ns) DropOutgoingHandshake:457(504ns) SwitchToState:179(751ns) ProcessPendingSessionEvents:495(37665761ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37574601ns max# 8681282ns}}}
2025-11-28T14:53:31.111873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789517609974298:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.332615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789515132763707:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.332676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789515132763707:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.111946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:31.249202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789518651958215:2075] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.227805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789515961169787:2049] [node 2] ICP22 created new session: [3:7577789520256137170:2049]
2025-11-28T14:53:31.406414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789520977910309:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:31.425606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789516682942923:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14911) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789520977910309:2118] held: no
2025-11-28T14:53:31.111990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789517609974298:2048] [node 1] ICP22 created new session: [5:7577789521904941680:2048]
2025-11-28T14:53:31.332806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789515132763691:2104] [node 5] ICP35 event processing took too much time PendingConnection:279(40131423ns) {HandleHandshakeStatus:329(40127470ns) {DropIncomingHandshake:442(32162ns) DropOutgoingHandshake:457(602ns) SwitchToState:179(349ns) ProcessPendingSessionEvents:495(600236ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 537643ns max# 74193ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.112039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789521904941680:2048] [node 1] ICS09 handshake done sender: [5:7577789517609974369:2113] self: [5:7577789521904941674:0] peer: [1:7577789515132763700:0] socket: 384 qp: -1
2025-11-28T14:53:31.112072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789521904941680:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:31.112099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:31.112803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789521904941673:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.147471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789517609974299:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.208851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789517609974299:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11986) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789521904941673:2117] held: no
2025-11-28T14:53:31.219600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789521904941688:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39277
2025-11-28T14:53:31.277484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789517609974299:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(67837777ns) {HandleHandshakeStatus:395(67834290ns) {LogHandshakeFail:472(13848ns) {UpdateErrorStateLog:545(6921ns)} DropOutgoingHandshake:457(550ns) ProcessPendingSessionEvents:495(67759176ns) {unordered DropSessionEvent:512 num# 100 sum# 67694370ns max# 13962382ns}}}
2025-11-28T14:53:31.277798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789521904941683:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.277816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789521904941683:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.278402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789521904941673:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:31.321185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789521904941689:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39277
2025-11-28T14:53:31.321308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789517609974299:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.276608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789514356990848:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.227850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789520256137170:2049] [node 2] ICS09 handshake done sender: [3:7577789520256137150:2112] self: [3:7577789520256137162:0] peer: [2:7577789516682942988:0] socket: 411 qp: -1
2025-11-28T14:53:31.336698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:31.321358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:31.296715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789518651958215:2075] [node 5] ICP35 event processing took too much time PendingConnection:279(24292558ns) {IncomingHandshake:261(10601192ns) {DropIncomingHandshake:442(2437ns) PrepareNewSessionHandshake:477(3926ns) {SwitchToState:179(2741ns)} GenerateSessionVirtualId:988(994ns) IssueIncomingHandshakeReply:193(53514ns) {SwitchToState:179(993ns)}}}
2025-11-28T14:53:31.227887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789520256137170:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:31.345416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789515132763706:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.321407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789517609974299:2049] [node 2] ICP22 created new session: [5:7577789521904941691:2049]
2025-11-28T14:53:31.227943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:31.249065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789520256137169:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2932
2025-11-28T14:53:31.253954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789520256137152:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.253982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789520256137152:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.256190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789520256137154:2114] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:31.260541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789520256137152:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:31.284679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789515961169806:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.284757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:31.284812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789515961169806:2063] [node 4] ICP22 created new session: [3:7577789520256137176:2063]
2025-11-28T14:53:31.284860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789520256137176:2063] [node 4] ICS09 handshake done sender: [3:7577789520256137154:2114] self: [3:7577789520256137153:0] peer: [4:7577789520461221509:0] socket: 417 qp: -1
2025-11-28T14:53:31.284895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789520256137176:2063] [node 4] ICS10 traffic start
2025-11-28T14:53:31.284931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:31.286232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789515961169813:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.286307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:31.286350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789515961169813:2065] [node 6] ICP22 created new session: [3:7577789520256137178:2065]
2025-11-28T14:53:31.286400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789520256137178:2065] [node 6] ICS09 handshake done sender: [3:7577789520256137152:2113] self: [3:7577789520256137151:0] peer: [6:7577789518651958226:0] socket: 422 qp: -1
2025-11-28T14:53:31.286442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789520256137178:2065] [node 6] ICS10 traffic start
2025-11-28T14:53:31.286482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:31.318838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789520256137148:2110] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:31.326066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789515961169786:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.333009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:31.333071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789515961169786:2048] [node 1] ICP22 created new session: [3:7577789520256137181:2048]
2025-11-28T14:53:31.333128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789520256137181:2048] [node 1] ICS09 handshake done sender: [3:7577789520256137148:2110] self: [3:7577789520256137167:0] peer: [1:7577789515132763695:0] socket: 409 qp: -1
2025-11-28T14:53:31.333166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789520256137181:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:31.333212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:31.441280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789515961169787:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(178368802ns) {HandleHandshakeStatus:329(178365850ns) {DropIncomingHandshake:442(1005ns) DropOutgoingHandshake:457(5700ns) SwitchToState:179(874ns) ProcessPendingSessionEvents:495(177929071ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 162266010ns max# 15150973ns}}}
2025-11-28T14:53:31.471683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789520256137158:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.471742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789520256137158:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.320022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958230:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.461490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789516682942923:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(34009279ns) {HandleHandshakeStatus:395(21116139ns) {LogHandshakeFail:472(26445ns) {UpdateErrorStateLog:545(7165ns)} DropOutgoingHandshake:457(3801ns) ProcessPendingSessionEvents:495(15495883ns) {unordered DropSessionEvent:512 num# 100 sum# 15452960ns max# 6612077ns}}}
2025-11-28T14:53:31.321464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789521904941691:2049] [node 2] ICS09 handshake done sender: [5:7577789521904941673:2117] self: [5:7577789521904941685:0] peer: [2:7577789516682942992:0] socket: 408 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.321500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789521904941691:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:31.321538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:31.344307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789521904941683:2120] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:31.360642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789521904941689:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.360704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789517609974374:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.372717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:31.379541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789517609974374:2072] [node 6] ICP22 created new session: [5:7577789521904941693:2072]
2025-11-28T14:53:31.379618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789521904941693:2072] [node 6] ICS09 handshake done sender: [5:7577789521904941683:2120] self: [5:7577789521904941682:0] peer: [6:7577789518651958227:0] socket: 390 qp: -1
2025-11-28T14:53:31.379663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789521904941693:2072] [node 6] ICS10 traffic start
2025-11-28T14:53:31.379717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:31.428855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789517609974298:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(185111569ns) {HandleHandshakeStatus:329(185109533ns) {DropIncomingHandshake:442(585ns) DropOutgoingHandshake:457(4005ns) SwitchToState:179(368ns) ProcessPendingSessionEvents:495(184801100ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 184609747ns max# 21230540ns}}}
2025-11-28T14:53:31.470766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789517609974300:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.481372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789521904941696:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39277
2025-11-28T14:53:31.483175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789517609974374:2072] [node 6] ICP35 event processing took too much time PendingConnection:279(84164177ns) {HandleHandshakeStatus:329(84161016ns) {DropIncomingHandshake:442(20234ns) DropOutgoingHandshake:457(154ns) SwitchToState:179(154ns) ProcessPendingSessionEvents:495(65025818ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 64890680ns max# 9896338ns}}}
2025-11-28T14:53:31.345447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789515132763706:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.347502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789519427731020:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16186
2025-11-28T14:53:31.352536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789515132763706:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:31.360642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789515132763688:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.360709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:31.360772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789515132763688:2102] [node 6] ICP22 created new session: [1:7577789519427731023:2102]
2025-11-28T14:53:31.360821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789519427731023:2102] [node 6] ICS09 handshake done sender: [1:7577789515132763706:2114] self: [1:7577789515132763705:0] peer: [6:7577789518651958229:0] socket: 392 qp: -1
2025-11-28T14:53:31.360859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789519427731023:2102] [node 6] ICS10 traffic start
2025-11-28T14:53:31.360900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:31.365965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789515132763707:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:31.384790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789515132763628:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.384866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:31.384919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789515132763628:2048] [node 2] ICP22 created new session: [1:7577789519427731025:2048]
2025-11-28T14:53:31.384970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789519427731025:2048] [node 2] ICS09 handshake done sender: [1:7577789515132763707:2115] self: [1:7577789515132763699:0] peer: [2:7577789520977910315:0] socket: 436 qp: -1
2025-11-28T14:53:31.385005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789519427731025:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:31.385052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:31.483632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789515132763629:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(209373844ns) {HandleHandshakeStatus:329(203940399ns) {DropIncomingHandshake:442(576ns) DropOutgoingHandshake:457(918ns) SwitchToState:179(767ns) ProcessPendingSessionEvents:495(117646282ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 117508874ns max# 28170526ns}}}
2025-11-28T14:53:31.344775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789518651958223:2116] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:31.350465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958232:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.357472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958233:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.365456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958234:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.381137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789514356990850:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.381211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:31.381272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789514356990850:2050] [node 3] ICP22 created new session: [6:7577789518651958235:2050]
2025-11-28T14:53:31.381346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789518651958235:2050] [node 3] ICS09 handshake done sender: [6:7577789518651958223:2116] self: [6:7577789518651958226:0] peer: [3:7577789520256137151:0] socket: 424 qp: -1
2025-11-28T14:53:31.381384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789518651958235:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:31.381419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:31.396728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789514356990848:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14911) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789518651958225:2118] held: no
2025-11-28T14:53:31.416849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789514356990848:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22678950ns) {HandleHandshakeStatus:395(22677134ns) {LogHandshakeFail:472(3429345ns) {UpdateErrorStateLog:545(10473ns)} DropOutgoingHandshake:457(498ns) ProcessPendingSessionEvents:495(19166381ns) {unordered DropSessionEvent:512 num# 100 sum# 19129556ns max# 8125746ns}}}
2025-11-28T14:53:31.424894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789518651958224:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:31.429983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789518651958225:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:31.430263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789518651958222:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:31.432300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789514356990867:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20111) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789518651958224:2117] held: no
2025-11-28T14:53:31.432979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789514356990867:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.433067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:31.433125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789514356990867:2061] [node 4] ICP22 created new session: [6:7577789518651958237:2061]
2025-11-28T14:53:31.433179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789518651958237:2061] [node 4] ICS09 handshake done sender: [6:7577789518651958224:2117] self: [6:7577789518651958228:0] peer: [4:7577789516166254194:0] socket: 426 qp: -1
2025-11-28T14:53:31.433221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789518651958237:2061] [node 4] ICS10 traffic start
2025-11-28T14:53:31.433258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:31.436042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789518651958215:2075] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.436102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958231:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.436143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:31.436193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789518651958215:2075] [node 5] ICP22 created new session: [6:7577789518651958239:2075]
2025-11-28T14:53:31.436273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789518651958239:2075] [node 5] ICS09 handshake done sender: [6:7577789518651958222:2115] self: [6:7577789518651958227:0] peer: [5:7577789521904941682:0] socket: 423 qp: -1
2025-11-28T14:53:31.436297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789514356990848:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.436322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789518651958239:2075] [node 5] ICS10 traffic start
2025-11-28T14:53:31.436387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:31.436389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:31.436449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789514356990848:2048] [node 1] ICP22 created new session: [6:7577789518651958241:2048]
2025-11-28T14:53:31.436520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789518651958241:2048] [node 1] ICS09 handshake done sender: [6:7577789518651958225:2118] self: [6:7577789518651958229:0] peer: [1:7577789515132763705:0] socket: 429 qp: -1
2025-11-28T14:53:31.436561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789518651958241:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:31.436642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:31.454714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789518651958215:2075] [node 5] ICP35 event processing took too much time PendingConnection:279(17732726ns) {HandleHandshakeStatus:329(17729317ns) {DropIncomingHandshake:442(937ns) DropOutgoingHandshake:457(5534ns) SwitchToState:179(1083ns) ProcessPendingSessionEvents:495(17165544ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17058666ns max# 11480725ns}}}
2025-11-28T14:53:31.486093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789518651958231:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.486208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789518651958215:2075] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:31.461591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789516682942923:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.461642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:31.461700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789516682942923:2048] [node 1] ICP22 created new session: [2:7577789520977910317:2048]
2025-11-28T14:53:31.461752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789520977910317:2048] [node 1] ICS09 handshake done sender: [2:7577789520977910309:2118] self: [2:7577789520977910315:0] peer: [1:7577789515132763699:0] socket: 440 qp: -1
2025-11-28T14:53:31.461795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789520977910317:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:31.461845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:31.477355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789520256137158:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.490359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789520977910299:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.490401Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789520977910299:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:31.491467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789520977910319:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53806
2025-11-28T14:53:31.498051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789515961169810:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.492625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789521904941689:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:31.489780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789514356990849:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:31.497714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789520977910299:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:31.498153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:31.498206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789515961169810:2067] [node 5] ICP22 created new session: [3:7577789520256137184:2067]
2025-11-28T14:53:31.509921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789520256137184:2067] [node 5] ICS09 handshake done sender: [3:7577789520256137158:2116] self: [3:7577789520256137157:0] peer: [5:7577789521904941695:0] socket: 438 qp: -1
2025-11-28T14:53:31.509977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789520256137184:2067] [node 5] ICS10 traffic start
2025-11-28T14:53:31.510023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:31.518215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789518651958245:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41046
2025-11-28T14:53:31.517773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789520977910319:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:31.518800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789517609974300:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.523092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789518651958231:2120] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:31.517988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789516682942990:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:31.525340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789514356990849:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:31.526128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:31.526171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789514356990849:2049] [node 2] ICP22 created new session: [6:7577789518651958246:2049]
2025-11-28T14:53:31.526210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789518651958246:2049] [node 2] ICS09 handshake done sender: [6:7577789518651958231:2120] self: [6:7577789518651958244:0] peer: [2:7577789520977910298:0] socket: 400 qp: -1
2025-11-28T14:53:31.526236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789518651958246:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:31.526269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:31.535947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789515961169810:2067] [node 5] ICP35 event processing took too much time PendingConnection:279(36979446ns) {HandleHandshakeStatus:329(36975772ns) {DropIncomingHandshake:442(18499ns) DropOutgoingHandshake:457(900ns) SwitchToState:179(888ns) ProcessPendingSessionEvents:495(11529601ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11442035ns max# 8377173ns}}}
2025-11-28T14:53:31.518900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:31.518205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:31.518257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789516682942990:2074] [node 6] ICP22 created new session: [2:7577789520977910321:2074]
2025-11-28T14:53:31.518670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789520977910321:2074] [node 6] ICS09 handshake done sender: [2:7577789520977910299:2116] self: [2:7577789520977910298:0] peer: [6:7577789518651958244:0] socket: 442 qp: -1
2025-11-28T14:53:31.518716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789520977910321:2074] [node 6] ICS10 traffic start
2025-11-28T14:53:31.518775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:31.520115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789516682942990:2074] [node 6] ICP18 handshake (actor [2:7577789520977910319:2120]) is too old
2025-11-28T14:53:31.518953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789517609974300:2050] [node 3] ICP22 created new session: [5:7577789521904941697:2050]
2025-11-28T14:53:31.523236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789521904941697:2050] [node 3] ICS09 handshake done sender: [5:7577789521904941689:2122] self: [5:7577789521904941695:0] peer: [3:7577789520256137157:0] socket: 446 qp: -1
2025-11-28T14:53:31.523290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789521904941697:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:31.524822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:31.533263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789517609974300:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:31.640624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:31.640868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789520977910317:2048] [node 1] ICS01 socket: 440 reason# EndOfStream
2025-11-28T14:53:31.640914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789516682942923:2048] [node 1] ICP30 unregister session Session# [2:7577789520977910317:2048] VirtualId# [2:7577789520977910315:0]
2025-11-28T14:53:31.640952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789520977910317:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.641053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:31.641097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:31.714326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:31.714569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789520256137181:2048] [node 1] ICS01 socket: 409 reason# EndOfStream
2025-11-28T14:53:31.714616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789515961169786:2048] [node 1] ICP30 unregister session Session# [3:7577789520256137181:2048] VirtualId# [3:7577789520256137167:0]
2025-11-28T14:53:31.714657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789520256137181:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.714773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:31.714820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:31.730219Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:31.730411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789521904941680:2048] [node 1] ICS07 socket disconnect 384 reason# EndOfStream
2025-11-28T14:53:31.730439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789521904941680:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.730504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:31.730548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789521904941680:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:31.728681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:31.728931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789518651958241:2048] [node 1] ICS01 socket: 429 reason# EndOfStream
2025-11-28T14:53:31.728979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789514356990848:2048] [node 1] ICP30 unregister session Session# [6:7577789518651958241:2048] VirtualId# [6:7577789518651958229:0]
2025-11-28T14:53:31.729019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789518651958241:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.729129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:31.729182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:31.731550Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789517609974298:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14911) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:31.731614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789521904941680:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:31.731651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789517609974298:2048] [node 1] ICP30 unregister session Session# [5:7577789521904941680:2048] VirtualId# [5:7577789521904941674:0]
2025-11-28T14:53:31.731769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:31.731802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789517609974298:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:31.731818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:31.729293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:31.729499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789520461221513:2048] [node 1] ICS07 socket disconnect 389 reason# EndOfStream
2025-11-28T14:53:31.729536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789520461221513:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.729602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:31.729643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789520461221513:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:31.740958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789516166254124:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14911) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:31.743504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789520461221513:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:31.743560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789516166254124:2048] [node 1] ICP30 unregister session Session# [4:7577789520461221513:2048] VirtualId# [4:7577789520461221506:0]
2025-11-28T14:53:31.743625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:31.743671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789516166254124:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:31.743692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:31.747016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789517609974298:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:31.752795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789516166254124:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:31.852674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:31.852916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789520256137170:2049] [node 2] ICS07 socket disconnect 411 reason# EndOfStream
2025-11-28T14:53:31.852952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789520256137170:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.853070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:31.853114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789520256137170:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:31.861514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789515961169787:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11986) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:31.861568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789520256137170:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:31.863212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789515961169787:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11986) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:31.863265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789520256137170:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:31.863305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789515961169787:2049] [node 2] ICP30 unregister session Session# [3:7577789520256137170:2049] VirtualId# [3:7577789520256137162:0]
2025-11-28T14:53:31.863428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:31.863461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789515961169787:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11986) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:31.863476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11986) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:31.868615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:31.868831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789521904941691:2049] [node 2] ICS01 socket: 408 reason# EndOfStream
2025-11-28T14:53:31.868882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789517609974299:2049] [node 2] ICP30 unregister session Session# [5:7577789521904941691:2049] VirtualId# [5:7577789521904941685:0]
2025-11-28T14:53:31.868920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789521904941691:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.869012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:31.869061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:31.869165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:31.869362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789520461221522:2049] [node 2] ICS01 socket: 432 reason# EndOfStream
2025-11-28T14:53:31.869403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789516166254125:2049] [node 2] ICP30 unregister session Session# [4:7577789520461221522:2049] VirtualId# [4:7577789520461221512:0]
2025-11-28T14:53:31.869433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789520461221522:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.869499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:31.869543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:31.869644Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:31.869855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789518651958246:2049] [node 2] ICS07 socket disconnect 400 reason# EndOfStream
2025-11-28T14:53:31.869888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789518651958246:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.869953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:31.869992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789518651958246:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:31.872078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789514356990849:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11986) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:31.872145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789518651958246:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:31.872185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789514356990849:2049] [node 2] ICP30 unregister session Session# [6:7577789518651958246:2049] VirtualId# [6:7577789518651958244:0]
2025-11-28T14:53:31.872230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:31.872262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789514356990849:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11986) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:31.872282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11986) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:31.872952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789515961169787:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:31.880614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789514356990849:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:31.975506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:31.975759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789520461221520:2050] [node 3] ICS01 socket: 380 reason# EndOfStream
2025-11-28T14:53:31.975813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789516166254126:2050] [node 3] ICP30 unregister session Session# [4:7577789520461221520:2050] VirtualId# [4:7577789520461221509:0]
2025-11-28T14:53:31.975847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789520461221520:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.975942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:31.975530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:31.975993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:31.975759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789521904941697:2050] [node 3] ICS01 socket: 446 reason# EndOfStream
2025-11-28T14:53:31.975808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789517609974300:2050] [node 3] ICP30 unregister session Session# [5:7577789521904941697:2050] VirtualId# [5:7577789521904941695:0]
2025-11-28T14:53:31.975847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789521904941697:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.975941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:31.976007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:31.980584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:31.980865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789518651958235:2050] [node 3] ICS07 socket disconnect 424 reason# EndOfStream
2025-11-28T14:53:31.980900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789518651958235:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:31.981076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:31.981181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789518651958235:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:31.982579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789514356990850:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29707) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:31.982650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789518651958235:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:31.982692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789514356990850:2050] [node 3] ICP30 unregister session Session# [6:7577789518651958235:2050] VirtualId# [6:7577789518651958226:0]
2025-11-28T14:53:31.982744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:31.982779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789514356990850:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29707) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:31.982793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29707) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:32.000695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789514356990850:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:32.072637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:32.072861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789521904941678:2062] [node 4] ICS01 socket: 406 reason# EndOfStream
2025-11-28T14:53:32.072908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789517609974314:2062] [node 4] ICP30 unregister session Session# [5:7577789521904941678:2062] VirtualId# [5:7577789521904941675:0]
2025-11-28T14:53:32.072943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789521904941678:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:32.073043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:32.073087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:32.073202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:32.073382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789518651958237:2061] [node 4] ICS01 socket: 426 reason# EndOfStream
2025-11-28T14:53:32.073415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789514356990867:2061] [node 4] ICP30 unregister session Session# [6:7577789518651958237:2061] VirtualId# [6:7577789518651958228:0]
2025-11-28T14:53:32.073441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789518651958237:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:32.073499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:32.073542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:32.160673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:32.160896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789518651958239:2075] [node 5] ICS07 socket disconnect 423 reason# EndOfStream
2025-11-28T14:53:32.160932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789518651958239:2075] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:32.161023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:32.161071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789518651958239:2075] [node 5] ICS15 start handshake
2025-11-28T14:53:32.162117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789518651958215:2075] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28057) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:32.162177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789518651958239:2075] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:32.162216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789518651958215:2075] [node 5] ICP30 unregister session Session# [6:7577789518651958239:2075] VirtualId# [6:7577789518651958227:0]
2025-11-28T14:53:32.162319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:53:32.162347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789518651958215:2075] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28057) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:32.162361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28057) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:32.300468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789526168948879:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:32.300537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789526168948880:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:32.329443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789523463460692:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:32.329497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789523463460693:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:32.353697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789526400409407:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:32.353741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789526400409408:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:32.402431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789522958242978:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:32.402488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789522958242979:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:32.410513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789522958242977:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:32.436402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789524848795135:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:32.436453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789524848795136:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:32.436509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789524848795137:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:32.765397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789524848795148:2059] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:32.501492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789523999264920:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:32.932483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789522958242996:2062] [node 6] ICP01 ready to work
2025-11-28T14:53:32.501538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789523999264921:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:32.951491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789526168948942:2095] [node 4] ICP01 ready to work
2025-11-28T14:53:32.961757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789526168948943:2096] [node 6] ICP01 ready to work
2025-11-28T14:53:32.964440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789523463460755:2066] [node 6] ICP01 ready to work
2025-11-28T14:53:32.980693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789523463460756:2065] [node 4] ICP01 ready to work
2025-11-28T14:53:32.501573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789523999264922:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:33.011416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789526168948944:2097] [node 5] ICP01 ready to work
2025-11-28T14:53:32.996702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789523463460757:2064] [node 5] ICP01 ready to work
2025-11-28T14:53:32.802056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789523999264936:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:32.974944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789522958243035:2063] [node 5] ICP01 ready to work
2025-11-28T14:53:32.996567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789524848795163:2062] [node 4] ICP01 ready to work
2025-11-28T14:53:32.996448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789526400409470:2063] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.121880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789530463916241:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54255
2025-11-28T14:53:32.980484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789526400409471:2060] [node 4] ICP01 ready to work
2025-11-28T14:53:32.808952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789523999264922:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(59187461ns) {Configure:104(59182742ns) {ProcessConfigured:135(87291ns) {StartInitialHandshake:157(84361ns) {DropHandshakes:470(3536ns) {DropIncomingHandshake:442(636ns) DropOutgoingHandshake:457(377ns)} GenerateSessionVirtualId:988(921ns) PrepareNewSessionHandshake:477(1323ns) {SwitchToState:179(498ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.162025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789530695376770:2070] [node 5] ICP01 ready to work
2025-11-28T14:53:33.165303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789523463460693:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33732181ns) {Configure:104(33726827ns) {ProcessConfigured:135(74662ns) {StartInitialHandshake:157(72119ns) {DropHandshakes:470(2711ns) {DropIncomingHandshake:442(1049ns) DropOutgoingHandshake:457(376ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:477(1710ns) {SwitchToState:179(526ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.158217Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789522958242979:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(277373755ns) {Configure:104(277363825ns) {ProcessConfigured:135(88437ns) {StartInitialHandshake:157(85885ns) {DropHandshakes:470(2976ns) {DropIncomingHandshake:442(823ns) DropOutgoingHandshake:457(579ns)} GenerateSessionVirtualId:988(1105ns) PrepareNewSessionHandshake:477(2115ns) {SwitchToState:179(624ns)}}}}}
2025-11-28T14:53:33.061258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789528294232285:2077] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.193069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789529143762507:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28267
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.187902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789530463916241:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.066848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789523999264936:2063] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.197506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789522958243035:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19465410ns) {Configure:104(19458585ns) {ProcessConfigured:135(17587963ns) {StartInitialHandshake:157(17583780ns) {DropHandshakes:470(3933ns) {DropIncomingHandshake:442(806ns) DropOutgoingHandshake:457(529ns)} GenerateSessionVirtualId:988(924ns) PrepareNewSessionHandshake:477(8533ns) {SwitchToState:179(2261ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.257042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789526168948944:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(176839349ns) {Configure:104(176834531ns) {ProcessConfigured:135(80789ns) {StartInitialHandshake:157(77953ns) {DropHandshakes:470(2335ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(535ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:477(1670ns) {SwitchToState:179(654ns)}}}}}
2025-11-28T14:53:33.260762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789529143762508:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28267
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.262546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789529143762508:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.262718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789524848795163:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.081439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789528294232265:2068] [node 5] ICP01 ready to work
2025-11-28T14:53:33.197684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789527253210339:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4150
2025-11-28T14:53:33.234738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789527758428066:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21592
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.305008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789530463916254:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54255
2025-11-28T14:53:33.161464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789528294232291:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:33.236819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789523463460692:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(89670024ns) {Configure:104(89664190ns) {ProcessConfigured:135(15633232ns) {StartInitialHandshake:157(15629503ns) {DropHandshakes:470(3455ns) {DropIncomingHandshake:442(946ns) DropOutgoingHandshake:457(398ns)} GenerateSessionVirtualId:988(1212ns) PrepareNewSessionHandshake:477(4123ns) {SwitchToState:179(1113ns)}}}}}
2025-11-28T14:53:33.217171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789527253210339:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.312255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789530463916247:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.312299Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789530463916247:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.281038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789526400409408:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(400201316ns) {Configure:104(400174604ns) {ProcessConfigured:135(94783ns) {StartInitialHandshake:157(91946ns) {DropHandshakes:470(2880ns) {DropIncomingHandshake:442(697ns) DropOutgoingHandshake:457(704ns)} GenerateSessionVirtualId:988(1109ns) PrepareNewSessionHandshake:477(2349ns) {SwitchToState:179(1466ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.328786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789527758428061:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.328827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789527758428061:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.289480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789529143762510:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28267
2025-11-28T14:53:33.256441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789523999264921:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(456841298ns) {Configure:104(456836055ns) {ProcessConfigured:135(15028797ns) {StartInitialHandshake:157(15024219ns) {DropHandshakes:470(2754ns) {DropIncomingHandshake:442(823ns) DropOutgoingHandshake:457(418ns)} GenerateSessionVirtualId:988(2155ns) PrepareNewSessionHandshake:477(6546ns) {SwitchToState:179(3095ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.247135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789522958242996:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(123691250ns) {Configure:104(123684055ns) {ProcessConfigured:135(18263323ns) {StartInitialHandshake:157(18258902ns) {DropHandshakes:470(8426570ns) {DropIncomingHandshake:442(2524ns) DropOutgoingHandshake:457(380ns)} GenerateSessionVirtualId:988(1383ns) PrepareNewSessionHandshake:477(4644ns) {SwitchToState:179(2643ns)}}}}}
2025-11-28T14:53:33.250024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789527253210351:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4150
2025-11-28T14:53:33.262000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789527253210353:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4150
2025-11-28T14:53:33.263140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789527253210353:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.263289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789527253210345:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.263314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789527253210345:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.313897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789527253210345:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:33.314271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789522958242977:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(496151491ns) {Configure:104(484190674ns) {ProcessConfigured:135(19721202ns) {StartInitialHandshake:157(19717260ns) {DropHandshakes:470(1865ns) {DropIncomingHandshake:442(569ns) DropOutgoingHandshake:457(361ns)} GenerateSessionVirtualId:988(849ns) PrepareNewSessionHandshake:477(5509ns) {SwitchToState:179(2662ns)}}}}}
2025-11-28T14:53:33.314412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789527253210351:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.316586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789527253210338:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4150
2025-11-28T14:53:33.328705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789522958242978:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.289879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789528294232291:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.290082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789523999264920:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.333316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789528294232295:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:33.300956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789526400409407:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(368170446ns) {Configure:104(368156667ns) {ProcessConfigured:135(24399943ns) {StartInitialHandshake:157(24395753ns) {DropHandshakes:470(2831ns) {DropIncomingHandshake:442(757ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(773ns) PrepareNewSessionHandshake:477(6313ns) {SwitchToState:179(3196ns)}}}}}
2025-11-28T14:53:33.357828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789527758428055:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.357898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789527758428055:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.322095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789526168948879:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(83321400ns) {Configure:104(83310387ns) {ProcessConfigured:135(84192ns) {StartInitialHandshake:157(81338ns) {DropHandshakes:470(3366ns) {DropIncomingHandshake:442(807ns) DropOutgoingHandshake:457(709ns)} GenerateSessionVirtualId:988(1029ns) PrepareNewSessionHandshake:477(1864ns) {SwitchToState:179(668ns)}}}}}
2025-11-28T14:53:33.304125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789529143762511:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28267
2025-11-28T14:53:33.302272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789530695376769:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
2025-11-28T14:53:33.324754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789529143762507:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.364443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789527758428059:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.322901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789530463916247:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:33.360775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789522958242977:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.302696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789530695376774:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.396691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789530463916245:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.396727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789530463916245:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.397346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789526168948943:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.397446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:33.397509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789526168948943:2096] [node 6] ICP22 created new session: [1:7577789530463916258:2096]
2025-11-28T14:53:33.397579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789530463916258:2096] [node 6] ICS09 handshake done sender: [1:7577789530463916247:2113] self: [1:7577789530463916246:0] peer: [6:7577789528294232294:0] socket: 398 qp: -1
2025-11-28T14:53:33.397623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789530463916258:2096] [node 6] ICS10 traffic start
2025-11-28T14:53:33.397684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.364539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789527758428059:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.410589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789527253210359:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4150
2025-11-28T14:53:33.302830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789530695376775:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
2025-11-28T14:53:33.324938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789524848795136:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.421943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789530463916254:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.357083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789524848795135:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(391900326ns) {Configure:104(391892735ns) {ProcessConfigured:135(11300204ns) {StartInitialHandshake:157(11296495ns) {DropHandshakes:470(2346ns) {DropIncomingHandshake:442(434ns) DropOutgoingHandshake:457(929ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:477(5493ns) {SwitchToState:179(3123ns)}}}}}
2025-11-28T14:53:33.421871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789528294232291:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:33.302948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789530695376776:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
2025-11-28T14:53:33.314386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789530695376769:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.314507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789530695376774:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.314632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789530695376775:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.325444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789526400409471:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(284172765ns) {Configure:104(284166578ns) {ProcessConfigured:135(13619482ns) {StartInitialHandshake:157(13615206ns) {DropHandshakes:470(3355523ns) {DropIncomingHandshake:442(762ns) DropOutgoingHandshake:457(1219ns)} GenerateSessionVirtualId:988(1476ns) PrepareNewSessionHandshake:477(3601ns) {SwitchToState:179(1649ns)}}}}}
2025-11-28T14:53:33.327189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789530695376776:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.327366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789526400409407:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.327442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789526400409408:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.342990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789526400409470:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(58664919ns) {Configure:104(58658553ns) {ProcessConfigured:135(81020ns) {StartInitialHandshake:157(78873ns) {DropHandshakes:470(2645ns) {DropIncomingHandshake:442(639ns) DropOutgoingHandshake:457(494ns)} GenerateSessionVirtualId:988(929ns) PrepareNewSessionHandshake:477(1826ns) {SwitchToState:179(846ns)}}}}}
2025-11-28T14:53:33.343598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789530695376768:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
2025-11-28T14:53:33.421557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789530695376791:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
2025-11-28T14:53:33.426286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789530695376774:2113] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:33.428358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789526400409407:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.432891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789530695376768:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.364656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789527758428068:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21592
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.392894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789529143762508:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:33.409034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789529143762511:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.420939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789524848795137:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.434647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789524848795163:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.455026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789524848795150:2060] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.455059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789524848795150:2060] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.462145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789529143762516:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28267
2025-11-28T14:53:33.462898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789524848795150:2060] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:33.463241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789529143762507:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:33.463671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789524848795136:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.463754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:33.463816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789524848795136:2049] [node 2] ICP22 created new session: [5:7577789529143762520:2049]
2025-11-28T14:53:33.463875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789529143762520:2049] [node 2] ICS09 handshake done sender: [5:7577789529143762507:2114] self: [5:7577789529143762512:0] peer: [2:7577789527758428060:0] socket: 403 qp: -1
2025-11-28T14:53:33.463913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789529143762520:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:33.463958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.417864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789527253210351:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:33.424417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789530463916243:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.503321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789529143762517:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28267
2025-11-28T14:53:33.503556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.388845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789527758428068:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.425341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789528294232296:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:33.520909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789524848795163:2062] [node 4] ICP22 created new session: [5:7577789529143762522:2062]
2025-11-28T14:53:33.522585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789524848795136:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(57844092ns) {HandleHandshakeStatus:329(57840544ns) {DropIncomingHandshake:442(925ns) DropOutgoingHandshake:457(5630ns) SwitchToState:179(509ns) ProcessPendingSessionEvents:495(57361865ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56712654ns max# 27870521ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.424335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789522958243035:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.425676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789530463916245:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.486177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789530695376794:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
2025-11-28T14:53:33.442168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789528294232265:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.389042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789527758428061:2113] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:33.424515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789527253210338:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.426554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789526168948880:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.486863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789530695376775:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:33.525866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789524848795148:2059] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.409283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789527758428055:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.442572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789528294232265:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(65633552ns) {Configure:104(65631958ns) {ProcessConfigured:135(245030ns) {StartInitialHandshake:157(100333ns) {DropHandshakes:470(2991ns) {DropIncomingHandshake:442(754ns) DropOutgoingHandshake:457(417ns)} GenerateSessionVirtualId:988(8520ns) PrepareNewSessionHandshake:477(1766ns) {SwitchToState:179(802ns)}} IncomingHandshake:261(100661ns) {DropIncomingHandshake:442(148ns) PrepareNewSessionHandshake:477(2884ns) {SwitchToState:179(152ns)} GenerateSessionVirtualId:988(145ns) IssueIncomingHandshakeReply:193(68322ns) {SwitchToState:179(240ns)}}}}}
2025-11-28T14:53:33.429061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789522958242978:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.429102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:33.434950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:33.448640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789522958243035:2063] [node 5] ICP22 created new session: [4:7577789527253210360:2063]
2025-11-28T14:53:33.456598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789522958242978:2049] [node 2] ICP22 created new session: [4:7577789527253210361:2049]
2025-11-28T14:53:33.460495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789527253210360:2063] [node 5] ICS09 handshake done sender: [4:7577789527253210345:2114] self: [4:7577789527253210344:0] peer: [5:7577789529143762509:0] socket: 407 qp: -1
2025-11-28T14:53:33.464977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789527253210360:2063] [node 5] ICS10 traffic start
2025-11-28T14:53:33.480533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789527253210364:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4150
2025-11-28T14:53:33.484514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789527253210361:2049] [node 2] ICS09 handshake done sender: [4:7577789527253210351:2116] self: [4:7577789527253210357:0] peer: [2:7577789527758428054:0] socket: 413 qp: -1
2025-11-28T14:53:33.488936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789527253210366:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4150
2025-11-28T14:53:33.493231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:33.496556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789527253210361:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:33.516589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:33.549110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789527253210353:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:33.549357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789522958242977:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.549419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:33.549475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789522958242977:2048] [node 1] ICP22 created new session: [4:7577789527253210368:2048]
2025-11-28T14:53:33.549533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789527253210368:2048] [node 1] ICS09 handshake done sender: [4:7577789527253210353:2117] self: [4:7577789527253210358:0] peer: [1:7577789530463916242:0] socket: 415 qp: -1
2025-11-28T14:53:33.549572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789527253210368:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:33.549610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:33.562755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789527253210350:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.562805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789527253210350:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.592812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789527253210366:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.592991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789522958242979:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.599318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789527253210371:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4150
2025-11-28T14:53:33.599839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789527253210366:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:33.600065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789522958242979:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.600137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:33.600191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789522958242979:2050] [node 3] ICP22 created new session: [4:7577789527253210372:2050]
2025-11-28T14:53:33.600256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789527253210372:2050] [node 3] ICS09 handshake done sender: [4:7577789527253210366:2121] self: [4:7577789527253210370:0] peer: [3:7577789530695376783:0] socket: 426 qp: -1
2025-11-28T14:53:33.600301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789527253210372:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:33.600342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:33.426612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:33.487165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789526400409408:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.544749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:33.412573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789523463460757:2064] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.447414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789523999264920:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789528294232291:2114] held: no
2025-11-28T14:53:33.489941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789528294232300:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:33.490084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789528294232301:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:33.513148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789528294232300:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.523713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789528294232285:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.548697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789523999264936:2063] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:33.568854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789528294232296:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.573335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789528294232265:2068] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.585078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789523999264921:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.598915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789528294232305:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:33.629536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:33.640202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789528294232265:2068] [node 5] ICP22 created new session: [6:7577789528294232306:2068]
2025-11-28T14:53:33.640822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789528294232305:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.645521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789523999264922:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.645623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789528294232307:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:33.544846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789524848795148:2059] [node 6] ICP22 created new session: [5:7577789529143762523:2059]
2025-11-28T14:53:33.424924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789527758428066:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.487224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:33.426671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789526168948880:2049] [node 3] ICP22 created new session: [1:7577789530463916262:2049]
2025-11-28T14:53:33.426733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789530463916262:2049] [node 3] ICS09 handshake done sender: [1:7577789530463916245:2112] self: [1:7577789530463916244:0] peer: [3:7577789530695376786:0] socket: 397 qp: -1
2025-11-28T14:53:33.426773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789530463916262:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:33.426825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:33.434091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789526168948880:2049] [node 3] ICP18 handshake (actor [1:7577789530463916254:2115]) is too old
2025-11-28T14:53:33.434766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789530463916243:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.520795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789530463916243:2111] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:33.536672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789526168948942:2095] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.536769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:33.536835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789526168948942:2095] [node 4] ICP22 created new session: [1:7577789530463916268:2095]
2025-11-28T14:53:33.536912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789530463916268:2095] [node 4] ICS09 handshake done sender: [1:7577789530463916243:2111] self: [1:7577789530463916242:0] peer: [4:7577789527253210358:0] socket: 414 qp: -1
2025-11-28T14:53:33.536961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789530463916268:2095] [node 4] ICS10 traffic start
2025-11-28T14:53:33.537012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:33.540573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789530463916267:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54255
2025-11-28T14:53:33.540881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789530463916257:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.626701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789522958242979:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25612396ns) {HandleHandshakeStatus:329(25608915ns) {DropIncomingHandshake:442(1164ns) DropOutgoingHandshake:457(6216ns) SwitchToState:179(644ns) ProcessPendingSessionEvents:495(20617580ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20550642ns max# 20005410ns}}}
2025-11-28T14:53:33.540921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789530463916257:2116] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.487271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789526400409408:2049] [node 2] ICP22 created new session: [3:7577789530695376796:2049]
2025-11-28T14:53:33.544912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789529143762523:2059] [node 6] ICS09 handshake done sender: [5:7577789524848795150:2060] self: [5:7577789524848795149:0] peer: [6:7577789528294232299:0] socket: 384 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:33.557354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789530463916266:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54255
2025-11-28T14:53:33.673125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789528294232309:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:33.448538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:33.682507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789527253210350:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:33.544959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789529143762523:2059] [node 6] ICS10 traffic start
2025-11-28T14:53:33.487323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789530695376796:2049] [node 2] ICS09 handshake done sender: [3:7577789530695376775:2114] self: [3:7577789530695376787:0] peer: [2:7577789527758428058:0] socket: 419 qp: -1
2025-11-28T14:53:33.563601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789530463916267:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.454558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789527758428074:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21592
2025-11-28T14:53:33.673212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789528294232310:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53873
2025-11-28T14:53:33.733346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789522958242996:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.545037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:33.563698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789530463916257:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:33.487370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789530695376796:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:33.733473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:33.733539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789522958242996:2062] [node 6] ICP22 created new session: [4:7577789527253210375:2062]
2025-11-28T14:53:33.733599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789527253210375:2062] [node 6] ICS09 handshake done sender: [4:7577789527253210350:2115] self: [4:7577789527253210349:0] peer: [6:7577789528294232303:0] socket: 440 qp: -1
2025-11-28T14:53:33.733648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789527253210375:2062] [node 6] ICS10 traffic start
2025-11-28T14:53:33.733720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:33.682286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789528294232306:2068] [node 5] ICS09 handshake done sender: [6:7577789523999264936:2063] self: [6:7577789528294232299:0] peer: [5:7577789524848795149:0] socket: 385 qp: -1
2025-11-28T14:53:33.468485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789523463460757:2064] [node 5] ICP22 created new session: [2:7577789527758428073:2064]
2025-11-28T14:53:33.546833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789524848795148:2059] [node 6] ICP35 event processing took too much time PendingConnection:279(19973106ns) {HandleHandshakeStatus:329(19969724ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(680ns) SwitchToState:179(342ns) ProcessPendingSessionEvents:495(626430ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 563046ns max# 65780ns}}}
2025-11-28T14:53:33.692515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789528294232306:2068] [node 5] ICS10 traffic start
2025-11-28T14:53:33.468777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789527758428059:2112] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:33.577635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789530463916253:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.487406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:33.489419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789523463460756:2065] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.490550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:33.490619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789523463460756:2065] [node 4] ICP22 created new session: [2:7577789527758428078:2065]
2025-11-28T14:53:33.490678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789527758428078:2065] [node 4] ICS09 handshake done sender: [2:7577789527758428055:2110] self: [2:7577789527758428054:0] peer: [4:7577789527253210357:0] socket: 410 qp: -1
2025-11-28T14:53:33.490713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789527758428078:2065] [node 4] ICS10 traffic start
2025-11-28T14:53:33.490756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:33.492016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789523463460693:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.492119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:33.492177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789523463460693:2049] [node 3] ICP22 created new session: [2:7577789527758428080:2049]
2025-11-28T14:53:33.492236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789527758428080:2049] [node 3] ICS09 handshake done sender: [2:7577789527758428059:2112] self: [2:7577789527758428058:0] peer: [3:7577789530695376787:0] socket: 401 qp: -1
2025-11-28T14:53:33.492275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789527758428080:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:33.492309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:33.500479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789527758428073:2064] [node 5] ICS09 handshake done sender: [2:7577789527758428061:2113] self: [2:7577789527758428060:0] peer: [5:7577789529143762512:0] socket: 402 qp: -1
2025-11-28T14:53:33.500602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789527758428073:2064] [node 5] ICS10 traffic start
2025-11-28T14:53:33.500665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:33.502560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789523463460757:2064] [node 5] ICP35 event processing took too much time PendingConnection:279(100713985ns) {HandleHandshakeStatus:329(100711184ns) {DropIncomingHandshake:442(1989ns) DropOutgoingHandshake:457(4104ns) SwitchToState:179(783ns) ProcessPendingSessionEvents:495(650264ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 582120ns max# 75144ns}}}
2025-11-28T14:53:33.502802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789527758428074:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.510182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789523463460693:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17105185ns) {HandleHandshakeStatus:329(17102266ns) {DropIncomingHandshake:442(14691ns) DropOutgoingHandshake:457(318ns) SwitchToState:179(434ns) ProcessPendingSessionEvents:495(16676368ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16593483ns max# 16073434ns}}}
2025-11-28T14:53:33.524909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789523463460692:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.525032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789527758428083:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21592
2025-11-28T14:53:33.577142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789527758428086:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21592
2025-11-28T14:53:33.579897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789527758428074:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:33.604868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789523463460692:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.627737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789527758428057:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.627772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789527758428057:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.636890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:33.640972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789523463460692:2048] [node 1] ICP22 created new session: [2:7577789527758428087:2048]
2025-11-28T14:53:33.653597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789527758428057:2111] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:33.653791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789527758428087:2048] [node 1] ICS09 handshake done sender: [2:7577789527758428074:2117] self: [2:7577789527758428084:0] peer: [1:7577789530463916256:0] socket: 444 qp: -1
2025-11-28T14:53:33.673674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789527758428087:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:33.707026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:33.721774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789523463460755:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.721860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:33.721923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789523463460755:2066] [node 6] ICP22 created new session: [2:7577789527758428090:2066]
2025-11-28T14:53:33.721981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789527758428090:2066] [node 6] ICS09 handshake done sender: [2:7577789527758428057:2111] self: [2:7577789527758428056:0] peer: [6:7577789528294232304:0] socket: 435 qp: -1
2025-11-28T14:53:33.722029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789527758428090:2066] [node 6] ICS10 traffic start
2025-11-28T14:53:33.722072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:33.732862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789527758428085:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21592
2025-11-28T14:53:33.737350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789527758428085:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.749048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789523463460755:2066] [node 6] ICP18 handshake (actor [2:7577789527758428085:2119]) is too old
2025-11-28T14:53:33.793667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789523463460692:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(180050512ns) {HandleHandshakeStatus:329(180044359ns) {DropIncomingHandshake:442(3253ns) DropOutgoingHandshake:457(11588ns) SwitchToState:179(491ns) ProcessPendingSessionEvents:495(18317141ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18245613ns max# 17791246ns}}}
2025-11-28T14:53:33.547174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789529143762517:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.492516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789530695376778:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.706867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:33.577665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789530463916253:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.492568Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789530695376778:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.492722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:33.492825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789526400409470:2063] [node 6] ICP35 event processing took too much time PendingConnection:279(20031231ns) {IncomingHandshake:261(9121448ns) {DropIncomingHandshake:442(919ns) PrepareNewSessionHandshake:477(2361ns) {SwitchToState:179(1783ns)} GenerateSessionVirtualId:988(756ns) IssueIncomingHandshakeReply:193(6276ns)}}
2025-11-28T14:53:33.528748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789530695376778:2116] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:33.544467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789526400409407:2048] [node 1] ICP22 created new session: [3:7577789530695376793:2048]
2025-11-28T14:53:33.576568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789530695376793:2048] [node 1] ICS09 handshake done sender: [3:7577789530695376774:2113] self: [3:7577789530695376786:0] peer: [1:7577789530463916244:0] socket: 418 qp: -1
2025-11-28T14:53:33.576624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789530695376793:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:33.586904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:33.590268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789530695376770:2070] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.590348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:33.590403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789530695376770:2070] [node 5] ICP22 created new session: [3:7577789530695376800:2070]
2025-11-28T14:53:33.590458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789530695376800:2070] [node 5] ICS09 handshake done sender: [3:7577789530695376778:2116] self: [3:7577789530695376777:0] peer: [5:7577789529143762515:0] socket: 422 qp: -1
2025-11-28T14:53:33.590494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789530695376800:2070] [node 5] ICS10 traffic start
2025-11-28T14:53:33.590528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:33.597507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789530695376784:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.597540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789530695376784:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.604955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789530695376784:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:33.640545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789526400409471:2060] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.666417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789530695376789:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:33.666449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789530695376789:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:33.553723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789529143762524:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28267
2025-11-28T14:53:33.737270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789528294232300:2119] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:33.752650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789528294232296:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:33.754248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789523999264921:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.754353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:33.754416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789523999264921:2049] [node 2] ICP22 created new session: [6:7577789528294232312:2049]
2025-11-28T14:53:33.754470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789528294232312:2049] [node 2] ICS09 handshake done sender: [6:7577789528294232296:2117] self: [6:7577789528294232304:0] peer: [2:7577789527758428056:0] socket: 393 qp: -1
2025-11-28T14:53:33.754508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789528294232312:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:33.754560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:33.765522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789523999264920:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(227996893ns) {HandleHandshakeStatus:395(227991011ns) {LogHandshakeFail:472(10864278ns) {UpdateErrorStateLog:545(8046ns)} DropOutgoingHandshake:457(608ns) ProcessPendingSessionEvents:495(217032125ns) {unordered DropSessionEvent:512 num# 100 sum# 216924616ns max# 29850056ns}}}
2025-11-28T14:53:33.765586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789528294232285:2077] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.765664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:33.765721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789528294232285:2077] [node 4] ICP22 created new session: [6:7577789528294232314:2077]
2025-11-28T14:53:33.765773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789528294232314:2077] [node 4] ICS09 handshake done sender: [6:7577789528294232300:2119] self: [6:7577789528294232303:0] peer: [4:7577789527253210349:0] socket: 442 qp: -1
2025-11-28T14:53:33.765814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789528294232314:2077] [node 4] ICS10 traffic start
2025-11-28T14:53:33.765861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:33.772988Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789528294232285:2077] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:33.774448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789528294232305:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:33.805314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789528294232265:2068] [node 5] ICP35 event processing took too much time PendingConnection:279(223759252ns) {HandleHandshakeStatus:329(223755994ns) {DropIncomingHandshake:442(950ns) DropOutgoingHandshake:457(10762ns) SwitchToState:179(541ns) ProcessPendingSessionEvents:495(20187136ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20116406ns max# 19594976ns}}}
2025-11-28T14:53:33.805389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789523999264920:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.805435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:33.805489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789523999264920:2048] [node 1] ICP22 created new session: [6:7577789528294232316:2048]
2025-11-28T14:53:33.805537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789528294232316:2048] [node 1] ICS09 handshake done sender: [6:7577789528294232291:2114] self: [6:7577789528294232294:0] peer: [1:7577789530463916246:0] socket: 400 qp: -1
2025-11-28T14:53:33.805579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789528294232316:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:33.805631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:33.810571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789523999264922:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.810651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:33.810706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789523999264922:2050] [node 3] ICP22 created new session: [6:7577789528294232318:2050]
2025-11-28T14:53:33.810756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789528294232318:2050] [node 3] ICS09 handshake done sender: [6:7577789528294232305:2121] self: [6:7577789528294232308:0] peer: [3:7577789530695376788:0] socket: 394 qp: -1
2025-11-28T14:53:33.810793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789528294232318:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:33.810851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:33.815501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789523999264922:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:33.578642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789530463916271:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54255
2025-11-28T14:53:33.670532Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789526400409407:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(241142006ns) {HandleHandshakeStatus:329(241139972ns) {DropIncomingHandshake:442(780ns) DropOutgoingHandshake:457(6862ns) SwitchToState:179(985ns) ProcessPendingSessionEvents:495(28995774ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28927902ns max# 28460896ns}}}
2025-11-28T14:53:33.680154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:33.688605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789526400409471:2060] [node 4] ICP22 created new session: [3:7577789530695376805:2060]
2025-11-28T14:53:33.688676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789530695376805:2060] [node 4] ICS09 handshake done sender: [3:7577789530695376784:2118] self: [3:7577789530695376783:0] peer: [4:7577789527253210370:0] socket: 423 qp: -1
2025-11-28T14:53:33.688734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789530695376805:2060] [node 4] ICS10 traffic start
2025-11-28T14:53:33.688806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:33.690529Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789526400409471:2060] [node 4] ICP35 event processing took too much time PendingConnection:279(49049371ns) {HandleHandshakeStatus:329(49045990ns) {DropIncomingHandshake:442(14818ns) DropOutgoingHandshake:457(588ns) SwitchToState:179(1159ns) ProcessPendingSessionEvents:495(591728ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 523391ns max# 58110ns}}}
2025-11-28T14:53:33.692875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789530695376789:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:33.736173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789526400409470:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.736260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:33.736315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789526400409470:2063] [node 6] ICP22 created new session: [3:7577789530695376807:2063]
2025-11-28T14:53:33.736393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789530695376807:2063] [node 6] ICS09 handshake done sender: [3:7577789530695376789:2119] self: [3:7577789530695376788:0] peer: [6:7577789528294232308:0] socket: 455 qp: -1
2025-11-28T14:53:33.736433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789530695376807:2063] [node 6] ICS10 traffic start
2025-11-28T14:53:33.736469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:33.762297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789526400409470:2063] [node 6] ICP35 event processing took too much time PendingConnection:279(25213569ns) {HandleHandshakeStatus:329(25210689ns) {DropIncomingHandshake:442(15031ns) DropOutgoingHandshake:457(379ns) SwitchToState:179(633ns) ProcessPendingSessionEvents:495(24778262ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24699405ns max# 11538391ns}}}
2025-11-28T14:53:33.579512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789530463916253:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:33.580546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789526168948944:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.588611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:33.588694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789526168948944:2097] [node 5] ICP22 created new session: [1:7577789530463916273:2097]
2025-11-28T14:53:33.588760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789530463916273:2097] [node 5] ICS09 handshake done sender: [1:7577789530463916253:2114] self: [1:7577789530463916252:0] peer: [5:7577789529143762527:0] socket: 437 qp: -1
2025-11-28T14:53:33.588806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789530463916273:2097] [node 5] ICS10 traffic start
2025-11-28T14:53:33.588870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:33.644614Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789526168948879:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(32134076ns) {IncomingHandshake:261(12173602ns) {DropIncomingHandshake:442(571ns) PrepareNewSessionHandshake:477(2162ns) {SwitchToState:179(990ns)} GenerateSessionVirtualId:988(775ns) IssueIncomingHandshakeReply:193(11942ns)}}
2025-11-28T14:53:33.652563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789526168948879:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:33.664844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:33.664911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789526168948879:2048] [node 2] ICP22 created new session: [1:7577789530463916276:2048]
2025-11-28T14:53:33.664973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789530463916276:2048] [node 2] ICS09 handshake done sender: [1:7577789530463916257:2116] self: [1:7577789530463916256:0] peer: [2:7577789527758428084:0] socket: 438 qp: -1
2025-11-28T14:53:33.665031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789530463916276:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:33.665102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:33.668105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789526168948879:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(21146203ns) {HandleHandshakeStatus:329(21143267ns) {DropIncomingHandshake:442(33540ns) DropOutgoingHandshake:457(439ns) SwitchToState:179(522ns) ProcessPendingSessionEvents:495(601566ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 534188ns max# 51115ns}}}
2025-11-28T14:53:33.749105Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789526168948944:2097] [node 5] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:33.850016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789526168948942:2095] [node 4] ICP35 event processing took too much time PendingConnection:279(296917093ns) {HandleHandshakeStatus:329(296913298ns) {DropIncomingHandshake:442(481ns) DropOutgoingHandshake:457(1057ns) SwitchToState:179(642ns) ProcessPendingSessionEvents:495(296322732ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 296036279ns max# 19917612ns}}}
2025-11-28T14:53:33.851363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577789526168948942:2095] [node 4] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:33.556559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789529143762522:2062] [node 4] ICS09 handshake done sender: [5:7577789529143762508:2115] self: [5:7577789529143762509:0] peer: [4:7577789527253210344:0] socket: 408 qp: -1
2025-11-28T14:53:33.857360Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789522958243035:2063] [node 5] ICP35 event processing took too much time PendingConnection:279(431998393ns) {HandleHandshakeStatus:329(431994621ns) {DropIncomingHandshake:442(505ns) DropOutgoingHandshake:457(3296ns) SwitchToState:179(337ns) ProcessPendingSessionEvents:495(318606577ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 318391532ns max# 24825324ns}}}
2025-11-28T14:53:33.857464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577789522958243035:2063] [node 5] ICP18 handshake (actor [4:7577789527253210338:2110]) is too old
2025-11-28T14:53:33.860708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789522958242978:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(430564238ns) {HandleHandshakeStatus:329(430561573ns) {DropIncomingHandshake:442(354ns) DropOutgoingHandshake:457(4685ns) SwitchToState:179(86ns) ProcessPendingSessionEvents:495(302922080ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 271827442ns max# 31987251ns}}}
2025-11-28T14:53:33.557625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789524848795135:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:33.557817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789529143762522:2062] [node 4] ICS10 traffic start
2025-11-28T14:53:33.563887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:33.596741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789529143762529:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28267
2025-11-28T14:53:33.604732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789524848795137:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17846830ns)
2025-11-28T14:53:33.636709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789529143762511:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:33.637071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789524848795137:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789529143762511:2117] held: no
2025-11-28T14:53:33.638176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789529143762517:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:33.698029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789524848795163:2062] [node 4] ICP35 event processing took too much time PendingConnection:279(275197708ns) {HandleHandshakeStatus:329(275194829ns) {DropIncomingHandshake:442(5031ns) DropOutgoingHandshake:457(17550ns) SwitchToState:179(4136ns) ProcessPendingSessionEvents:495(66385570ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 66289132ns max# 17818367ns}}}
2025-11-28T14:53:33.700151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789524848795135:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.700255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:33.700312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789524848795135:2048] [node 1] ICP22 created new session: [5:7577789529143762530:2048]
2025-11-28T14:53:33.700378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789529143762530:2048] [node 1] ICS09 handshake done sender: [5:7577789529143762517:2120] self: [5:7577789529143762527:0] peer: [1:7577789530463916252:0] socket: 439 qp: -1
2025-11-28T14:53:33.700428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789529143762530:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:33.700488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:33.702568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789524848795137:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:33.702642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:33.702692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789524848795137:2050] [node 3] ICP22 created new session: [5:7577789529143762532:2050]
2025-11-28T14:53:33.702748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789529143762532:2050] [node 3] ICS09 handshake done sender: [5:7577789529143762511:2117] self: [5:7577789529143762515:0] peer: [3:7577789530695376777:0] socket: 424 qp: -1
2025-11-28T14:53:33.702780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789529143762532:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:33.702813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:33.954804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:33.955076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789527758428087:2048] [node 1] ICS07 socket disconnect 444 reason# EndOfStream
2025-11-28T14:53:33.955110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789527758428087:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:33.955244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:33.955293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789527758428087:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:33.960560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:33.960845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789530695376793:2048] [node 1] ICS07 socket disconnect 418 reason# EndOfStream
2025-11-28T14:53:33.960894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789530695376793:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:33.961012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:33.961059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789530695376793:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:33.968901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789526400409407:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:33.968982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789530695376793:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:33.969023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789526400409407:2048] [node 1] ICP30 unregister session Session# [3:7577789530695376793:2048] VirtualId# [3:7577789530695376786:0]
2025-11-28T14:53:33.969154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:33.969188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789526400409407:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:33.969204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:33.968080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789523463460692:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:33.968144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789527758428087:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:33.969887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789523463460692:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:33.969942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789527758428087:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:33.969989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789523463460692:2048] [node 1] ICP30 unregister session Session# [2:7577789527758428087:2048] VirtualId# [2:7577789527758428084:0]
2025-11-28T14:53:33.970092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:33.970123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789523463460692:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:33.970138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:33.979839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789526400409407:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:33.979976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:33.980213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789527253210368:2048] [node 1] ICS01 socket: 415 reason# EndOfStream
2025-11-28T14:53:33.980264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789522958242977:2048] [node 1] ICP30 unregister session Session# [4:7577789527253210368:2048] VirtualId# [4:7577789527253210358:0]
2025-11-28T14:53:33.980299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789527253210368:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:33.980721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:33.980796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:33.980628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:33.980823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789529143762530:2048] [node 1] ICS07 socket disconnect 439 reason# EndOfStream
2025-11-28T14:53:33.980851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789529143762530:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:33.980919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:33.980972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789529143762530:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:33.981237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:33.981430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789528294232316:2048] [node 1] ICS01 socket: 400 reason# EndOfStream
2025-11-28T14:53:33.981470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789523999264920:2048] [node 1] ICP30 unregister session Session# [6:7577789528294232316:2048] VirtualId# [6:7577789528294232294:0]
2025-11-28T14:53:33.981501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789528294232316:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:33.987442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577789523463460692:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:33.981566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:33.982324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:34.008565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789524848795135:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:34.008649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789529143762530:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:34.008691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789524848795135:2048] [node 1] ICP30 unregister session Session# [5:7577789529143762530:2048] VirtualId# [5:7577789529143762527:0]
2025-11-28T14:53:34.008824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:34.008868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789524848795135:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.008885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.020570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789524848795135:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:34.308631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:34.308899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789530695376796:2049] [node 2] ICS01 socket: 419 reason# EndOfStream
2025-11-28T14:53:34.308955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789526400409408:2049] [node 2] ICP30 unregister session Session# [3:7577789530695376796:2049] VirtualId# [3:7577789530695376787:0]
2025-11-28T14:53:34.309003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789530695376796:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.309145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:34.309202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:34.334410Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:34.344457Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:34.344774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789528294232312:2049] [node 2] ICS07 socket disconnect 393 reason# EndOfStream
2025-11-28T14:53:34.344811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789528294232312:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.344933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:34.344979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789528294232312:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:34.341219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789527253210361:2049] [node 2] ICS07 socket disconnect 413 reason# EndOfStream
2025-11-28T14:53:34.320764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:34.321026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789529143762520:2049] [node 2] ICS07 socket disconnect 403 reason# EndOfStream
2025-11-28T14:53:34.321085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789529143762520:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.321238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:34.321290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789529143762520:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:34.364546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789524848795136:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:34.364623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789529143762520:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:34.364664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789524848795136:2049] [node 2] ICP30 unregister session Session# [5:7577789529143762520:2049] VirtualId# [5:7577789529143762512:0]
2025-11-28T14:53:34.364816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:34.364857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789524848795136:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.364911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.341277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789527253210361:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.341374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:34.341430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789527253210361:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:34.367009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789522958242978:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:34.367073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789527253210361:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:34.367113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789522958242978:2049] [node 2] ICP30 unregister session Session# [4:7577789527253210361:2049] VirtualId# [4:7577789527253210357:0]
2025-11-28T14:53:34.367275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:34.367313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789522958242978:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.367331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.370324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789523999264921:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:34.370399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789528294232312:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:34.370447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789523999264921:2049] [node 2] ICP30 unregister session Session# [6:7577789528294232312:2049] VirtualId# [6:7577789528294232304:0]
2025-11-28T14:53:34.370561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:34.370593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789523999264921:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.370609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.376545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789524848795136:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:34.377001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789522958242978:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:34.381344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789523999264921:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:34.424628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:34.424853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789527253210372:2050] [node 3] ICS01 socket: 426 reason# EndOfStream
2025-11-28T14:53:34.424901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789522958242979:2050] [node 3] ICP30 unregister session Session# [4:7577789527253210372:2050] VirtualId# [4:7577789527253210370:0]
2025-11-28T14:53:34.424938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789527253210372:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.425015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:34.425059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:34.425173Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:34.425373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789529143762532:2050] [node 3] ICS01 socket: 424 reason# EndOfStream
2025-11-28T14:53:34.425411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789524848795137:2050] [node 3] ICP30 unregister session Session# [5:7577789529143762532:2050] VirtualId# [5:7577789529143762515:0]
2025-11-28T14:53:34.425440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789529143762532:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.425497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:34.425537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:34.423361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:34.423706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789528294232318:2050] [node 3] ICS01 socket: 394 reason# EndOfStream
2025-11-28T14:53:34.423808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789523999264922:2050] [node 3] ICP30 unregister session Session# [6:7577789528294232318:2050] VirtualId# [6:7577789528294232308:0]
2025-11-28T14:53:34.423896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789528294232318:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.424113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:34.424228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:34.692608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:34.692865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789528294232314:2077] [node 4] ICS01 socket: 442 reason# EndOfStream
2025-11-28T14:53:34.692925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789528294232285:2077] [node 4] ICP30 unregister session Session# [6:7577789528294232314:2077] VirtualId# [6:7577789528294232303:0]
2025-11-28T14:53:34.692964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789528294232314:2077] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.693078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:34.693134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:34.693233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:34.693440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789529143762522:2062] [node 4] ICS01 socket: 408 reason# EndOfStream
2025-11-28T14:53:34.693476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789524848795163:2062] [node 4] ICP30 unregister session Session# [5:7577789529143762522:2062] VirtualId# [5:7577789529143762509:0]
2025-11-28T14:53:34.693516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789529143762522:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.693581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:34.693620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:34.872560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:34.872831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789528294232306:2068] [node 5] ICS07 socket disconnect 385 reason# EndOfStream
2025-11-28T14:53:34.872868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789528294232306:2068] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:34.872915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:34.873043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789528294232306:2068] [node 5] ICS15 start handshake
2025-11-28T14:53:34.884250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789528294232265:2068] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27502) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:34.884318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789528294232306:2068] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:34.884384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789528294232265:2068] [node 5] ICP30 unregister session Session# [6:7577789528294232306:2068] VirtualId# [6:7577789528294232299:0]
2025-11-28T14:53:34.884521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-28T14:53:34.884552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789528294232265:2068] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27502) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.884568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27502) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:34.897241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789528294232265:2068] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:35.259944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789537922856691:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:35.259998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789537922856692:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:35.440157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789536612097436:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:35.440212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789536612097437:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:35.543672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789536507311160:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:35.543732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789536507311161:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:35.668904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789538793346425:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:35.668960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789538793346426:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:35.669003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789538793346427:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:35.796537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789539075512795:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:35.796612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789539075512796:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:35.796647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789539075512797:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:35.938880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789537907954504:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:35.938938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789537907954505:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:35.938967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789537907954506:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:36.193386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789543088313740:2063] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.211104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789540907064768:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26473
2025-11-28T14:53:36.196631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921820:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.216514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789540907064760:2067] [node 5] ICP01 ready to work
2025-11-28T14:53:36.200795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789540802278479:2066] [node 6] ICP01 ready to work
2025-11-28T14:53:36.273104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789542202921820:2064] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.228896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789540802278481:2060] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.323789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789542202921829:2071] [node 5] ICP01 ready to work
2025-11-28T14:53:36.311218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789536507311160:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(110528216ns) {Configure:104(110519697ns) {ProcessConfigured:135(99945ns) {StartInitialHandshake:157(95840ns) {DropHandshakes:470(2840ns) {DropIncomingHandshake:442(679ns) DropOutgoingHandshake:457(502ns)} GenerateSessionVirtualId:988(1319ns) PrepareNewSessionHandshake:477(2422ns) {SwitchToState:179(1223ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.258199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789542217824043:2075] [node 6] ICP01 ready to work
2025-11-28T14:53:36.312108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789540802278487:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-28T14:53:36.318858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789543088313746:2066] [node 5] ICP01 ready to work
2025-11-28T14:53:36.258730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789540907064777:2066] [node 4] ICP01 ready to work
2025-11-28T14:53:36.325243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789543370480107:2060] [node 6] ICP01 ready to work
2025-11-28T14:53:36.431684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789543370480118:2072] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.441236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789543088313780:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53324
2025-11-28T14:53:36.385754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789542217824054:2073] [node 4] ICP01 ready to work
2025-11-28T14:53:36.292597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789540907064768:2082] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.312188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789540802278488:2076] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.300845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789540907064780:2069] [node 6] ICP01 ready to work
2025-11-28T14:53:36.389303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789536612097436:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(140326687ns) {Configure:104(140321792ns) {ProcessConfigured:135(115773ns) {StartInitialHandshake:157(111903ns) {DropHandshakes:470(3100ns) {DropIncomingHandshake:442(902ns) DropOutgoingHandshake:457(410ns)} GenerateSessionVirtualId:988(1432ns) PrepareNewSessionHandshake:477(2392ns) {SwitchToState:179(921ns)}}}}}
2025-11-28T14:53:36.414331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789542217824050:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5898
2025-11-28T14:53:36.498941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789543088313746:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(179703959ns) {Configure:104(179699084ns) {ProcessConfigured:135(24200667ns) {StartInitialHandshake:157(24197621ns) {DropHandshakes:470(2927ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(468ns)} GenerateSessionVirtualId:988(515ns) PrepareNewSessionHandshake:477(4276ns) {SwitchToState:179(1538ns)}}}}}
2025-11-28T14:53:36.381783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789540802278498:2064] [node 4] ICP01 ready to work
2025-11-28T14:53:36.480102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789540907064806:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26473
2025-11-28T14:53:36.505311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789540907064777:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(107184708ns) {Configure:104(107180260ns) {ProcessConfigured:135(5429977ns) {StartInitialHandshake:157(5424014ns) {DropHandshakes:470(3212ns) {DropIncomingHandshake:442(734ns) DropOutgoingHandshake:457(423ns)} GenerateSessionVirtualId:988(846ns) PrepareNewSessionHandshake:477(3283ns) {SwitchToState:179(1796ns)}}}}}
2025-11-28T14:53:36.444602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789542217824051:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5898
2025-11-28T14:53:36.479301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789540802278527:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-28T14:53:36.543720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789543370480107:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(39760895ns) {Configure:104(39753807ns) {ProcessConfigured:135(19090482ns) {StartInitialHandshake:157(19087263ns) {DropHandshakes:470(2354ns) {DropIncomingHandshake:442(722ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(892ns) PrepareNewSessionHandshake:477(5127395ns) {SwitchToState:179(7248ns)}}}}}
2025-11-28T14:53:36.445321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789542217824055:2074] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.480811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789540802278527:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.469063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824050:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.533923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789537907954505:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(211949796ns) {Configure:104(211944057ns) {ProcessConfigured:135(36790269ns) {StartInitialHandshake:157(36786916ns) {DropHandshakes:470(1509ns) {DropIncomingHandshake:442(387ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(801ns) PrepareNewSessionHandshake:477(4516ns) {SwitchToState:179(2569ns)}}}}}
2025-11-28T14:53:36.551874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789538793346425:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(395305124ns) {Configure:104(395299444ns) {ProcessConfigured:135(22460958ns) {StartInitialHandshake:157(22458006ns) {DropHandshakes:470(2162ns) {DropIncomingHandshake:442(488ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(1002ns) PrepareNewSessionHandshake:477(3971ns) {SwitchToState:179(1678ns)}}}}}
2025-11-28T14:53:36.548590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789540907064806:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.553017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789542202921874:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.609510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789543370480161:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37711
2025-11-28T14:53:36.609762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789543370480162:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37711
2025-11-28T14:53:36.557095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789543088313742:2064] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.535125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789540802278487:2075] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.545896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824051:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.576791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789536612097436:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.555252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789542202921874:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.594040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921879:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.604756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789542202921820:2064] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:36.610101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789542202921829:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(285952275ns) {Configure:104(285947950ns) {ProcessConfigured:135(23836580ns) {StartInitialHandshake:157(23832651ns) {DropHandshakes:470(2422ns) {DropIncomingHandshake:442(615ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(704ns) PrepareNewSessionHandshake:477(4168ns) {SwitchToState:179(2716ns)}}}}}
2025-11-28T14:53:36.616821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921881:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.617394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789543370480162:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.637774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789542202921881:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.637914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789542202921874:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.541365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789540802278488:2076] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.557125Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789543088313742:2064] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.638241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789537907954506:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.580443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789540907064811:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26473
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.566013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789542217824055:2074] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20875799ns) {Configure:104(20868733ns) {ProcessConfigured:135(85457ns) {StartInitialHandshake:157(82577ns) {DropHandshakes:470(6139ns) {DropIncomingHandshake:442(676ns) DropOutgoingHandshake:457(505ns)} GenerateSessionVirtualId:988(593ns) PrepareNewSessionHandshake:477(1220ns) {SwitchToState:179(486ns)}}}}}
2025-11-28T14:53:36.598031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789540802278479:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(121293395ns) {Configure:104(112882761ns) {ProcessConfigured:135(63428ns) {StartInitialHandshake:157(60916ns) {DropHandshakes:470(2565ns) {DropIncomingHandshake:442(660ns) DropOutgoingHandshake:457(499ns)} GenerateSessionVirtualId:988(644ns) PrepareNewSessionHandshake:477(1663ns) {SwitchToState:179(974ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.573637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789543088313800:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53324
2025-11-28T14:53:36.617436Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789540907064780:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(272131298ns) {Configure:104(272125570ns) {ProcessConfigured:135(29121406ns) {StartInitialHandshake:157(29117458ns) {DropHandshakes:470(2727ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(571ns)} GenerateSessionVirtualId:988(630ns) PrepareNewSessionHandshake:477(2415ns) {SwitchToState:179(1184ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.643333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789543370480161:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.638335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921882:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.636962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789540802278533:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-28T14:53:36.588480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789543088313742:2064] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.578862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824053:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.636633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789536612097436:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25114062ns) {IncomingHandshake:261(12240808ns) {DropIncomingHandshake:442(1605ns) PrepareNewSessionHandshake:477(2870ns) {SwitchToState:179(1470ns)} GenerateSessionVirtualId:988(929ns) IssueIncomingHandshakeReply:193(77568ns) {SwitchToState:179(604ns)}}}
2025-11-28T14:53:36.644749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:36.669947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789539075512797:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(199789940ns) {Configure:104(199784770ns) {ProcessConfigured:135(11337356ns) {StartInitialHandshake:157(11334682ns) {DropHandshakes:470(1775ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:477(1337ns) {SwitchToState:179(88ns)}}}}}
2025-11-28T14:53:36.578894Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789542217824053:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.604599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789543088313780:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.680380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789539075512796:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(468609483ns) {Configure:104(468602324ns) {ProcessConfigured:135(12246408ns) {StartInitialHandshake:157(12243312ns) {DropHandshakes:470(2838ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(482ns)} GenerateSessionVirtualId:988(822ns) PrepareNewSessionHandshake:477(4588ns) {SwitchToState:179(2062ns)}}}}}
2025-11-28T14:53:36.697016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789543370480171:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37711
2025-11-28T14:53:36.697127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789543370480172:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37711
2025-11-28T14:53:36.724391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789543370480172:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.744452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789539075512795:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(313936810ns) {Configure:104(313930690ns) {ProcessConfigured:135(23569663ns) {StartInitialHandshake:157(23565418ns) {DropHandshakes:470(1519ns) {DropIncomingHandshake:442(256ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(487ns) PrepareNewSessionHandshake:477(3478ns) {SwitchToState:179(1828ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.644708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789540802278529:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.668530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789540907064811:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.644817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789542202921874:2068] [node 4] ICP22 created new session: [6:7577789542202921884:2068]
2025-11-28T14:53:36.784515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789543370480171:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.632982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789543088313800:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.644737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789540802278529:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.720741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789540907064808:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.591015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789542217824053:2111] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:36.668772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789542202921884:2068] [node 4] ICS09 handshake done sender: [6:7577789542202921820:2064] self: [6:7577789542202921877:0] peer: [4:7577789543088313741:0] socket: 397 qp: -1
2025-11-28T14:53:36.788583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789543370480118:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.645451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789540802278533:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.591402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789537922856692:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(391136185ns) {Configure:104(391125729ns) {ProcessConfigured:135(16590923ns) {StartInitialHandshake:157(16587885ns) {DropHandshakes:470(2543ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(438ns)} GenerateSessionVirtualId:988(798ns) PrepareNewSessionHandshake:477(3267ns) {SwitchToState:179(1543ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.636549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789543088313740:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.720785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789540907064808:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.668850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789542202921884:2068] [node 4] ICS10 traffic start
2025-11-28T14:53:36.645762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789536507311160:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.663524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789543088313803:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53324
2025-11-28T14:53:36.812527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789543370480118:2072] [node 4] ICP35 event processing took too much time PendingConnection:279(64021542ns) {IncomingHandshake:261(39113791ns) {DropIncomingHandshake:442(462ns) PrepareNewSessionHandshake:477(3040ns) {SwitchToState:179(1484ns)} GenerateSessionVirtualId:988(3578ns) IssueIncomingHandshakeReply:193(15521604ns) {SwitchToState:179(2310ns)}}}
2025-11-28T14:53:36.606387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789537922856691:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.764707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789540907064808:2115] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:36.668948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.646252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789540802278529:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.672118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:36.644589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:36.765494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789540907064777:2066] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.672445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789542202921882:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.820571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789543370480159:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.646549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789540802278479:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.660664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824064:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.679975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789543088313740:2063] [node 6] ICP22 created new session: [4:7577789543088313805:2063]
2025-11-28T14:53:36.686137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789543088313803:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.686281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789538793346427:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.706081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789538793346426:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.706315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789538793346426:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(48879840ns) {IncomingHandshake:261(35325050ns) {DropIncomingHandshake:442(812ns) PrepareNewSessionHandshake:477(1392ns) {SwitchToState:179(375ns)} GenerateSessionVirtualId:988(662ns) IssueIncomingHandshakeReply:193(26173581ns) {SwitchToState:179(3167ns)}}}
2025-11-28T14:53:36.724535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789543088313805:2063] [node 6] ICS09 handshake done sender: [4:7577789543088313742:2064] self: [4:7577789543088313741:0] peer: [6:7577789542202921877:0] socket: 396 qp: -1
2025-11-28T14:53:36.728797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789538793346427:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789543088313803:2117] held: no
2025-11-28T14:53:36.744482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789543088313805:2063] [node 6] ICS10 traffic start
2025-11-28T14:53:36.752655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:36.780546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789538793346426:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26983) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789543088313800:2116] held: no
2025-11-28T14:53:36.784375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789543088313808:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53324
2025-11-28T14:53:36.796405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789543088313811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53324
2025-11-28T14:53:36.812414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789543088313797:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.812466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789543088313797:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.825459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789543088313803:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:36.825644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789543088313800:2116] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:36.828275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789538793346427:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.836868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789538793346426:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.836925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:36.836983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789538793346426:2049] [node 2] ICP22 created new session: [4:7577789543088313812:2049]
2025-11-28T14:53:36.837038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789543088313812:2049] [node 2] ICS09 handshake done sender: [4:7577789543088313800:2116] self: [4:7577789543088313804:0] peer: [2:7577789540907064807:0] socket: 417 qp: -1
2025-11-28T14:53:36.837081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789543088313812:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:36.837131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:36.848490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789543088313797:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:36.859060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:36.864557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789538793346427:2050] [node 3] ICP22 created new session: [4:7577789543088313815:2050]
2025-11-28T14:53:36.868972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789543088313746:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.869076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:36.869133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789543088313746:2066] [node 5] ICP22 created new session: [4:7577789543088313816:2066]
2025-11-28T14:53:36.869183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789543088313816:2066] [node 5] ICS09 handshake done sender: [4:7577789543088313797:2114] self: [4:7577789543088313794:0] peer: [5:7577789543370480173:0] socket: 444 qp: -1
2025-11-28T14:53:36.869222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789543088313816:2066] [node 5] ICS10 traffic start
2025-11-28T14:53:36.869265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:36.873013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789543088313818:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53324
2025-11-28T14:53:36.877427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789543088313815:2050] [node 3] ICS09 handshake done sender: [4:7577789543088313803:2117] self: [4:7577789543088313807:0] peer: [3:7577789540802278536:0] socket: 437 qp: -1
2025-11-28T14:53:36.877476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789543088313815:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:36.882923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789543088313818:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.883090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789538793346425:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.908342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:36.674729Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789542202921874:2068] [node 4] ICP35 event processing took too much time PendingConnection:279(35850168ns) {HandleHandshakeStatus:329(35847824ns) {DropIncomingHandshake:442(1411ns) DropOutgoingHandshake:457(6762690ns) SwitchToState:179(219ns) ProcessPendingSessionEvents:495(4605027ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4529530ns max# 3897193ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.765577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.660707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789542217824064:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.820638Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789543370480159:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.646629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:36.949646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789538793346427:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(112352816ns) {HandleHandshakeStatus:329(112348245ns) {DropIncomingHandshake:442(644ns) DropOutgoingHandshake:457(452ns) SwitchToState:179(461ns) ProcessPendingSessionEvents:495(1936ns)}}
2025-11-28T14:53:36.765645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789540907064777:2066] [node 4] ICP22 created new session: [2:7577789540907064820:2066]
2025-11-28T14:53:36.675471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789537907954504:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.824772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789539075512796:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.646685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789540802278479:2066] [node 6] ICP22 created new session: [3:7577789540802278539:2066]
2025-11-28T14:53:36.665987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789537922856691:2048] [node 2] ICP22 created new session: [1:7577789542217824070:2048]
2025-11-28T14:53:36.765712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789540907064820:2066] [node 4] ICS09 handshake done sender: [2:7577789540907064808:2115] self: [2:7577789540907064807:0] peer: [4:7577789543088313804:0] socket: 416 qp: -1
2025-11-28T14:53:36.676196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921888:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.976814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789538793346425:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2583) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789543088313818:2120] held: no
2025-11-28T14:53:36.646734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789540802278539:2066] [node 6] ICS09 handshake done sender: [3:7577789540802278529:2114] self: [3:7577789540802278528:0] peer: [6:7577789542202921883:0] socket: 426 qp: -1
2025-11-28T14:53:36.834094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789543370480176:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37711
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.678151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824068:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.765764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789540907064820:2066] [node 4] ICS10 traffic start
2025-11-28T14:53:36.676690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921889:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.852844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789543370480159:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:36.679623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921890:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.646770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789540802278539:2066] [node 6] ICS10 traffic start
2025-11-28T14:53:36.678185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789542217824068:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.765807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:37.012799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789543088313821:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53324
2025-11-28T14:53:36.859246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789543370480176:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.646806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:36.697785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789542217824070:2048] [node 2] ICS09 handshake done sender: [1:7577789542217824053:2111] self: [1:7577789542217824052:0] peer: [2:7577789540907064812:0] socket: 410 qp: -1
2025-11-28T14:53:36.680725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789537907954506:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789542202921881:2117] held: no
2025-11-28T14:53:36.792422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789540907064815:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26473
2025-11-28T14:53:37.033828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789538793346425:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(66802879ns) {HandleHandshakeStatus:395(66799977ns) {LogHandshakeFail:472(14764ns) {UpdateErrorStateLog:545(5608ns)} DropOutgoingHandshake:457(1796ns) ProcessPendingSessionEvents:495(56076965ns) {unordered DropSessionEvent:512 num# 100 sum# 13295267ns max# 8749915ns}}}
2025-11-28T14:53:36.859573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789539075512795:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.661117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789540802278498:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(114899520ns) {Configure:104(114891225ns) {ProcessConfigured:135(22869496ns) {StartInitialHandshake:157(22866937ns) {DropHandshakes:470(1221ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(1099ns) PrepareNewSessionHandshake:477(3036ns) {SwitchToState:179(1746ns)}}}}}
2025-11-28T14:53:37.034146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789543088313818:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:36.684480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789542202921882:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:36.704595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789542217824068:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:36.712423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789542217824043:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(114495516ns) {Configure:104(114489153ns) {ProcessConfigured:135(53592ns) {StartInitialHandshake:157(51109ns) {DropHandshakes:470(1942ns) {DropIncomingHandshake:442(560ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(675ns) PrepareNewSessionHandshake:477(1359ns) {SwitchToState:179(522ns)}}}}}
2025-11-28T14:53:36.712452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789542217824070:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:36.714058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789542217824072:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5898
2025-11-28T14:53:36.718397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:36.745738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789542217824043:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.876885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789539075512797:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.745844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:36.745902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789542217824043:2075] [node 6] ICP22 created new session: [1:7577789542217824076:2075]
2025-11-28T14:53:36.882341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789543370480180:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37711
2025-11-28T14:53:36.745959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789542217824076:2075] [node 6] ICS09 handshake done sender: [1:7577789542217824068:2116] self: [1:7577789542217824067:0] peer: [6:7577789542202921887:0] socket: 430 qp: -1
2025-11-28T14:53:36.908527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789539075512797:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17427776ns) {IncomingHandshake:261(17426272ns) {DropIncomingHandshake:442(317ns) PrepareNewSessionHandshake:477(1101ns) {SwitchToState:179(429ns)} GenerateSessionVirtualId:988(536ns) IssueIncomingHandshakeReply:193(11712412ns) {SwitchToState:179(2442ns)}}}
2025-11-28T14:53:36.745996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789542217824076:2075] [node 6] ICS10 traffic start
2025-11-28T14:53:36.916448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789543370480118:2072] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19664) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789543370480172:2118] held: no
2025-11-28T14:53:36.746034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:36.757043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789542217824064:2115] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:36.917078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789543370480107:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.917170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:36.760042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789537922856692:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.917231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789543370480107:2060] [node 6] ICP22 created new session: [5:7577789543370480183:2060]
2025-11-28T14:53:36.760128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:36.760189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789537922856692:2049] [node 3] ICP22 created new session: [1:7577789542217824078:2049]
2025-11-28T14:53:36.917281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789543370480183:2060] [node 6] ICS09 handshake done sender: [5:7577789543370480159:2111] self: [5:7577789543370480157:0] peer: [6:7577789542202921893:0] socket: 439 qp: -1
2025-11-28T14:53:36.760247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789542217824078:2049] [node 3] ICS09 handshake done sender: [1:7577789542217824064:2115] self: [1:7577789542217824062:0] peer: [3:7577789540802278538:0] socket: 423 qp: -1
2025-11-28T14:53:36.917324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789543370480183:2060] [node 6] ICS10 traffic start
2025-11-28T14:53:36.917402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:36.760296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789542217824078:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:36.925399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789543370480182:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37711
2025-11-28T14:53:36.760337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:36.770214Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789542217824043:2075] [node 6] ICP35 event processing took too much time PendingConnection:279(21957633ns) {HandleHandshakeStatus:329(1164633ns) {DropIncomingHandshake:442(17370ns) DropOutgoingHandshake:457(349ns) SwitchToState:179(466ns) ProcessPendingSessionEvents:495(596383ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 532916ns max# 56728ns}}}
2025-11-28T14:53:36.686530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789540802278541:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.801890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789542217824080:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5898
2025-11-28T14:53:36.686557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789540802278541:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.802001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789542217824081:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5898
2025-11-28T14:53:36.687479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789540802278541:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:36.803579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789537922856692:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(42654807ns) {HandleHandshakeStatus:329(42651129ns) {DropIncomingHandshake:442(17819ns) DropOutgoingHandshake:457(675ns) SwitchToState:179(838ns) ProcessPendingSessionEvents:495(14086800ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14023668ns max# 13583865ns}}}
2025-11-28T14:53:36.687754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789540802278498:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.828193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824081:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.687814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:36.844681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824080:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.687872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789540802278498:2064] [node 4] ICP22 created new session: [3:7577789540802278546:2064]
2025-11-28T14:53:36.844761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824072:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.846472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789542217824054:2073] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(84954781ns) {Configure:104(84948198ns) {ProcessConfigured:135(113100ns) {StartInitialHandshake:157(109225ns) {DropHandshakes:470(2388ns) {DropIncomingHandshake:442(501ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(907ns) PrepareNewSessionHandshake:477(1429ns) {SwitchToState:179(501ns)}}}}}
2025-11-28T14:53:36.687920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789540802278546:2064] [node 4] ICS09 handshake done sender: [3:7577789540802278541:2117] self: [3:7577789540802278536:0] peer: [4:7577789543088313807:0] socket: 400 qp: -1
2025-11-28T14:53:36.687956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789540802278546:2064] [node 4] ICS10 traffic start
2025-11-28T14:53:36.883477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824083:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.687996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:36.883505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789542217824083:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.711794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789536507311161:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(412154900ns) {Configure:104(412145019ns) {ProcessConfigured:135(32387885ns) {StartInitialHandshake:157(32382611ns) {DropHandshakes:470(2998ns) {DropIncomingHandshake:442(884ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(1362ns) PrepareNewSessionHandshake:477(3467ns) {SwitchToState:179(1505ns)}}}}}
2025-11-28T14:53:36.724567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789540802278481:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(282643760ns) {Configure:104(282634634ns) {ProcessConfigured:135(27974128ns) {StartInitialHandshake:157(27970566ns) {DropHandshakes:470(3162ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(641ns)} GenerateSessionVirtualId:988(419ns) PrepareNewSessionHandshake:477(4944ns) {SwitchToState:179(1949ns)}}}}}
2025-11-28T14:53:36.732214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789540802278549:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-28T14:53:36.772884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789540802278533:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:36.777758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789536507311160:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2583) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789540802278533:2115] held: no
2025-11-28T14:53:36.789092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789536507311160:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.817532Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789540802278498:2064] [node 4] ICP35 event processing took too much time PendingConnection:279(128832991ns) {HandleHandshakeStatus:329(128830785ns) {DropIncomingHandshake:442(12864ns) DropOutgoingHandshake:457(868ns) SwitchToState:179(210ns) ProcessPendingSessionEvents:495(128453731ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 128304689ns max# 29944174ns}}}
2025-11-28T14:53:36.818853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:36.818909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789536507311160:2048] [node 1] ICP22 created new session: [3:7577789540802278551:2048]
2025-11-28T14:53:36.848073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789540802278551:2048] [node 1] ICS09 handshake done sender: [3:7577789540802278533:2115] self: [3:7577789540802278538:0] peer: [1:7577789542217824062:0] socket: 424 qp: -1
2025-11-28T14:53:36.862795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789540802278551:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:36.862874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:36.892569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789536507311161:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.895759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789540802278548:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.895806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789540802278548:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.904848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789536507311160:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(102094953ns) {HandleHandshakeStatus:329(102091545ns) {DropIncomingHandshake:442(1539ns) DropOutgoingHandshake:457(441ns) SwitchToState:179(339ns) ProcessPendingSessionEvents:495(1334ns)}}
2025-11-28T14:53:36.908758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789536507311161:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(24488639ns) {IncomingHandshake:261(4816191ns) {DropIncomingHandshake:442(984ns) PrepareNewSessionHandshake:477(1377ns) {SwitchToState:179(156ns)} GenerateSessionVirtualId:988(309ns) IssueIncomingHandshakeReply:193(109555ns) {SwitchToState:179(342ns)}}}
2025-11-28T14:53:36.910245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789540802278548:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:36.933042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789540802278481:2060] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.933148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:36.938114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789540802278481:2060] [node 5] ICP22 created new session: [3:7577789540802278555:2060]
2025-11-28T14:53:36.945345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789540802278556:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-28T14:53:36.949264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789540802278555:2060] [node 5] ICS09 handshake done sender: [3:7577789540802278548:2118] self: [3:7577789540802278544:0] peer: [5:7577789543370480179:0] socket: 402 qp: -1
2025-11-28T14:53:36.954057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789540802278555:2060] [node 5] ICS10 traffic start
2025-11-28T14:53:36.965642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:36.984559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789540802278488:2076] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:36.999214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789536507311161:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.999338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:36.999409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789536507311161:2049] [node 2] ICP22 created new session: [3:7577789540802278559:2049]
2025-11-28T14:53:36.999463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789540802278559:2049] [node 2] ICS09 handshake done sender: [3:7577789540802278488:2076] self: [3:7577789540802278553:0] peer: [2:7577789540907064769:0] socket: 401 qp: -1
2025-11-28T14:53:36.999498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789540802278559:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:36.999549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:37.025058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789540802278558:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23076
2025-11-28T14:53:37.053627Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789536507311161:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:36.833113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789540907064806:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:36.844583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789536612097436:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.852891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789540907064817:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26473
2025-11-28T14:53:36.853037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789540907064801:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.853078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789540907064801:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.853213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:36.853277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789536612097436:2048] [node 1] ICP22 created new session: [2:7577789540907064823:2048]
2025-11-28T14:53:36.871053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789540907064817:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.885623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789540907064816:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.885652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789540907064816:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.886966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789540907064816:2118] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:36.887356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789540907064780:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.887425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:36.887489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789540907064780:2069] [node 6] ICP22 created new session: [2:7577789540907064827:2069]
2025-11-28T14:53:36.887536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789540907064827:2069] [node 6] ICS09 handshake done sender: [2:7577789540907064816:2118] self: [2:7577789540907064814:0] peer: [6:7577789542202921896:0] socket: 454 qp: -1
2025-11-28T14:53:36.887571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789540907064827:2069] [node 6] ICS10 traffic start
2025-11-28T14:53:36.887604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:36.892742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789540907064823:2048] [node 1] ICS09 handshake done sender: [2:7577789540907064806:2114] self: [2:7577789540907064812:0] peer: [1:7577789542217824052:0] socket: 413 qp: -1
2025-11-28T14:53:36.892796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789540907064823:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:36.892862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:36.894652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789536612097436:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(57153990ns) {HandleHandshakeStatus:329(57150311ns) {DropIncomingHandshake:442(1965ns) DropOutgoingHandshake:457(10930ns) SwitchToState:179(806ns) ProcessPendingSessionEvents:495(664811ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 588890ns max# 57779ns}}}
2025-11-28T14:53:36.930992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789540907064770:2083] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.931038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789540907064770:2083] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:36.940798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789540907064801:2109] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:36.948949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789540907064832:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26473
2025-11-28T14:53:36.954243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789540907064770:2083] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:36.958835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789540907064760:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.981002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789536612097437:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.981139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:36.981214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789536612097437:2049] [node 3] ICP22 created new session: [2:7577789540907064833:2049]
2025-11-28T14:53:36.981270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789540907064833:2049] [node 3] ICS09 handshake done sender: [2:7577789540907064770:2083] self: [2:7577789540907064769:0] peer: [3:7577789540802278553:0] socket: 398 qp: -1
2025-11-28T14:53:36.981315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789540907064833:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:36.981357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:36.998492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:37.000701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789540907064832:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:37.007755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789540907064760:2067] [node 5] ICP22 created new session: [2:7577789540907064835:2067]
2025-11-28T14:53:37.028245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789540907064835:2067] [node 5] ICS09 handshake done sender: [2:7577789540907064801:2109] self: [2:7577789540907064800:0] peer: [5:7577789543370480175:0] socket: 421 qp: -1
2025-11-28T14:53:37.035772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789540907064835:2067] [node 5] ICS10 traffic start
2025-11-28T14:53:37.045105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:37.060647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789540907064760:2067] [node 5] ICP35 event processing took too much time PendingConnection:279(100838870ns) {HandleHandshakeStatus:329(100834899ns) {DropIncomingHandshake:442(696ns) DropOutgoingHandshake:457(626ns) SwitchToState:179(445ns) ProcessPendingSessionEvents:495(1275307ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1217057ns max# 819758ns}}}
2025-11-28T14:53:37.060735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789540907064760:2067] [node 5] ICP18 handshake (actor [2:7577789540907064832:2120]) is too old
2025-11-28T14:53:37.065103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789536612097437:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17306867ns) {HandleHandshakeStatus:329(17303223ns) {DropIncomingHandshake:442(23384ns) DropOutgoingHandshake:457(429ns) SwitchToState:179(712ns) ProcessPendingSessionEvents:495(16714412ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16647280ns max# 15825897ns}}}
2025-11-28T14:53:37.066352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789540907064780:2069] [node 6] ICP35 event processing took too much time PendingConnection:279(178062364ns) {HandleHandshakeStatus:329(178059193ns) {DropIncomingHandshake:442(13351ns) DropOutgoingHandshake:457(495ns) SwitchToState:179(291ns) ProcessPendingSessionEvents:495(134409775ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 134281203ns max# 23888328ns}}}
2025-11-28T14:53:36.958537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789543370480181:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37711
2025-11-28T14:53:36.888644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789542217824060:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.684736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789537907954504:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:37.065261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789540802278481:2060] [node 5] ICP35 event processing took too much time PendingConnection:279(131191066ns) {HandleHandshakeStatus:329(131187910ns) {DropIncomingHandshake:442(406ns) DropOutgoingHandshake:457(462ns) SwitchToState:179(830ns) ProcessPendingSessionEvents:495(49116451ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47804413ns max# 20224929ns}}}
2025-11-28T14:53:36.888683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789542217824060:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:37.052189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789538793346425:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.964618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789543370480176:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:36.965366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789543370480187:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37711
2025-11-28T14:53:36.989248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789539075512797:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.997218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789543370480162:2113] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:37.003400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789539075512796:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:37.003471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:37.003519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789539075512796:2049] [node 2] ICP22 created new session: [5:7577789547665447484:2049]
2025-11-28T14:53:37.003586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789547665447484:2049] [node 2] ICS09 handshake done sender: [5:7577789543370480162:2113] self: [5:7577789543370480175:0] peer: [2:7577789540907064800:0] socket: 429 qp: -1
2025-11-28T14:53:37.003628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789547665447484:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:37.003693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:37.004910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789543370480161:2112] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:37.005409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789539075512795:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:37.005486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:37.005524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789539075512795:2048] [node 1] ICP22 created new session: [5:7577789547665447486:2048]
2025-11-28T14:53:37.005568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789547665447486:2048] [node 1] ICS09 handshake done sender: [5:7577789543370480161:2112] self: [5:7577789543370480178:0] peer: [1:7577789542217824059:0] socket: 428 qp: -1
2025-11-28T14:53:37.005605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789547665447486:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:37.005633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:37.014524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:37.024535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789539075512797:2050] [node 3] ICP22 created new session: [5:7577789547665447488:2050]
2025-11-28T14:53:37.036940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789547665447488:2050] [node 3] ICS09 handshake done sender: [5:7577789543370480176:2119] self: [5:7577789543370480179:0] peer: [3:7577789540802278544:0] socket: 403 qp: -1
2025-11-28T14:53:37.051725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789547665447488:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:37.051800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:37.056457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789539075512797:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(66335345ns) {HandleHandshakeStatus:329(66332033ns) {DropIncomingHandshake:442(679ns) DropOutgoingHandshake:457(5992ns) SwitchToState:179(587ns) ProcessPendingSessionEvents:495(3613840ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3556650ns max# 3102102ns}}}
2025-11-28T14:53:37.073576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789543370480118:2072] [node 4] ICP35 event processing took too much time PendingConnection:279(96855084ns) {HandleHandshakeStatus:395(96833923ns) {LogHandshakeFail:472(15102ns) {UpdateErrorStateLog:545(4936ns)} DropOutgoingHandshake:457(545ns) ProcessPendingSessionEvents:495(96748941ns) {unordered DropSessionEvent:512 num# 100 sum# 32238208ns max# 12694369ns}}}
2025-11-28T14:53:37.073674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789543370480172:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:36.905032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789542217824083:2120] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:36.926566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789542217824054:2073] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:36.926636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:36.926698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789542217824054:2073] [node 4] ICP22 created new session: [1:7577789542217824088:2073]
2025-11-28T14:53:36.926749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789542217824088:2073] [node 4] ICS09 handshake done sender: [1:7577789542217824083:2120] self: [1:7577789542217824082:0] peer: [4:7577789543088313819:0] socket: 455 qp: -1
2025-11-28T14:53:36.926790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789542217824088:2073] [node 4] ICS10 traffic start
2025-11-28T14:53:36.926852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:36.976578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789542217824060:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:37.002320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789542217824055:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:37.007679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:37.011807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789542217824055:2074] [node 5] ICP22 created new session: [1:7577789546512791387:2074]
2025-11-28T14:53:37.011871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789546512791387:2074] [node 5] ICS09 handshake done sender: [1:7577789542217824060:2114] self: [1:7577789542217824059:0] peer: [5:7577789543370480178:0] socket: 418 qp: -1
2025-11-28T14:53:37.012997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789546512791387:2074] [node 5] ICS10 traffic start
2025-11-28T14:53:37.014393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:37.062657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789537922856691:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(392474210ns) {HandleHandshakeStatus:329(392471970ns) {DropIncomingHandshake:442(444ns) DropOutgoingHandshake:457(496ns) SwitchToState:179(527ns) ProcessPendingSessionEvents:495(218980018ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 203175069ns max# 19878247ns}}}
2025-11-28T14:53:37.063759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789537922856691:2048] [node 2] ICP18 handshake (actor [1:7577789542217824081:2119]) is too old
2025-11-28T14:53:37.078868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789542217824055:2074] [node 5] ICP35 event processing took too much time PendingConnection:279(75714527ns) {HandleHandshakeStatus:329(75712393ns) {DropIncomingHandshake:442(15664ns) DropOutgoingHandshake:457(420ns) SwitchToState:179(476ns) ProcessPendingSessionEvents:495(23299940ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23217075ns max# 9968697ns}}}
2025-11-28T14:53:37.052253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:37.061214Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789543088313740:2063] [node 6] ICP35 event processing took too much time PendingConnection:279(435598637ns) {HandleHandshakeStatus:329(423727337ns) {DropIncomingHandshake:442(1628ns) DropOutgoingHandshake:457(2512ns) SwitchToState:179(359ns) ProcessPendingSessionEvents:495(254992984ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 240108875ns max# 34137412ns}}}
2025-11-28T14:53:37.064663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789538793346425:2048] [node 1] ICP22 created new session: [4:7577789547383281118:2048]
2025-11-28T14:53:37.064720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789547383281118:2048] [node 1] ICS09 handshake done sender: [4:7577789543088313818:2120] self: [4:7577789543088313819:0] peer: [1:7577789542217824082:0] socket: 449 qp: -1
2025-11-28T14:53:37.064770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789547383281118:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:37.072167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:36.684817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:36.684869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789537907954504:2048] [node 1] ICP22 created new session: [6:7577789542202921891:2048]
2025-11-28T14:53:36.684928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789542202921891:2048] [node 1] ICS09 handshake done sender: [6:7577789542202921882:2118] self: [6:7577789542202921887:0] peer: [1:7577789542217824067:0] socket: 432 qp: -1
2025-11-28T14:53:36.700823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789542202921891:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:36.700892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:36.783856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789542202921889:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.785929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789542202921829:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.832588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789537907954504:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(146658170ns) {HandleHandshakeStatus:329(146653024ns) {DropIncomingHandshake:442(941ns) DropOutgoingHandshake:457(5207ns) SwitchToState:179(876ns) ProcessPendingSessionEvents:495(109390970ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 98199904ns max# 29771427ns}}}
2025-11-28T14:53:36.884774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921895:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.885251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789542202921895:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:36.885369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789537907954505:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:36.886780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921897:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.888465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789542202921895:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:36.919054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789542202921894:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29468
2025-11-28T14:53:36.928608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789537907954505:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.928700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:36.928753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789537907954505:2049] [node 2] ICP22 created new session: [6:7577789542202921898:2049]
2025-11-28T14:53:36.928810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789542202921898:2049] [node 2] ICS09 handshake done sender: [6:7577789542202921895:2123] self: [6:7577789542202921896:0] peer: [2:7577789540907064814:0] socket: 456 qp: -1
2025-11-28T14:53:36.928848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789542202921898:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:36.928883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:36.929729Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789537907954505:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:36.956698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789542202921829:2071] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20371) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789542202921889:2120] held: no
2025-11-28T14:53:36.957478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789537907954506:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(175821520ns) {HandleHandshakeStatus:395(175817848ns) {LogHandshakeFail:472(8162397ns) {UpdateErrorStateLog:545(6486ns)} DropOutgoingHandshake:457(524ns) ProcessPendingSessionEvents:495(167593680ns) {unordered DropSessionEvent:512 num# 100 sum# 167497914ns max# 20546101ns}}}
2025-11-28T14:53:36.957764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789542202921881:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:36.972774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789537907954506:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:36.972875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:36.972941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789537907954506:2050] [node 3] ICP22 created new session: [6:7577789542202921900:2050]
2025-11-28T14:53:36.973011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789542202921900:2050] [node 3] ICS09 handshake done sender: [6:7577789542202921881:2117] self: [6:7577789542202921883:0] peer: [3:7577789540802278528:0] socket: 427 qp: -1
2025-11-28T14:53:36.973063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789542202921900:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:36.973107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:37.041602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789542202921829:2071] [node 5] ICP35 event processing took too much time PendingConnection:279(88224656ns) {HandleHandshakeStatus:395(88219525ns) {LogHandshakeFail:472(4153793ns) {UpdateErrorStateLog:545(5675ns)} DropOutgoingHandshake:457(593ns) ProcessPendingSessionEvents:495(83991462ns) {unordered DropSessionEvent:512 num# 100 sum# 83930473ns max# 18205207ns}}}
2025-11-28T14:53:37.041873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789542202921889:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:37.042238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789542202921829:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:37.042358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:37.042418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789542202921829:2071] [node 5] ICP22 created new session: [6:7577789546497889198:2071]
2025-11-28T14:53:37.042471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789546497889198:2071] [node 5] ICS09 handshake done sender: [6:7577789542202921889:2120] self: [6:7577789542202921893:0] peer: [5:7577789543370480157:0] socket: 440 qp: -1
2025-11-28T14:53:37.042509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789546497889198:2071] [node 5] ICS10 traffic start
2025-11-28T14:53:37.042554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:37.083607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789538793346425:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31356020ns) {HandleHandshakeStatus:329(31352633ns) {DropIncomingHandshake:442(734ns) DropOutgoingHandshake:457(563ns) SwitchToState:179(495ns) ProcessPendingSessionEvents:495(1919ns)}}
2025-11-28T14:53:37.084665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789543370480118:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:37.086918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:37.087009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789543370480118:2072] [node 4] ICP22 created new session: [5:7577789547665447490:2072]
2025-11-28T14:53:37.087069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789547665447490:2072] [node 4] ICS09 handshake done sender: [5:7577789543370480172:2118] self: [5:7577789543370480173:0] peer: [4:7577789543088313794:0] socket: 447 qp: -1
2025-11-28T14:53:37.087109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789547665447490:2072] [node 4] ICS10 traffic start
2025-11-28T14:53:37.087181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:37.120016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:37.120268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789540907064823:2048] [node 1] ICS01 socket: 413 reason# EndOfStream
2025-11-28T14:53:37.120308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789536612097436:2048] [node 1] ICP30 unregister session Session# [2:7577789540907064823:2048] VirtualId# [2:7577789540907064812:0]
2025-11-28T14:53:37.120352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789540907064823:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.120456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:37.120518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:37.122589Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:37.122811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789540802278551:2048] [node 1] ICS01 socket: 424 reason# EndOfStream
2025-11-28T14:53:37.122854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789536507311160:2048] [node 1] ICP30 unregister session Session# [3:7577789540802278551:2048] VirtualId# [3:7577789540802278538:0]
2025-11-28T14:53:37.122888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789540802278551:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.122973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:37.123020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:37.128603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:37.128838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789547383281118:2048] [node 1] ICS01 socket: 449 reason# EndOfStream
2025-11-28T14:53:37.128885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789538793346425:2048] [node 1] ICP30 unregister session Session# [4:7577789547383281118:2048] VirtualId# [4:7577789543088313819:0]
2025-11-28T14:53:37.128920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789547383281118:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.128997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:37.129041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:37.129141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:37.129326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789542202921891:2048] [node 1] ICS07 socket disconnect 432 reason# EndOfStream
2025-11-28T14:53:37.129355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789542202921891:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.129423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:37.129468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789542202921891:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:37.130748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789537907954504:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2583) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:37.130799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789542202921891:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:37.130832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789537907954504:2048] [node 1] ICP30 unregister session Session# [6:7577789542202921891:2048] VirtualId# [6:7577789542202921887:0]
2025-11-28T14:53:37.130952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:37.130984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789537907954504:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2583) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:37.131006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2583) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:37.132537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:37.132797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789547665447486:2048] [node 1] ICS01 socket: 428 reason# EndOfStream
2025-11-28T14:53:37.132842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789539075512795:2048] [node 1] ICP30 unregister session Session# [5:7577789547665447486:2048] VirtualId# [5:7577789543370480178:0]
2025-11-28T14:53:37.132873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789547665447486:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.133075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:37.133189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:37.137222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789537907954504:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:37.258640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:37.258882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789540802278559:2049] [node 2] ICS01 socket: 401 reason# EndOfStream
2025-11-28T14:53:37.258932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789536507311161:2049] [node 2] ICP30 unregister session Session# [3:7577789540802278559:2049] VirtualId# [3:7577789540802278553:0]
2025-11-28T14:53:37.258975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789540802278559:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.259068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:37.259109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:37.264626Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:37.264858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789543088313812:2049] [node 2] ICS01 socket: 417 reason# EndOfStream
2025-11-28T14:53:37.264901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789538793346426:2049] [node 2] ICP30 unregister session Session# [4:7577789543088313812:2049] VirtualId# [4:7577789543088313804:0]
2025-11-28T14:53:37.264937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789543088313812:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.265019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:37.265068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:37.265585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:37.265754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789542202921898:2049] [node 2] ICS01 socket: 456 reason# EndOfStream
2025-11-28T14:53:37.265784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789537907954505:2049] [node 2] ICP30 unregister session Session# [6:7577789542202921898:2049] VirtualId# [6:7577789542202921896:0]
2025-11-28T14:53:37.265808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789542202921898:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.265854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:37.265885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:37.265157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:37.265334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789547665447484:2049] [node 2] ICS01 socket: 429 reason# EndOfStream
2025-11-28T14:53:37.265375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789539075512796:2049] [node 2] ICP30 unregister session Session# [5:7577789547665447484:2049] VirtualId# [5:7577789543370480175:0]
2025-11-28T14:53:37.265403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789547665447484:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.265457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:37.265497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:37.377135Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:37.377383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789547665447488:2050] [node 3] ICS01 socket: 403 reason# EndOfStream
2025-11-28T14:53:37.377431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789539075512797:2050] [node 3] ICP30 unregister session Session# [5:7577789547665447488:2050] VirtualId# [5:7577789543370480179:0]
2025-11-28T14:53:37.377471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789547665447488:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.377555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:37.377597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:37.384571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:37.384571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:37.384796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789542202921900:2050] [node 3] ICS01 socket: 427 reason# EndOfStream
2025-11-28T14:53:37.384838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789537907954506:2050] [node 3] ICP30 unregister session Session# [6:7577789542202921900:2050] VirtualId# [6:7577789542202921883:0]
2025-11-28T14:53:37.384807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789543088313815:2050] [node 3] ICS01 socket: 437 reason# EndOfStream
2025-11-28T14:53:37.384873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789542202921900:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.384859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789538793346427:2050] [node 3] ICP30 unregister session Session# [4:7577789543088313815:2050] VirtualId# [4:7577789543088313807:0]
2025-11-28T14:53:37.384973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:37.384891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789543088313815:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.385017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:37.384978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:37.385017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:37.517655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:37.517891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789547665447490:2072] [node 4] ICS01 socket: 447 reason# EndOfStream
2025-11-28T14:53:37.517940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789543370480118:2072] [node 4] ICP30 unregister session Session# [5:7577789547665447490:2072] VirtualId# [5:7577789543370480173:0]
2025-11-28T14:53:37.517975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789547665447490:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.518046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:37.518086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:37.518178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:37.518365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789542202921884:2068] [node 4] ICS01 socket: 397 reason# EndOfStream
2025-11-28T14:53:37.518398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789542202921874:2068] [node 4] ICP30 unregister session Session# [6:7577789542202921884:2068] VirtualId# [6:7577789542202921877:0]
2025-11-28T14:53:37.518427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789542202921884:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.518494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:37.518532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:37.610191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:37.610432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789546497889198:2071] [node 5] ICS01 socket: 440 reason# EndOfStream
2025-11-28T14:53:37.610477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789542202921829:2071] [node 5] ICP30 unregister session Session# [6:7577789546497889198:2071] VirtualId# [6:7577789542202921893:0]
2025-11-28T14:53:37.610529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789546497889198:2071] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:37.610662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:37.610704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:37.831439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789546272486817:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:37.831489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789546272486818:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:37.884562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789544329983474:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:37.884629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789544329983475:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:37.935575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789547893817046:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:37.935628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789547893817047:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:37.957035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789546319959122:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:37.957089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789546319959123:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:37.957119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789546319959124:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:38.028769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789547618906140:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:38.028822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789547618906141:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:38.028857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789547618906142:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:38.136735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789549212222297:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:38.136797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789549212222298:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:38.136851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789549212222299:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:38.332731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789549212222310:2059] [node 5] ICP01 ready to work
2025-11-28T14:53:38.408531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789552188784371:2067] [node 5] ICP01 ready to work
2025-11-28T14:53:38.417615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789548624950804:2073] [node 5] ICP01 ready to work
2025-11-28T14:53:38.425081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789550614926443:2064] [node 6] ICP01 ready to work
2025-11-28T14:53:38.479015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789548624950830:2074] [node 4] ICP01 ready to work
2025-11-28T14:53:38.459320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789550567454168:2085] [node 5] ICP01 ready to work
2025-11-28T14:53:38.469847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789552188784374:2068] [node 4] ICP01 ready to work
2025-11-28T14:53:38.461201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789550614926444:2067] [node 5] ICP01 ready to work
2025-11-28T14:53:38.571305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789550567454177:2086] [node 4] ICP01 ready to work
2025-11-28T14:53:38.603046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789548624950835:2075] [node 6] ICP01 ready to work
2025-11-28T14:53:38.586201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789552188784408:2070] [node 6] ICP01 ready to work
2025-11-28T14:53:38.616616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789550567454178:2087] [node 6] ICP01 ready to work
2025-11-28T14:53:38.621942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789544329983475:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18262020ns) {Configure:104(18256856ns) {ProcessConfigured:135(77764ns) {StartInitialHandshake:157(75600ns) {DropHandshakes:470(2575ns) {DropIncomingHandshake:442(675ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(927ns) PrepareNewSessionHandshake:477(1260ns) {SwitchToState:179(531ns)}}}}}
2025-11-28T14:53:38.664464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789547618906142:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(295166956ns) {Configure:104(295162224ns) {ProcessConfigured:135(3446749ns) {StartInitialHandshake:157(3445339ns) {DropHandshakes:470(1638ns) {DropIncomingHandshake:442(242ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(660ns) PrepareNewSessionHandshake:477(3241ns) {SwitchToState:179(1936ns)}}}}}
2025-11-28T14:53:38.648817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789548624950830:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24277911ns) {Configure:104(24273547ns) {ProcessConfigured:135(70069ns) {StartInitialHandshake:157(68358ns) {DropHandshakes:470(1263ns) {DropIncomingHandshake:442(163ns) DropOutgoingHandshake:457(157ns)} GenerateSessionVirtualId:988(798ns) PrepareNewSessionHandshake:477(1507ns) {SwitchToState:179(536ns)}}}}}
2025-11-28T14:53:38.655889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789546319959122:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(109783698ns) {Configure:104(109775612ns) {ProcessConfigured:135(74074ns) {StartInitialHandshake:157(71841ns) {DropHandshakes:470(2867ns) {DropIncomingHandshake:442(755ns) DropOutgoingHandshake:457(622ns)} GenerateSessionVirtualId:988(836ns) PrepareNewSessionHandshake:477(1739ns) {SwitchToState:179(930ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.692150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789550614926493:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23868
2025-11-28T14:53:38.665337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789551913873501:2068] [node 6] ICP01 ready to work
2025-11-28T14:53:38.696750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789549212222310:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(279886643ns) {Configure:104(279874938ns) {ProcessConfigured:135(21422584ns) {StartInitialHandshake:157(21419190ns) {DropHandshakes:470(2520ns) {DropIncomingHandshake:442(389ns) DropOutgoingHandshake:457(432ns)} GenerateSessionVirtualId:988(706ns) PrepareNewSessionHandshake:477(4525051ns) {SwitchToState:179(4754ns)}}}}}
2025-11-28T14:53:38.697600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789549212222363:2063] [node 4] ICP01 ready to work
2025-11-28T14:53:38.665905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789551913873502:2071] [node 4] ICP01 ready to work
2025-11-28T14:53:38.700914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789550567454178:2087] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48645626ns) {Configure:104(48640843ns) {ProcessConfigured:135(26611972ns) {StartInitialHandshake:157(26609093ns) {DropHandshakes:470(2557ns) {DropIncomingHandshake:442(543ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(484ns) PrepareNewSessionHandshake:477(6247ns) {SwitchToState:179(4207ns)}}}}}
2025-11-28T14:53:38.721745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789547618906141:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34119957ns) {Configure:104(34116391ns) {ProcessConfigured:135(24149668ns) {StartInitialHandshake:157(24146346ns) {DropHandshakes:470(2873ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(550ns)} GenerateSessionVirtualId:988(781ns) PrepareNewSessionHandshake:477(3011ns) {SwitchToState:179(1483ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.737570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789551913873503:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19507
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.738413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789551913873512:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19507
2025-11-28T14:53:38.739642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789550567454189:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3433
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.747435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789548624950849:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42014
2025-11-28T14:53:38.747507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789548624950850:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42014
2025-11-28T14:53:38.748008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789550567454193:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3433
2025-11-28T14:53:38.750034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789550614926493:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.750170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789546319959123:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.756036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789548624950841:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.756076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789548624950841:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.757753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789550614926498:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23868
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.754772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789552188784413:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.760780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789551913873512:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.765191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789548624950841:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:38.758174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454189:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.761501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789552188784413:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.761765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789552188784418:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-11-28T14:53:38.780413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789551913873516:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19507
2025-11-28T14:53:38.761018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789549212222372:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
2025-11-28T14:53:38.789729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789552188784419:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-11-28T14:53:38.784810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789549212222372:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.780518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789551913873502:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.780747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789551913873517:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19507
2025-11-28T14:53:38.805135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789551913873503:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.802985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789548624950830:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:38.803070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:38.803125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789548624950830:2074] [node 4] ICP22 created new session: [2:7577789548624950852:2074]
2025-11-28T14:53:38.803171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789548624950852:2074] [node 4] ICS09 handshake done sender: [2:7577789548624950841:2112] self: [2:7577789548624950840:0] peer: [4:7577789550614926497:0] socket: 398 qp: -1
2025-11-28T14:53:38.803247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789548624950852:2074] [node 4] ICS10 traffic start
2025-11-28T14:53:38.803317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:38.814663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789548624950849:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.814791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789548624950839:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.814817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789548624950839:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.807765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789550614926473:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.824273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789549212222363:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.817336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789548624950855:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42014
2025-11-28T14:53:38.820567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789547893817047:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.805351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789547618906141:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.807797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789550614926473:2098] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.823239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789550567454194:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3433
2025-11-28T14:53:38.829983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789551913873516:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.823310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789550567454195:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3433
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.823369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789550567454196:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3433
2025-11-28T14:53:38.828465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789548624950839:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:38.829485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789549212222377:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
2025-11-28T14:53:38.829649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454193:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.839550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454196:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.820701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789550614926493:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:38.828886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789548624950850:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.839773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454194:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.837668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789549212222378:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
2025-11-28T14:53:38.857622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789549212222378:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.857842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789549212222298:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.826487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789546319959123:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.826578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:38.826636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789546319959123:2049] [node 2] ICP22 created new session: [4:7577789550614926500:2049]
2025-11-28T14:53:38.826695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789550614926500:2049] [node 2] ICS09 handshake done sender: [4:7577789550614926493:2115] self: [4:7577789550614926497:0] peer: [2:7577789548624950840:0] socket: 400 qp: -1
2025-11-28T14:53:38.826734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789550614926500:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:38.826822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:38.866910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789546319959123:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(39500490ns) {HandleHandshakeStatus:329(39498086ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(5118ns) SwitchToState:179(361ns) ProcessPendingSessionEvents:495(39015129ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 38930258ns max# 19698731ns}}}
2025-11-28T14:53:38.867772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789550614926469:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.867801Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789550614926469:2095] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.834258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789548624950804:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:38.839929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454184:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.870138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789552188784418:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.834320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:38.878905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789550614926473:2098] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:38.872580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789549212222380:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.839956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789550567454184:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:38.870269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789552188784419:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.862672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789550567454184:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:38.879295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789546272486817:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:38.879367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:38.879413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789546272486817:2048] [node 2] ICP22 created new session: [1:7577789550567454202:2048]
2025-11-28T14:53:38.879449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789550567454202:2048] [node 2] ICS09 handshake done sender: [1:7577789550567454184:2112] self: [1:7577789550567454183:0] peer: [2:7577789548624950861:0] socket: 417 qp: -1
2025-11-28T14:53:38.879474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789550567454202:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:38.879499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:38.834380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789548624950804:2073] [node 5] ICP22 created new session: [2:7577789548624950858:2073]
2025-11-28T14:53:38.874999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789551913873520:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19507
2025-11-28T14:53:38.889928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789551913873521:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19507
2025-11-28T14:53:38.893084Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789547618906141:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789551913873503:2111] held: no
2025-11-28T14:53:38.893987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789551913873517:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.887466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789550614926444:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:38.887553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:38.887610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789550614926444:2067] [node 5] ICP22 created new session: [4:7577789550614926503:2067]
2025-11-28T14:53:38.887671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789550614926503:2067] [node 5] ICS09 handshake done sender: [4:7577789550614926473:2098] self: [4:7577789550614926472:0] peer: [5:7577789551913873518:0] socket: 402 qp: -1
2025-11-28T14:53:38.887707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789550614926503:2067] [node 5] ICS10 traffic start
2025-11-28T14:53:38.887739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:38.889372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789550614926469:2095] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:38.895988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789550614926443:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:38.899749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:38.834438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789548624950858:2073] [node 5] ICS09 handshake done sender: [2:7577789548624950839:2111] self: [2:7577789548624950838:0] peer: [5:7577789551913873519:0] socket: 397 qp: -1
2025-11-28T14:53:38.834475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789548624950858:2073] [node 5] ICS10 traffic start
2025-11-28T14:53:38.834534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:38.836658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789544329983474:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.855300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789548624950862:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42014
2025-11-28T14:53:38.857384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789548624950850:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:38.858301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789548624950845:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.858331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789548624950845:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:38.858802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789544329983474:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.858854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:38.858916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789544329983474:2048] [node 1] ICP22 created new session: [2:7577789548624950863:2048]
2025-11-28T14:53:38.858964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789548624950863:2048] [node 1] ICS09 handshake done sender: [2:7577789548624950850:2116] self: [2:7577789548624950861:0] peer: [1:7577789550567454183:0] socket: 419 qp: -1
2025-11-28T14:53:38.859001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789548624950863:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:38.859040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:38.870901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789548624950845:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:38.876221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789548624950837:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.876251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789548624950837:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:38.876803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789548624950835:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:38.876854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:38.876908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789548624950835:2075] [node 6] ICP22 created new session: [2:7577789548624950866:2075]
2025-11-28T14:53:38.876959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789548624950866:2075] [node 6] ICS09 handshake done sender: [2:7577789548624950845:2114] self: [2:7577789548624950844:0] peer: [6:7577789549212222379:0] socket: 418 qp: -1
2025-11-28T14:53:38.876998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789548624950866:2075] [node 6] ICS10 traffic start
2025-11-28T14:53:38.877038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:38.881789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789552188784425:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-11-28T14:53:38.882147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789547893817046:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.896119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789549212222378:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:38.905378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789550614926443:2064] [node 6] ICP22 created new session: [4:7577789550614926506:2064]
2025-11-28T14:53:38.905458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789550614926506:2064] [node 6] ICS09 handshake done sender: [4:7577789550614926469:2095] self: [4:7577789550614926468:0] peer: [6:7577789549212222376:0] socket: 401 qp: -1
2025-11-28T14:53:38.905498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789550614926506:2064] [node 6] ICS10 traffic start
2025-11-28T14:53:38.905549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:38.894115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789547618906140:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.894540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789551913873503:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.911407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454180:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.895056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789551913873512:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:38.895247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789547618906141:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.895299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:38.899901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789547618906141:2049] [node 2] ICP22 created new session: [5:7577789551913873523:2049]
2025-11-28T14:53:38.901057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789551913873523:2049] [node 2] ICS09 handshake done sender: [5:7577789551913873503:2111] self: [5:7577789551913873519:0] peer: [2:7577789548624950838:0] socket: 399 qp: -1
2025-11-28T14:53:38.901109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789551913873523:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:38.901169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:38.913697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789551913873502:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.913752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:38.913809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789551913873502:2071] [node 4] ICP22 created new session: [5:7577789551913873526:2071]
2025-11-28T14:53:38.913858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789551913873526:2071] [node 4] ICS09 handshake done sender: [5:7577789551913873512:2116] self: [5:7577789551913873518:0] peer: [4:7577789550614926472:0] socket: 409 qp: -1
2025-11-28T14:53:38.913891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789551913873526:2071] [node 4] ICS10 traffic start
2025-11-28T14:53:38.913950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:38.915328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789550614926508:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23868
2025-11-28T14:53:38.914543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789549212222298:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.914077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789548624950837:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.911448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789550567454180:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.915489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:38.912992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789552188784425:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.915133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789552188784429:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-11-28T14:53:38.921008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789552188784430:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-11-28T14:53:38.921493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789552188784413:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.912068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454182:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.922166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789551913873529:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19507
2025-11-28T14:53:38.948959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789552188784430:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.950083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789551913873516:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:38.925626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789548624950855:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.925871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789544329983475:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:38.925967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:38.926036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789544329983475:2049] [node 3] ICP22 created new session: [2:7577789548624950869:2049]
2025-11-28T14:53:38.926096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789548624950869:2049] [node 3] ICS09 handshake done sender: [2:7577789548624950837:2110] self: [2:7577789548624950836:0] peer: [3:7577789552188784426:0] socket: 396 qp: -1
2025-11-28T14:53:38.926156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789548624950869:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:38.926204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:38.927392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789544329983475:2049] [node 3] ICP18 handshake (actor [2:7577789548624950855:2117]) is too old
2025-11-28T14:53:38.949026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789547893817047:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.915556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789549212222298:2049] [node 2] ICP22 created new session: [6:7577789549212222382:2049]
2025-11-28T14:53:38.915607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789549212222382:2049] [node 2] ICS09 handshake done sender: [6:7577789549212222378:2117] self: [6:7577789549212222379:0] peer: [2:7577789548624950844:0] socket: 445 qp: -1
2025-11-28T14:53:38.915640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789549212222382:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:38.915676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:38.936697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789549212222298:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(21131560ns) {HandleHandshakeStatus:329(21127697ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(5271ns) SwitchToState:179(2166ns) ProcessPendingSessionEvents:495(19877248ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19799697ns max# 10138939ns}}}
2025-11-28T14:53:38.936895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789549212222377:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.938729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789549212222381:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
2025-11-28T14:53:38.912089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789550567454182:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:38.913416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789550567454180:2110] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:38.915088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789550567454182:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:38.950240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789546272486818:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:38.950316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:38.950384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789546272486818:2049] [node 3] ICP22 created new session: [1:7577789550567454207:2049]
2025-11-28T14:53:38.950436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789550567454207:2049] [node 3] ICS09 handshake done sender: [1:7577789550567454180:2110] self: [1:7577789550567454179:0] peer: [3:7577789552188784428:0] socket: 415 qp: -1
2025-11-28T14:53:38.950471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789550567454207:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:38.950506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:38.954570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454195:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.954670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789550567454168:2085] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.949114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:38.954791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:38.949174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789547893817047:2049] [node 2] ICP22 created new session: [3:7577789552188784433:2049]
2025-11-28T14:53:38.949228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789552188784433:2049] [node 2] ICS09 handshake done sender: [3:7577789552188784413:2113] self: [3:7577789552188784426:0] peer: [2:7577789548624950836:0] socket: 407 qp: -1
2025-11-28T14:53:38.949262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789552188784433:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:38.949300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:38.958604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789552188784431:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-11-28T14:53:38.960720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789547618906140:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.960807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:38.960859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789547618906140:2048] [node 1] ICP22 created new session: [5:7577789551913873530:2048]
2025-11-28T14:53:38.960906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789551913873530:2048] [node 1] ICS09 handshake done sender: [5:7577789551913873516:2117] self: [5:7577789551913873522:0] peer: [1:7577789550567454181:0] socket: 421 qp: -1
2025-11-28T14:53:38.960948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789551913873530:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:38.960995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:38.954858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789550567454168:2085] [node 5] ICP22 created new session: [1:7577789550567454209:2085]
2025-11-28T14:53:38.954918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789550567454209:2085] [node 5] ICS09 handshake done sender: [1:7577789550567454182:2111] self: [1:7577789550567454181:0] peer: [5:7577789551913873522:0] socket: 416 qp: -1
2025-11-28T14:53:38.954958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789550567454209:2085] [node 5] ICS10 traffic start
2025-11-28T14:53:38.960119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789547893817047:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:38.961475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789552188784419:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:38.954993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:38.962113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789549212222299:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.965723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789547893817046:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.962191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789549212222372:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.965901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:38.966698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789547893817046:2048] [node 1] ICP22 created new session: [3:7577789552188784437:2048]
2025-11-28T14:53:38.966736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789552188784437:2048] [node 1] ICS09 handshake done sender: [3:7577789552188784419:2117] self: [3:7577789552188784428:0] peer: [1:7577789550567454179:0] socket: 423 qp: -1
2025-11-28T14:53:38.966763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789552188784437:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:38.966479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789546272486818:2049] [node 3] ICP18 handshake (actor [1:7577789550567454195:2118]) is too old
2025-11-28T14:53:38.967903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789550614926508:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.966813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.975634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789546319959122:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.975936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789550614926509:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23868
2025-11-28T14:53:38.976006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789550614926510:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23868
2025-11-28T14:53:38.967271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789549212222384:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
2025-11-28T14:53:38.967492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789549212222363:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.967577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:38.967631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789549212222363:2063] [node 4] ICP22 created new session: [6:7577789549212222387:2063]
2025-11-28T14:53:38.967678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789549212222387:2063] [node 4] ICS09 handshake done sender: [6:7577789549212222372:2115] self: [6:7577789549212222376:0] peer: [4:7577789550614926468:0] socket: 406 qp: -1
2025-11-28T14:53:38.967710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789549212222387:2063] [node 4] ICS10 traffic start
2025-11-28T14:53:38.967737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:38.978366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789552188784412:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.978404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789552188784412:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.980223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789551913873532:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19507
2025-11-28T14:53:38.977563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454186:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.977605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789550567454186:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:38.981997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789552188784412:2112] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:38.982332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789552188784408:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:38.982412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:38.982459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789552188784408:2070] [node 6] ICP22 created new session: [3:7577789552188784441:2070]
2025-11-28T14:53:38.982501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789552188784441:2070] [node 6] ICS09 handshake done sender: [3:7577789552188784412:2112] self: [3:7577789552188784411:0] peer: [6:7577789549212222385:0] socket: 434 qp: -1
2025-11-28T14:53:38.982539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789552188784441:2070] [node 6] ICS10 traffic start
2025-11-28T14:53:38.982573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:38.986902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789549212222384:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.987289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789551913873532:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.987436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789547618906142:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.992737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789549212222389:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.992965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789549212222310:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:38.993060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789549212222390:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
2025-11-28T14:53:38.996336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789552188784421:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:38.996397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789552188784421:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:38.999548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789549212222377:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:39.001080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789550614926512:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23868
2025-11-28T14:53:39.003510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789550567454186:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:39.003946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789550567454177:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:39.004019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:39.004058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789550567454177:2086] [node 4] ICP22 created new session: [1:7577789554862421509:2086]
2025-11-28T14:53:39.004098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789554862421509:2086] [node 4] ICS09 handshake done sender: [1:7577789550567454186:2113] self: [1:7577789550567454185:0] peer: [4:7577789550614926511:0] socket: 454 qp: -1
2025-11-28T14:53:39.004124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789554862421509:2086] [node 4] ICS10 traffic start
2025-11-28T14:53:39.004150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:38.999841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789549212222299:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:38.999914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:38.999964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789549212222299:2050] [node 3] ICP22 created new session: [6:7577789549212222392:2050]
2025-11-28T14:53:39.000015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789549212222392:2050] [node 3] ICS09 handshake done sender: [6:7577789549212222377:2116] self: [6:7577789549212222385:0] peer: [3:7577789552188784411:0] socket: 441 qp: -1
2025-11-28T14:53:39.000053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789549212222392:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:39.000087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:39.001236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789551913873509:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:39.010972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789549212222389:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:39.015163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789546319959122:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789550614926508:2117] held: no
2025-11-28T14:53:39.015817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789550614926508:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:39.013900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789550567454188:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:39.001265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789551913873509:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:39.013949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789550567454188:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:39.011414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789549212222297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:39.011529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789553507189690:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
2025-11-28T14:53:39.012291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789549212222384:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:39.019383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789549212222310:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:39.019448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:39.019505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789549212222310:2059] [node 5] ICP22 created new session: [6:7577789553507189692:2059]
2025-11-28T14:53:39.019549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789553507189692:2059] [node 5] ICS09 handshake done sender: [6:7577789549212222384:2120] self: [6:7577789549212222391:0] peer: [5:7577789551913873508:0] socket: 461 qp: -1
2025-11-28T14:53:39.019578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789553507189692:2059] [node 5] ICS10 traffic start
2025-11-28T14:53:39.019609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:39.011567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789551913873509:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:39.017653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789551913873501:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:39.017710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:39.017756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789551913873501:2068] [node 6] ICP22 created new session: [5:7577789556208840832:2068]
2025-11-28T14:53:39.017803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789556208840832:2068] [node 6] ICS09 handshake done sender: [5:7577789551913873509:2114] self: [5:7577789551913873508:0] peer: [6:7577789549212222391:0] socket: 459 qp: -1
2025-11-28T14:53:39.017858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789556208840832:2068] [node 6] ICS10 traffic start
2025-11-28T14:53:39.017901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:39.021603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789552188784421:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:39.021384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789550614926510:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:39.022493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789550567454188:2114] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:39.021794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789546319959122:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:39.021846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:39.021898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789546319959122:2048] [node 1] ICP22 created new session: [4:7577789554909893809:2048]
2025-11-28T14:53:39.021940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789554909893809:2048] [node 1] ICS09 handshake done sender: [4:7577789550614926508:2117] self: [4:7577789550614926511:0] peer: [1:7577789550567454185:0] socket: 457 qp: -1
2025-11-28T14:53:39.021968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789554909893809:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:39.022002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:39.024899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789556208840831:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19507
2025-11-28T14:53:39.026294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789546319959124:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:39.022927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789552188784371:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:39.023013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:39.023066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789552188784371:2067] [node 5] ICP22 created new session: [3:7577789556483751740:2067]
2025-11-28T14:53:39.023108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789556483751740:2067] [node 5] ICS09 handshake done sender: [3:7577789552188784421:2118] self: [3:7577789552188784420:0] peer: [5:7577789551913873533:0] socket: 463 qp: -1
2025-11-28T14:53:39.023137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789556483751740:2067] [node 5] ICS10 traffic start
2025-11-28T14:53:39.023495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:39.029103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789550567454178:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:39.029201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:39.029249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789550567454178:2087] [node 6] ICP22 created new session: [1:7577789554862421512:2087]
2025-11-28T14:53:39.029299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789554862421512:2087] [node 6] ICS09 handshake done sender: [1:7577789550567454188:2114] self: [1:7577789550567454187:0] peer: [6:7577789553507189691:0] socket: 412 qp: -1
2025-11-28T14:53:39.029340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789554862421512:2087] [node 6] ICS10 traffic start
2025-11-28T14:53:39.029374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:39.032008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789551913873532:2122] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:39.032645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789552188784417:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:39.032689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789552188784417:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:39.034204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789552188784417:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:39.036895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789554909893812:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23868
2025-11-28T14:53:39.037367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789550614926510:2119] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:39.037548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789546319959124:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:39.037625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:39.037674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789546319959124:2050] [node 3] ICP22 created new session: [4:7577789554909893813:2050]
2025-11-28T14:53:39.037720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789554909893813:2050] [node 3] ICS09 handshake done sender: [4:7577789550614926510:2119] self: [4:7577789554909893811:0] peer: [3:7577789552188784416:0] socket: 462 qp: -1
2025-11-28T14:53:39.037751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789554909893813:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:39.037786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:39.039626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789552188784374:2068] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:39.039718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:39.039775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789552188784374:2068] [node 4] ICP22 created new session: [3:7577789556483751743:2068]
2025-11-28T14:53:39.039828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789556483751743:2068] [node 4] ICS09 handshake done sender: [3:7577789552188784417:2115] self: [3:7577789552188784416:0] peer: [4:7577789554909893811:0] socket: 436 qp: -1
2025-11-28T14:53:39.039864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789556483751743:2068] [node 4] ICS10 traffic start
2025-11-28T14:53:39.039895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:39.041951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789547618906142:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:39.043501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:39.043580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789547618906142:2050] [node 3] ICP22 created new session: [5:7577789556208840834:2050]
2025-11-28T14:53:39.043630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789556208840834:2050] [node 3] ICS09 handshake done sender: [5:7577789551913873532:2122] self: [5:7577789551913873533:0] peer: [3:7577789552188784420:0] socket: 464 qp: -1
2025-11-28T14:53:39.043666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789556208840834:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:39.044041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789549212222297:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789549212222389:2121] held: no
2025-11-28T14:53:39.044842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789553507189694:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1857
2025-11-28T14:53:39.046704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:39.053327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789549212222389:2121] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:39.053634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789549212222297:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:39.053712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:39.053770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789549212222297:2048] [node 1] ICP22 created new session: [6:7577789553507189695:2048]
2025-11-28T14:53:39.053816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789553507189695:2048] [node 1] ICS09 handshake done sender: [6:7577789549212222389:2121] self: [6:7577789553507189691:0] peer: [1:7577789550567454187:0] socket: 414 qp: -1
2025-11-28T14:53:39.053858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789553507189695:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:39.053894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:39.089476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789547618906142:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:39.114232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:39.114506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789548624950863:2048] [node 1] ICS07 socket disconnect 419 reason# EndOfStream
2025-11-28T14:53:39.114556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789548624950863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.114648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:39.114698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789548624950863:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:39.116707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:39.117045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789552188784437:2048] [node 1] ICS07 socket disconnect 423 reason# EndOfStream
2025-11-28T14:53:39.117076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789552188784437:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.117242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:39.117295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789552188784437:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:39.120006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789544329983474:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:39.120058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789548624950863:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:39.121959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789544329983474:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.122010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789548624950863:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.122048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789544329983474:2048] [node 1] ICP30 unregister session Session# [2:7577789548624950863:2048] VirtualId# [2:7577789548624950861:0]
2025-11-28T14:53:39.122098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:39.122131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789544329983474:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.122148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.121299Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:39.123224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789551913873530:2048] [node 1] ICS07 socket disconnect 421 reason# EndOfStream
2025-11-28T14:53:39.123280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789551913873530:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.123365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:39.123415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789551913873530:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:39.124193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789547618906140:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.124256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789551913873530:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.124294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789547618906140:2048] [node 1] ICP30 unregister session Session# [5:7577789551913873530:2048] VirtualId# [5:7577789551913873522:0]
2025-11-28T14:53:39.124337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:39.124388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789547618906140:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.124405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.126430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789547893817046:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:39.127220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789552188784437:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:39.127895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:39.128120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789554909893809:2048] [node 1] ICS01 socket: 457 reason# EndOfStream
2025-11-28T14:53:39.128170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789546319959122:2048] [node 1] ICP30 unregister session Session# [4:7577789554909893809:2048] VirtualId# [4:7577789550614926511:0]
2025-11-28T14:53:39.128209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789554909893809:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.128287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:39.128328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:39.128766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789547893817046:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.128820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789552188784437:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.128869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789547893817046:2048] [node 1] ICP30 unregister session Session# [3:7577789552188784437:2048] VirtualId# [3:7577789552188784428:0]
2025-11-28T14:53:39.128924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:39.128964Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789547893817046:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.128984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.129499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:39.129632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789553507189695:2048] [node 1] ICS01 socket: 414 reason# EndOfStream
2025-11-28T14:53:39.129671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789549212222297:2048] [node 1] ICP30 unregister session Session# [6:7577789553507189695:2048] VirtualId# [6:7577789553507189691:0]
2025-11-28T14:53:39.129691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789553507189695:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.129742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:39.129769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:39.133714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789547618906140:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:39.138157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789547893817046:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:39.161863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:39.162106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789552188784433:2049] [node 2] ICS07 socket disconnect 407 reason# EndOfStream
2025-11-28T14:53:39.162135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789552188784433:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.162219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:39.162258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789552188784433:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:39.165716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:39.166014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789551913873523:2049] [node 2] ICS01 socket: 399 reason# EndOfStream
2025-11-28T14:53:39.166055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789547618906141:2049] [node 2] ICP30 unregister session Session# [5:7577789551913873523:2049] VirtualId# [5:7577789551913873519:0]
2025-11-28T14:53:39.166088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789551913873523:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.166170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:39.166210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:39.168330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789547893817047:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:39.168391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789552188784433:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:39.172242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:39.172345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789547893817047:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.172407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789552188784433:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.172544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789549212222382:2049] [node 2] ICS07 socket disconnect 445 reason# EndOfStream
2025-11-28T14:53:39.172797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789547893817047:2049] [node 2] ICP30 unregister session Session# [3:7577789552188784433:2049] VirtualId# [3:7577789552188784426:0]
2025-11-28T14:53:39.172575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789549212222382:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.171857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:39.172642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:39.172681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789549212222382:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:39.172035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789550614926500:2049] [node 2] ICS07 socket disconnect 400 reason# EndOfStream
2025-11-28T14:53:39.172063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789550614926500:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.172137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:39.172170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789550614926500:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:39.182231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789546319959123:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.182288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789550614926500:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.182319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789546319959123:2049] [node 2] ICP30 unregister session Session# [4:7577789550614926500:2049] VirtualId# [4:7577789550614926497:0]
2025-11-28T14:53:39.182418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:39.182439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789546319959123:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.182450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.177895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:39.177940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789547893817047:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.177952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.182962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789549212222298:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.183005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789549212222382:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.183027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789549212222298:2049] [node 2] ICP30 unregister session Session# [6:7577789549212222382:2049] VirtualId# [6:7577789549212222379:0]
2025-11-28T14:53:39.183117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:39.183146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789549212222298:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.183159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.189430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789549212222298:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:39.199952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789547893817047:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:39.200218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789546319959123:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:39.236723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:39.236799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:39.237052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789556208840834:2050] [node 3] ICS07 socket disconnect 464 reason# EndOfStream
2025-11-28T14:53:39.237090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789556208840834:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.236977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789549212222392:2050] [node 3] ICS01 socket: 441 reason# EndOfStream
2025-11-28T14:53:39.237020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789549212222299:2050] [node 3] ICP30 unregister session Session# [6:7577789549212222392:2050] VirtualId# [6:7577789549212222385:0]
2025-11-28T14:53:39.237216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:39.237296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789556208840834:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:39.236723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:39.236977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789554909893813:2050] [node 3] ICS07 socket disconnect 462 reason# EndOfStream
2025-11-28T14:53:39.237006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789554909893813:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.237052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789549212222392:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.239176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789547618906142:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.239247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789556208840834:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.239282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789547618906142:2050] [node 3] ICP30 unregister session Session# [5:7577789556208840834:2050] VirtualId# [5:7577789551913873533:0]
2025-11-28T14:53:39.239374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:39.239403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789547618906142:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.239417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.237104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:39.237141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:39.237092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:39.237135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789554909893813:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:39.238213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789546319959124:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.238291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789554909893813:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.238320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789546319959124:2050] [node 3] ICP30 unregister session Session# [4:7577789554909893813:2050] VirtualId# [4:7577789554909893811:0]
2025-11-28T14:53:39.238356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:39.238397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789546319959124:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.238414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.248924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789547618906142:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:39.278622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:39.281217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:39.281438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789551913873526:2071] [node 4] ICS07 socket disconnect 409 reason# EndOfStream
2025-11-28T14:53:39.281471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789551913873526:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.281551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:39.281591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789551913873526:2071] [node 4] ICS15 start handshake
2025-11-28T14:53:39.278856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789549212222387:2063] [node 4] ICS07 socket disconnect 406 reason# EndOfStream
2025-11-28T14:53:39.278893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789549212222387:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.278973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:39.279012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789549212222387:2063] [node 4] ICS15 start handshake
2025-11-28T14:53:39.282347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789549212222363:2063] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15453) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.282406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789549212222387:2063] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.282446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789549212222363:2063] [node 4] ICP30 unregister session Session# [6:7577789549212222387:2063] VirtualId# [6:7577789549212222376:0]
2025-11-28T14:53:39.282490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:39.282548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789549212222363:2063] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.282569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.283294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789551913873502:2071] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15453) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:39.283348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789551913873526:2071] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:39.283386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789551913873502:2071] [node 4] ICP30 unregister session Session# [5:7577789551913873526:2071] VirtualId# [5:7577789551913873518:0]
2025-11-28T14:53:39.283478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:39.283505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789551913873502:2071] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.283524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:39.294275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789549212222363:2063] [node 4] ICP33 wake up from error state
2025-11-28T14:53:39.294542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789551913873502:2071] [node 4] ICP33 wake up from error state
2025-11-28T14:53:39.354688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:39.354936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789553507189692:2059] [node 5] ICS07 socket disconnect 461 reason# EndOfStream
2025-11-28T14:53:39.354973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789553507189692:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:39.355104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:39.355148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789553507189692:2059] [node 5] ICS15 start handshake
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:39.483695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789554677881914:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:39.483743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789554677881915:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:39.524745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789556753985775:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:39.524815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789556753985776:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:39.545693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789553604997918:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:39.545749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789553604997919:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:39.581990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789555217569414:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:39.582044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789555217569415:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:39.582077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789555217569416:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:39.606459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789555315212865:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:39.606510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789555315212866:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:39.606766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789555315212867:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:39.772788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789555240833855:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:39.772865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789555240833857:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:39.817552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789555240833856:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:40.141603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789561048953134:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:40.141975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789561048953135:2108] [node 6] ICP01 ready to work
2025-11-28T14:53:40.142567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789561048953136:2109] [node 5] ICP01 ready to work
2025-11-28T14:53:40.136425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789559535801170:2066] [node 5] ICP01 ready to work
2025-11-28T14:53:40.245105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789557899965277:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:40.245473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789557899965278:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:40.256558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789555240833855:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(82323564ns) {Configure:104(82314372ns) {ProcessConfigured:135(44077908ns) {StartInitialHandshake:157(44074385ns) {DropHandshakes:470(2768ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(927ns) PrepareNewSessionHandshake:477(5936ns) {SwitchToState:179(2459ns)}}}}}
2025-11-28T14:53:40.290924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789561048953135:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(106018473ns) {Configure:104(106011532ns) {ProcessConfigured:135(98693ns) {StartInitialHandshake:157(94911ns) {DropHandshakes:470(4035ns) {DropIncomingHandshake:442(1045ns) DropOutgoingHandshake:457(619ns)} GenerateSessionVirtualId:988(1621ns) PrepareNewSessionHandshake:477(2339ns) {SwitchToState:179(880ns)}}}}}
2025-11-28T14:53:40.276647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789559610180232:2106] [node 4] ICP01 ready to work
2025-11-28T14:53:40.279167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789559512536774:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:40.277538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789559610180233:2108] [node 6] ICP01 ready to work
2025-11-28T14:53:40.336420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789558972849273:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:40.344501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789558972849274:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:40.352770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789555240833856:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(117422257ns) {Configure:104(117417558ns) {ProcessConfigured:135(44073871ns) {StartInitialHandshake:157(44069398ns) {DropHandshakes:470(3335ns) {DropIncomingHandshake:442(686ns) DropOutgoingHandshake:457(564ns)} GenerateSessionVirtualId:988(723ns) PrepareNewSessionHandshake:477(6477ns) {SwitchToState:179(2636ns)}}}}}
2025-11-28T14:53:40.367793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789559512536775:2108] [node 5] ICP01 ready to work
2025-11-28T14:53:40.317119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789559610180232:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(38734634ns) {Configure:104(38729044ns) {ProcessConfigured:135(84125ns) {StartInitialHandshake:157(81916ns) {DropHandshakes:470(3590ns) {DropIncomingHandshake:442(1073ns) DropOutgoingHandshake:457(619ns)} GenerateSessionVirtualId:988(1389ns) PrepareNewSessionHandshake:477(2301ns) {SwitchToState:179(1038ns)}}}}}
2025-11-28T14:53:40.353372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789559535801220:2076] [node 4] ICP01 ready to work
2025-11-28T14:53:40.364455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789557899965283:2107] [node 4] ICP01 ready to work
2025-11-28T14:53:40.374822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789559535801220:2076] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19855813ns) {Configure:104(19848200ns) {ProcessConfigured:135(74462ns) {StartInitialHandshake:157(71293ns) {DropHandshakes:470(3092ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(434ns)} GenerateSessionVirtualId:988(1397ns) PrepareNewSessionHandshake:477(2713ns) {SwitchToState:179(997ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.375205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801221:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.375567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789558972849275:2109] [node 6] ICP01 ready to work
2025-11-28T14:53:40.386807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789553604997918:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(134260248ns) {Configure:104(134255955ns) {ProcessConfigured:135(3798511ns) {StartInitialHandshake:157(3794667ns) {DropHandshakes:470(3783ns) {DropIncomingHandshake:442(838ns) DropOutgoingHandshake:457(675ns)} GenerateSessionVirtualId:988(714ns) PrepareNewSessionHandshake:477(4024ns) {SwitchToState:179(2050ns)}}}}}
2025-11-28T14:53:40.388839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789557899965278:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19287759ns) {Configure:104(19281579ns) {ProcessConfigured:135(64184ns) {StartInitialHandshake:157(61615ns) {DropHandshakes:470(1483ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(464ns) PrepareNewSessionHandshake:477(1056ns) {SwitchToState:179(545ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.411870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789555217569414:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25021733ns) {Configure:104(25015655ns) {ProcessConfigured:135(73292ns) {StartInitialHandshake:157(69655ns) {DropHandshakes:470(3890ns) {DropIncomingHandshake:442(837ns) DropOutgoingHandshake:457(1021ns)} GenerateSessionVirtualId:988(985ns) PrepareNewSessionHandshake:477(2163ns) {SwitchToState:179(920ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.400569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789555315212866:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(287148093ns) {Configure:104(269403078ns) {ProcessConfigured:135(1019322ns) {StartInitialHandshake:157(1017862ns) {DropHandshakes:470(3410ns) {DropIncomingHandshake:442(893ns) DropOutgoingHandshake:457(688ns)} GenerateSessionVirtualId:988(530ns) PrepareNewSessionHandshake:477(5016ns) {SwitchToState:179(1296ns)}}}}}
2025-11-28T14:53:40.403861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789561048953148:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31320
2025-11-28T14:53:40.404298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789555240833857:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(139454599ns) {Configure:104(139444574ns) {ProcessConfigured:135(6655817ns) {StartInitialHandshake:157(6650254ns) {DropHandshakes:470(3118ns) {DropIncomingHandshake:442(859ns) DropOutgoingHandshake:457(500ns)} GenerateSessionVirtualId:988(2489ns) PrepareNewSessionHandshake:477(3406ns) {SwitchToState:179(1702ns)}}}}}
2025-11-28T14:53:40.427147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789557899965288:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51530
2025-11-28T14:53:40.420563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789561048953136:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(207563879ns) {Configure:104(207555897ns) {ProcessConfigured:135(6708223ns) {StartInitialHandshake:157(6706309ns) {DropHandshakes:470(822ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(331ns) PrepareNewSessionHandshake:477(2233ns) {SwitchToState:179(849ns)}}}}}
2025-11-28T14:53:40.406766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789559610180245:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46578
2025-11-28T14:53:40.427409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789557899965289:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51530
2025-11-28T14:53:40.431796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789557899965289:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.442339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789557899965288:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.448551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789559610180245:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.440792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789559535801221:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.450970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789559610180248:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46578
2025-11-28T14:53:40.427473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789561048953151:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31320
2025-11-28T14:53:40.444744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789555240833856:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.442706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789561048953148:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.449819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789561048953138:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.449859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789561048953138:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.458207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789553604997919:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.456399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789561048953138:2110] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:40.456843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789561048953135:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.456933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:40.456987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789561048953135:2108] [node 6] ICP22 created new session: [2:7577789561048953154:2108]
2025-11-28T14:53:40.457033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789561048953154:2108] [node 6] ICS09 handshake done sender: [2:7577789561048953138:2110] self: [2:7577789561048953137:0] peer: [6:7577789559535801230:0] socket: 402 qp: -1
2025-11-28T14:53:40.457068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789561048953154:2108] [node 6] ICS10 traffic start
2025-11-28T14:53:40.457102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:40.459229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789555240833856:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789559535801221:2112] held: no
2025-11-28T14:53:40.455066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789555217569416:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(66559501ns) {Configure:104(66554014ns) {ProcessConfigured:135(8683768ns) {StartInitialHandshake:157(8680914ns) {DropHandshakes:470(1785ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:477(3968ns) {SwitchToState:179(2587ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.460207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801231:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.457980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789554677881915:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(81157086ns) {Configure:104(81151399ns) {ProcessConfigured:135(12435203ns) {StartInitialHandshake:157(12431751ns) {DropHandshakes:470(1670ns) {DropIncomingHandshake:442(463ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(777ns) PrepareNewSessionHandshake:477(3915ns) {SwitchToState:179(1455ns)}}}}}
2025-11-28T14:53:40.461079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789561048953140:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.461125Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789561048953140:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.461817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789559610180248:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.465551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789561048953140:2111] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:40.463546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789555315212867:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.464902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789559512536775:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(43482481ns) {Configure:104(43477402ns) {ProcessConfigured:135(7819431ns) {StartInitialHandshake:157(7815781ns) {DropHandshakes:470(3929157ns) {DropIncomingHandshake:442(1214ns) DropOutgoingHandshake:457(527ns)} GenerateSessionVirtualId:988(1190ns) PrepareNewSessionHandshake:477(2027ns) {SwitchToState:179(706ns)}}}}}
2025-11-28T14:53:40.469451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789559512536784:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14922
2025-11-28T14:53:40.469679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789559512536787:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14922
2025-11-28T14:53:40.467074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789557899965282:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.467115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789557899965282:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.468409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789561048953151:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.485458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789558972849275:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29562066ns) {Configure:104(29557252ns) {ProcessConfigured:135(128022ns) {StartInitialHandshake:157(124002ns) {DropHandshakes:470(3149ns) {DropIncomingHandshake:442(937ns) DropOutgoingHandshake:457(551ns)} GenerateSessionVirtualId:988(1272ns) PrepareNewSessionHandshake:477(2154ns) {SwitchToState:179(837ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.470022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789559535801221:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:40.468500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789556753985776:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.499018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789555240833856:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.506127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789557899965297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51530
2025-11-28T14:53:40.499111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:40.494542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789561048953159:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31320
2025-11-28T14:53:40.516933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789557899965282:2111] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:40.513379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789559512536784:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.520144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789559610180251:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46578
2025-11-28T14:53:40.528140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789557899965277:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.495951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:40.499169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789555240833856:2049] [node 2] ICP22 created new session: [6:7577789559535801232:2049]
2025-11-28T14:53:40.526196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789559512536787:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.520227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789559610180253:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46578
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.509474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789556753985776:2049] [node 3] ICP22 created new session: [2:7577789561048953158:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.547573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789558972849274:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(58471292ns) {Configure:104(58464331ns) {ProcessConfigured:135(65051ns) {StartInitialHandshake:157(61867ns) {DropHandshakes:470(3520ns) {DropIncomingHandshake:442(921ns) DropOutgoingHandshake:457(583ns)} GenerateSessionVirtualId:988(725ns) PrepareNewSessionHandshake:477(9714ns) {SwitchToState:179(579ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.499235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789559535801232:2049] [node 2] ICS09 handshake done sender: [6:7577789559535801221:2112] self: [6:7577789559535801230:0] peer: [2:7577789561048953137:0] socket: 408 qp: -1
2025-11-28T14:53:40.528343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789559610180253:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.526504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789559512536791:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14922
2025-11-28T14:53:40.549538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789558972849276:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-28T14:53:40.550289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789558972849278:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-28T14:53:40.550456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789558972849281:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-28T14:53:40.550569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789558972849277:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-28T14:53:40.552607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849278:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.513368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789561048953158:2049] [node 3] ICS09 handshake done sender: [2:7577789561048953140:2111] self: [2:7577789561048953139:0] peer: [3:7577789557899965296:0] socket: 403 qp: -1
2025-11-28T14:53:40.536629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:40.529106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789555315212866:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.526963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789555217569415:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.499284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789559535801232:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:40.545344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789559512536794:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14922
2025-11-28T14:53:40.554100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849277:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.537009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789557899965288:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:40.543410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789557899965277:2108] [node 5] ICP22 created new session: [3:7577789557899965300:2108]
2025-11-28T14:53:40.545528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789557899965300:2108] [node 5] ICS09 handshake done sender: [3:7577789557899965282:2111] self: [3:7577789557899965281:0] peer: [5:7577789559610180250:0] socket: 425 qp: -1
2025-11-28T14:53:40.545573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789557899965300:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:40.545624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:40.499341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:40.564325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789559512536791:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.568008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789559512536787:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.546858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789559610180252:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46578
2025-11-28T14:53:40.572626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849276:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.513422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789561048953158:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.568096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789553604997919:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.516521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.568584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789559512536795:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14922
2025-11-28T14:53:40.554304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789559610180248:2124] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:40.568211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:40.580485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801235:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.578195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789555217569415:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.527288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789561048953144:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.555209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789559610180251:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.568261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789553604997919:2049] [node 2] ICP22 created new session: [3:7577789557899965302:2049]
2025-11-28T14:53:40.592927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:40.555295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789555315212867:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.527315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789561048953144:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.609363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849283:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.609397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789558972849283:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.568939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789559610180232:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.576046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:40.578519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789555315212867:2050] [node 3] ICP22 created new session: [5:7577789559610180258:2050]
2025-11-28T14:53:40.580820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789559610180258:2050] [node 3] ICS09 handshake done sender: [5:7577789559610180248:2124] self: [5:7577789559610180250:0] peer: [3:7577789557899965281:0] socket: 426 qp: -1
2025-11-28T14:53:40.581479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789559610180257:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46578
2025-11-28T14:53:40.581939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789559610180258:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:40.593114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:40.610609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789559610180253:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:40.568326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789557899965302:2049] [node 2] ICS09 handshake done sender: [3:7577789557899965288:2114] self: [3:7577789557899965296:0] peer: [2:7577789561048953139:0] socket: 419 qp: -1
2025-11-28T14:53:40.612727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789559535801235:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.612936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789559535801170:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.593267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789559512536783:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.593297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789559512536783:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.604178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789555217569415:2049] [node 2] ICP22 created new session: [4:7577789559512536797:2049]
2025-11-28T14:53:40.610981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789559512536797:2049] [node 2] ICS09 handshake done sender: [4:7577789559512536787:2117] self: [4:7577789559512536793:0] peer: [2:7577789561048953143:0] socket: 434 qp: -1
2025-11-28T14:53:40.568378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789557899965302:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:40.611025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789559512536797:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:40.611087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:40.568441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:40.612821Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789553604997919:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(43645822ns) {HandleHandshakeStatus:329(43642174ns) {DropIncomingHandshake:442(1021ns) DropOutgoingHandshake:457(4960ns) SwitchToState:179(860ns) ProcessPendingSessionEvents:495(43135991ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43051128ns max# 9531233ns}}}
2025-11-28T14:53:40.613559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789557899965299:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51530
2025-11-28T14:53:40.528987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789561048953144:2113] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:40.555773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789561048953150:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.555803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789561048953150:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.563283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789561048953134:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.563335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:40.563407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789561048953134:2107] [node 4] ICP22 created new session: [2:7577789561048953163:2107]
2025-11-28T14:53:40.563452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789561048953163:2107] [node 4] ICS09 handshake done sender: [2:7577789561048953144:2113] self: [2:7577789561048953143:0] peer: [4:7577789559512536793:0] socket: 415 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.615108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789557899965277:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(84004948ns) {HandleHandshakeStatus:329(84002250ns) {DropIncomingHandshake:442(11828ns) DropOutgoingHandshake:457(534ns) SwitchToState:179(591ns) ProcessPendingSessionEvents:495(66298127ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 61362271ns max# 11381191ns}}}
2025-11-28T14:53:40.617460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789557899965304:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51530
2025-11-28T14:53:40.563504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789561048953163:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.628581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789557899965299:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.629092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801237:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.613462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789558972849283:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:40.618425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789559610180241:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.563553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:40.618467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789559610180241:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.618887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789555315212866:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.632779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:40.619403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789555217569415:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(33974153ns) {HandleHandshakeStatus:329(33972147ns) {DropIncomingHandshake:442(581ns) DropOutgoingHandshake:457(5482ns) SwitchToState:179(622ns) ProcessPendingSessionEvents:495(955743ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 899517ns max# 493843ns}}}
2025-11-28T14:53:40.613810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849281:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.634297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789555315212866:2049] [node 2] ICP22 created new session: [5:7577789559610180260:2049]
2025-11-28T14:53:40.573850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789561048953159:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.574403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789561048953165:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31320
2025-11-28T14:53:40.574552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789561048953150:2115] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:40.574965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789561048953134:2107] [node 4] ICP18 handshake (actor [2:7577789561048953159:2117]) is too old
2025-11-28T14:53:40.575017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789561048953136:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.575056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:40.575104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789561048953136:2109] [node 5] ICP22 created new session: [2:7577789561048953167:2109]
2025-11-28T14:53:40.575173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789561048953167:2109] [node 5] ICS09 handshake done sender: [2:7577789561048953150:2115] self: [2:7577789561048953149:0] peer: [5:7577789559610180254:0] socket: 441 qp: -1
2025-11-28T14:53:40.575214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789561048953167:2109] [node 5] ICS10 traffic start
2025-11-28T14:53:40.575256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:40.581995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789556753985776:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(77125576ns) {HandleHandshakeStatus:329(77122235ns) {DropIncomingHandshake:442(202ns) DropOutgoingHandshake:457(590ns) SwitchToState:179(795ns) ProcessPendingSessionEvents:495(17028872ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16951237ns max# 8119268ns}}}
2025-11-28T14:53:40.596483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789556753985776:2049] [node 3] ICP18 handshake (actor [2:7577789561048953151:2116]) is too old
2025-11-28T14:53:40.608345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789561048953165:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.609074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789556753985775:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.636472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789559535801237:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.632932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789557899965304:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.625816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789559512536795:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.629653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789554677881914:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.658345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789556753985775:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789561048953165:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.661107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789561048953170:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31320
2025-11-28T14:53:40.640861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789559610180260:2049] [node 2] ICS09 handshake done sender: [5:7577789559610180253:2127] self: [5:7577789559610180254:0] peer: [2:7577789561048953149:0] socket: 444 qp: -1
2025-11-28T14:53:40.635415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789553604997918:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.657473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789557899965292:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.657500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789557899965292:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.662491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789557899965292:2116] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:40.660777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789559535801220:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.648474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789559610180260:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:40.632822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849287:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.628358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789559512536800:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14922
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.652451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:40.628562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789555217569414:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.632853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789558972849287:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.663135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801239:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.688993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789557899965283:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.696079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:40.696156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789557899965283:2107] [node 4] ICP22 created new session: [3:7577789557899965311:2107]
2025-11-28T14:53:40.672260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801240:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.678664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789559535801237:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:40.696222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789557899965311:2107] [node 4] ICS09 handshake done sender: [3:7577789557899965292:2116] self: [3:7577789557899965291:0] peer: [4:7577789559512536804:0] socket: 458 qp: -1
2025-11-28T14:53:40.696256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789557899965311:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:40.692625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789559535801220:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.696320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:40.692695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:40.692767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789559535801220:2076] [node 4] ICP22 created new session: [6:7577789559535801241:2076]
2025-11-28T14:53:40.692834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789559535801241:2076] [node 4] ICS09 handshake done sender: [6:7577789559535801237:2118] self: [6:7577789559535801238:0] peer: [4:7577789559512536785:0] socket: 461 qp: -1
2025-11-28T14:53:40.631970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789559512536783:2114] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:40.644614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:40.677920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789555315212866:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(58146408ns) {HandleHandshakeStatus:329(58141875ns) {DropIncomingHandshake:442(1084ns) DropOutgoingHandshake:457(6535ns) SwitchToState:179(799ns) ProcessPendingSessionEvents:495(12729907ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12666461ns max# 12237201ns}}}
2025-11-28T14:53:40.679670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789561048953165:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:40.646950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789559512536800:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.650542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789555217569416:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.656535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789559512536775:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.659168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789559512536786:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.659193Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789559512536786:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.662216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789559512536786:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:40.672533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:40.678507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789559512536775:2108] [node 5] ICP22 created new session: [4:7577789559512536806:2108]
2025-11-28T14:53:40.687125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789559512536774:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.689195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789559512536806:2108] [node 5] ICS09 handshake done sender: [4:7577789559512536783:2114] self: [4:7577789559512536782:0] peer: [5:7577789559610180255:0] socket: 438 qp: -1
2025-11-28T14:53:40.695550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789559512536806:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:40.652335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789554677881914:2048] [node 2] ICP22 created new session: [1:7577789558972849298:2048]
2025-11-28T14:53:40.660606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789558972849298:2048] [node 2] ICS09 handshake done sender: [1:7577789558972849283:2115] self: [1:7577789558972849282:0] peer: [2:7577789561048953169:0] socket: 450 qp: -1
2025-11-28T14:53:40.661366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789558972849298:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:40.662959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:40.673796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849280:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.673830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789558972849280:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.675205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789558972849303:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2079
2025-11-28T14:53:40.686187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789558972849287:2117] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:40.706646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789558972849280:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:40.706994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.692870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789559535801241:2076] [node 4] ICS10 traffic start
2025-11-28T14:53:40.687507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789555315212867:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(106172938ns) {HandleHandshakeStatus:329(106170255ns) {DropIncomingHandshake:442(1328ns) DropOutgoingHandshake:457(8107ns) SwitchToState:179(526ns) ProcessPendingSessionEvents:495(68168512ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 68052416ns max# 8150882ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.691605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789556753985775:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.691697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:40.691750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789556753985775:2048] [node 1] ICP22 created new session: [2:7577789561048953171:2048]
2025-11-28T14:53:40.691798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789561048953171:2048] [node 1] ICS09 handshake done sender: [2:7577789561048953165:2118] self: [2:7577789561048953169:0] peer: [1:7577789558972849282:0] socket: 454 qp: -1
2025-11-28T14:53:40.691838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789561048953171:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:40.691889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:40.702091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789559610180232:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19002) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789559610180251:2125] held: no
2025-11-28T14:53:40.702744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789555315212867:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:40.714819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789559610180263:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46578
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.718681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789559610180264:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46578
2025-11-28T14:53:40.718840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789559610180241:2121] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.718228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789557899965310:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51530
2025-11-28T14:53:40.710208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789559512536775:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(62214814ns) {HandleHandshakeStatus:329(57024394ns) {DropIncomingHandshake:442(26135ns) DropOutgoingHandshake:457(760ns) SwitchToState:179(821ns) ProcessPendingSessionEvents:495(585354ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 526062ns max# 51371ns}}}
2025-11-28T14:53:40.712665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.716026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789559512536774:2109] [node 6] ICP22 created new session: [4:7577789559512536810:2109]
2025-11-28T14:53:40.716095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789559512536810:2109] [node 6] ICS09 handshake done sender: [4:7577789559512536786:2116] self: [4:7577789559512536785:0] peer: [6:7577789559535801238:0] socket: 435 qp: -1
2025-11-28T14:53:40.716132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789559512536810:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:40.716207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:40.720101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789559512536774:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(32044385ns) {HandleHandshakeStatus:329(32041858ns) {DropIncomingHandshake:442(16918ns) DropOutgoingHandshake:457(642ns) SwitchToState:179(832ns) ProcessPendingSessionEvents:495(2839186ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 2771790ns max# 2350227ns}}}
2025-11-28T14:53:40.720634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789559512536807:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14922
2025-11-28T14:53:40.719433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789557899965283:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22811716ns) {HandleHandshakeStatus:329(22809068ns) {DropIncomingHandshake:442(10965ns) DropOutgoingHandshake:457(238ns) SwitchToState:179(810ns) ProcessPendingSessionEvents:495(15346140ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15281485ns max# 10656867ns}}}
2025-11-28T14:53:40.692924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:40.716220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789554677881914:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(85654805ns) {HandleHandshakeStatus:329(85651322ns) {DropIncomingHandshake:442(12484ns) DropOutgoingHandshake:457(541ns) SwitchToState:179(558ns) ProcessPendingSessionEvents:495(52178207ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 49977984ns max# 8452154ns}}}
2025-11-28T14:53:40.716294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789558972849274:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.726402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849303:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.727025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789554677881915:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.727147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:40.727198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789554677881915:2049] [node 3] ICP22 created new session: [1:7577789558972849304:2049]
2025-11-28T14:53:40.727244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789558972849304:2049] [node 3] ICS09 handshake done sender: [1:7577789558972849280:2113] self: [1:7577789558972849279:0] peer: [3:7577789557899965307:0] socket: 449 qp: -1
2025-11-28T14:53:40.727285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789558972849304:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:40.727336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:40.732719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.708157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801244:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.720290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789557899965313:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51530
2025-11-28T14:53:40.722663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789559610180233:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.739109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789558972849274:2107] [node 4] ICP22 created new session: [1:7577789558972849307:2107]
2025-11-28T14:53:40.725773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789559535801240:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.733965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789559512536808:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14922
2025-11-28T14:53:40.732061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789557899965313:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.726104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789559535801235:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:40.733579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789559535801170:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.733675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:40.733727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789559535801170:2066] [node 5] ICP22 created new session: [6:7577789559535801245:2066]
2025-11-28T14:53:40.734237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789559535801245:2066] [node 5] ICS09 handshake done sender: [6:7577789559535801235:2117] self: [6:7577789559535801236:0] peer: [5:7577789559610180240:0] socket: 453 qp: -1
2025-11-28T14:53:40.734288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789559535801245:2066] [node 5] ICS10 traffic start
2025-11-28T14:53:40.734356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:40.736860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789555240833857:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.744257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801248:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.722777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:40.740930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789558972849307:2107] [node 4] ICS09 handshake done sender: [1:7577789558972849287:2117] self: [1:7577789558972849286:0] peer: [4:7577789559512536802:0] socket: 451 qp: -1
2025-11-28T14:53:40.742866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789558972849307:2107] [node 4] ICS10 traffic start
2025-11-28T14:53:40.746876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.732126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789557899965304:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:40.743494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789553604997918:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.743612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789557899965287:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.743643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789557899965287:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.746254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:40.746309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789553604997918:2048] [node 1] ICP22 created new session: [3:7577789557899965316:2048]
2025-11-28T14:53:40.746551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789557899965287:2113] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:40.746951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789557899965316:2048] [node 1] ICS09 handshake done sender: [3:7577789557899965304:2119] self: [3:7577789557899965307:0] peer: [1:7577789558972849279:0] socket: 457 qp: -1
2025-11-28T14:53:40.747701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789557899965316:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:40.747754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:40.748549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577789553604997918:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:40.741234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789555217569416:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19145) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789559512536800:2121] held: no
2025-11-28T14:53:40.742350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789559512536800:2121] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:40.743332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789559512536795:2120] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:40.746483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789555217569416:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.746571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:40.746617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789555217569416:2050] [node 3] ICP22 created new session: [4:7577789559512536812:2050]
2025-11-28T14:53:40.747303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789559512536812:2050] [node 3] ICS09 handshake done sender: [4:7577789559512536800:2121] self: [4:7577789559512536804:0] peer: [3:7577789557899965291:0] socket: 459 qp: -1
2025-11-28T14:53:40.747352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789559512536812:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:40.722826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789559610180233:2108] [node 6] ICP22 created new session: [5:7577789559610180265:2108]
2025-11-28T14:53:40.749047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801249:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.758054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789558972849274:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(40403134ns) {HandleHandshakeStatus:329(40400274ns) {DropIncomingHandshake:442(765ns) DropOutgoingHandshake:457(740ns) SwitchToState:179(835ns) ProcessPendingSessionEvents:495(5325154ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5228498ns max# 4786863ns}}}
2025-11-28T14:53:40.758469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789558972849274:2107] [node 4] ICP18 handshake (actor [1:7577789558972849303:2119]) is too old
2025-11-28T14:53:40.753384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789557899965278:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.747411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:40.754268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:40.754345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789557899965278:2109] [node 6] ICP22 created new session: [3:7577789557899965318:2109]
2025-11-28T14:53:40.754395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789557899965318:2109] [node 6] ICS09 handshake done sender: [3:7577789557899965287:2113] self: [3:7577789557899965286:0] peer: [6:7577789559535801247:0] socket: 462 qp: -1
2025-11-28T14:53:40.754431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789557899965318:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:40.754477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:40.752711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789559535801240:2120] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:40.722878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789559610180265:2108] [node 6] ICS09 handshake done sender: [5:7577789559610180241:2121] self: [5:7577789559610180240:0] peer: [6:7577789559535801236:0] socket: 406 qp: -1
2025-11-28T14:53:40.763133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789555240833857:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.763230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:40.763276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789555240833857:2050] [node 3] ICP22 created new session: [6:7577789559535801250:2050]
2025-11-28T14:53:40.763325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789559535801250:2050] [node 3] ICS09 handshake done sender: [6:7577789559535801240:2120] self: [6:7577789559535801247:0] peer: [3:7577789557899965286:0] socket: 463 qp: -1
2025-11-28T14:53:40.763366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789559535801250:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:40.763401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:40.759828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789555217569414:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.722908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789559610180265:2108] [node 6] ICS10 traffic start
2025-11-28T14:53:40.770066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789555240833857:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:40.722944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:40.773003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849289:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.770750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:40.773045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789558972849289:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.741049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789559610180251:2125] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:40.770823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789555217569414:2048] [node 1] ICP22 created new session: [4:7577789559512536814:2048]
2025-11-28T14:53:40.780132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789559535801248:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.780301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789555240833855:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.770872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789559512536814:2048] [node 1] ICS09 handshake done sender: [4:7577789559512536795:2120] self: [4:7577789559512536802:0] peer: [1:7577789558972849286:0] socket: 420 qp: -1
2025-11-28T14:53:40.770908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789559512536814:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:40.770967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:40.772831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577789555217569414:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:40.750520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789559610180232:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.750596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:40.779885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789558972849289:2118] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:40.781134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789558972849285:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:40.789608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789559535801253:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41781
2025-11-28T14:53:40.781183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789558972849285:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:40.750633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789559610180232:2106] [node 4] ICP22 created new session: [5:7577789559610180267:2106]
2025-11-28T14:53:40.781499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789558972849273:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.782363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:40.782425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789558972849273:2108] [node 5] ICP22 created new session: [1:7577789558972849310:2108]
2025-11-28T14:53:40.782478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789558972849310:2108] [node 5] ICS09 handshake done sender: [1:7577789558972849289:2118] self: [1:7577789558972849288:0] peer: [5:7577789559610180269:0] socket: 467 qp: -1
2025-11-28T14:53:40.782522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789558972849310:2108] [node 5] ICS10 traffic start
2025-11-28T14:53:40.782569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:40.791015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789558972849285:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:40.750668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789559610180267:2106] [node 4] ICS09 handshake done sender: [5:7577789559610180251:2125] self: [5:7577789559610180255:0] peer: [4:7577789559512536782:0] socket: 439 qp: -1
2025-11-28T14:53:40.750692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789559610180267:2106] [node 4] ICS10 traffic start
2025-11-28T14:53:40.750731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:40.769038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789559610180264:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:40.769428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789555315212865:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:40.780936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789559610180270:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46578
2025-11-28T14:53:40.784426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789559610180264:2130] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:40.785565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789555315212865:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.785656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:40.785721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789555315212865:2048] [node 1] ICP22 created new session: [5:7577789559610180271:2048]
2025-11-28T14:53:40.785768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789559610180271:2048] [node 1] ICS09 handshake done sender: [5:7577789559610180264:2130] self: [5:7577789559610180269:0] peer: [1:7577789558972849288:0] socket: 470 qp: -1
2025-11-28T14:53:40.785803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789559610180271:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:40.785849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:40.786520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789555315212865:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:40.792500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789559535801248:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:40.792889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789555240833855:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:40.792967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:40.793012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789555240833855:2048] [node 1] ICP22 created new session: [6:7577789559535801254:2048]
2025-11-28T14:53:40.793058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789559535801254:2048] [node 1] ICS09 handshake done sender: [6:7577789559535801248:2122] self: [6:7577789559535801252:0] peer: [1:7577789558972849284:0] socket: 424 qp: -1
2025-11-28T14:53:40.793091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789559535801254:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:40.793141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:40.797503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789558972849275:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:40.797592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:40.797648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789558972849275:2109] [node 6] ICP22 created new session: [1:7577789558972849313:2109]
2025-11-28T14:53:40.797696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789558972849313:2109] [node 6] ICS09 handshake done sender: [1:7577789558972849285:2116] self: [1:7577789558972849284:0] peer: [6:7577789559535801252:0] socket: 412 qp: -1
2025-11-28T14:53:40.797728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789558972849313:2109] [node 6] ICS10 traffic start
2025-11-28T14:53:40.797775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:40.827425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:40.827674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789561048953171:2048] [node 1] ICS01 socket: 454 reason# EndOfStream
2025-11-28T14:53:40.827720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789556753985775:2048] [node 1] ICP30 unregister session Session# [2:7577789561048953171:2048] VirtualId# [2:7577789561048953169:0]
2025-11-28T14:53:40.827756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789561048953171:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.827848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:40.827893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:40.829805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:40.830022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789557899965316:2048] [node 1] ICS07 socket disconnect 457 reason# EndOfStream
2025-11-28T14:53:40.830053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789557899965316:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.830150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:40.830192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789557899965316:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:40.832990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789553604997918:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:40.833032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789557899965316:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:40.834012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789553604997918:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:40.834074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789557899965316:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:40.834114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789553604997918:2048] [node 1] ICP30 unregister session Session# [3:7577789557899965316:2048] VirtualId# [3:7577789557899965307:0]
2025-11-28T14:53:40.834220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:40.834266Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789553604997918:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.834284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.837848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:40.838109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789559512536814:2048] [node 1] ICS07 socket disconnect 420 reason# EndOfStream
2025-11-28T14:53:40.838148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789559512536814:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.838238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:40.838281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789559512536814:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:40.840684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:40.840919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789559610180271:2048] [node 1] ICS07 socket disconnect 470 reason# EndOfStream
2025-11-28T14:53:40.840955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789559610180271:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.841032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:40.841073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789559610180271:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:40.843009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789555217569414:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:40.843092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789559512536814:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:40.843165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789555217569414:2048] [node 1] ICP30 unregister session Session# [4:7577789559512536814:2048] VirtualId# [4:7577789559512536802:0]
2025-11-28T14:53:40.843260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:40.843350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789555217569414:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.843386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.842796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:40.849214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789553604997918:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:40.847183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789555315212865:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:40.850392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789559610180271:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:40.850439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789555315212865:2048] [node 1] ICP30 unregister session Session# [5:7577789559610180271:2048] VirtualId# [5:7577789559610180269:0]
2025-11-28T14:53:40.847235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789559535801254:2048] [node 1] ICS07 socket disconnect 424 reason# EndOfStream
2025-11-28T14:53:40.850487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:40.850518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789555315212865:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.850531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.847278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789559535801254:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.847386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:40.852825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789555217569414:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:40.847438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789559535801254:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:40.856698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789555240833855:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:40.857143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789559535801254:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:40.857183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789555240833855:2048] [node 1] ICP30 unregister session Session# [6:7577789559535801254:2048] VirtualId# [6:7577789559535801252:0]
2025-11-28T14:53:40.857244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:40.857292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789555240833855:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.857309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.860665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789555315212865:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:40.869954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789555240833855:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:40.881941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:40.882173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577789557899965302:2049] [node 2] ICS07 socket disconnect 419 reason# EndOfStream
2025-11-28T14:53:40.882210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789557899965302:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.882301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:40.882345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789557899965302:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:40.886244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789553604997919:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-28T14:53:40.886294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577789557899965302:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:40.887185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789553604997919:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:40.887240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789557899965302:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:40.887502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789553604997919:2049] [node 2] ICP30 unregister session Session# [3:7577789557899965302:2049] VirtualId# [3:7577789557899965296:0]
2025-11-28T14:53:40.888056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:40.888101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577789553604997919:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.888120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.891003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:40.891240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789559535801232:2049] [node 2] ICS01 socket: 408 reason# EndOfStream
2025-11-28T14:53:40.891289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789555240833856:2049] [node 2] ICP30 unregister session Session# [6:7577789559535801232:2049] VirtualId# [6:7577789559535801230:0]
2025-11-28T14:53:40.891322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789559535801232:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.891392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:40.891431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:40.891543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:40.891885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577789559512536797:2049] [node 2] ICS07 socket disconnect 434 reason# EndOfStream
2025-11-28T14:53:40.891921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789559512536797:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.891997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:40.892038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577789559512536797:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:40.892924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789555217569415:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:40.893313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789559512536797:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:40.893359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789555217569415:2049] [node 2] ICP30 unregister session Session# [4:7577789559512536797:2049] VirtualId# [4:7577789559512536793:0]
2025-11-28T14:53:40.893473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:40.893512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577789555217569415:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.893527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.894380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577789553604997919:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:40.900777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:40.901171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789559610180260:2049] [node 2] ICS07 socket disconnect 444 reason# EndOfStream
2025-11-28T14:53:40.901222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789559610180260:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.901393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:40.901451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789559610180260:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:40.910251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577789555217569415:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:40.909000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789555315212866:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:40.909078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789559610180260:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:40.909119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789555315212866:2049] [node 2] ICP30 unregister session Session# [5:7577789559610180260:2049] VirtualId# [5:7577789559610180254:0]
2025-11-28T14:53:40.909227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:40.909277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789555315212866:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.909298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:40.920535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789555315212866:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:40.992217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:40.992570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789559535801250:2050] [node 3] ICS07 socket disconnect 463 reason# EndOfStream
2025-11-28T14:53:40.992611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789559535801250:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:40.992696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:40.992735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789559535801250:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:41.004535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:41.004814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789559610180258:2050] [node 3] ICS07 socket disconnect 426 reason# EndOfStream
2025-11-28T14:53:41.004865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789559610180258:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:41.004987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:41.005033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789559610180258:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:41.020262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789555315212867:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19145) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:41.020334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789559610180258:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:41.020377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789555315212867:2050] [node 3] ICP30 unregister session Session# [5:7577789559610180258:2050] VirtualId# [5:7577789559610180250:0]
2025-11-28T14:53:41.020513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:41.020549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789555315212867:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19145) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:41.020563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19145) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:41.020062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789555240833857:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19145) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:41.021256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789559535801250:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:41.021294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789555240833857:2050] [node 3] ICP30 unregister session Session# [6:7577789559535801250:2050] VirtualId# [6:7577789559535801247:0]
2025-11-28T14:53:41.021327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:41.021367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789555240833857:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19145) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:41.021380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19145) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:41.036631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789555240833857:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:41.037143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789555315212867:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:41.085353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:41.084613Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:41.092952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789559535801241:2076] [node 4] ICS01 socket: 461 reason# EndOfStream
2025-11-28T14:53:41.093019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789559535801220:2076] [node 4] ICP30 unregister session Session# [6:7577789559535801241:2076] VirtualId# [6:7577789559535801238:0]
2025-11-28T14:53:41.093065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789559535801241:2076] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:41.093177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:41.093226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:41.084876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789559610180267:2106] [node 4] ICS01 socket: 439 reason# EndOfStream
2025-11-28T14:53:41.084928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789559610180232:2106] [node 4] ICP30 unregister session Session# [5:7577789559610180267:2106] VirtualId# [5:7577789559610180255:0]
2025-11-28T14:53:41.084963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789559610180267:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:41.085110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:41.085207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:41.168583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:41.168837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789559535801245:2066] [node 5] ICS01 socket: 453 reason# EndOfStream
2025-11-28T14:53:41.168888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789559535801170:2066] [node 5] ICP30 unregister session Session# [6:7577789559535801245:2066] VirtualId# [6:7577789559535801236:0]
2025-11-28T14:53:41.168926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789559535801245:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:41.169040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:41.169111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-28T14:53:41.290768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789561978238190:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:41.291356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789561978238189:2048] [node 2] ICP01 ready to work
2025-11-28T14:53:41.309611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789561665735078:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:41.309661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789561665735079:2049] [node 3] ICP01 ready to work
2025-11-28T14:53:41.330053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789565101426080:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:41.330100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789565101426081:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:41.354872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789563899083448:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:41.354919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789563899083449:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:41.354958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789563899083450:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:41.689229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789561545402931:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:41.689294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789561545402932:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:41.689333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789561545402933:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:41.718880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789562499748916:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:41.718923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789562499748917:2049] [node 2] ICP01 ready to work
2025-11-28T14:53:41.718960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789562499748918:2050] [node 3] ICP01 ready to work
2025-11-28T14:53:42.150840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789565101426081:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21764476ns) {Configure:104(21760046ns) {ProcessConfigured:135(81676ns) {StartInitialHandshake:157(78731ns) {DropHandshakes:470(2404ns) {DropIncomingHandshake:442(598ns) DropOutgoingHandshake:457(513ns)} GenerateSessionVirtualId:988(741ns) PrepareNewSessionHandshake:477(2057ns) {SwitchToState:179(1024ns)}}}}}
2025-11-28T14:53:42.113847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789561545402948:2063] [node 4] ICP01 ready to work
2025-11-28T14:53:42.188612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789569396393410:2070] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.204020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789565960702430:2076] [node 4] ICP01 ready to work
2025-11-28T14:53:42.299808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789569396393445:2071] [node 4] ICP01 ready to work
2025-11-28T14:53:42.268042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789566273205546:2092] [node 5] ICP01 ready to work
2025-11-28T14:53:42.312440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789568194050767:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3592
2025-11-28T14:53:42.284964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577789565840370250:2059] [node 6] ICP01 ready to work
2025-11-28T14:53:42.334138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577789569396393447:2069] [node 6] ICP01 ready to work
2025-11-28T14:53:42.324902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789566273205550:2091] [node 6] ICP01 ready to work
2025-11-28T14:53:42.357041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789565960702443:2080] [node 5] ICP01 ready to work
2025-11-28T14:53:42.372425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789566794716248:2064] [node 5] ICP01 ready to work
2025-11-28T14:53:42.312585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789568194050803:2065] [node 6] ICP01 ready to work
2025-11-28T14:53:42.336561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789566273205549:2090] [node 4] ICP01 ready to work
2025-11-28T14:53:42.416419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577789568194050813:2066] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.423724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789561545402933:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(139367860ns) {Configure:104(139362689ns) {ProcessConfigured:135(87066ns) {StartInitialHandshake:157(84329ns) {DropHandshakes:470(3050ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(624ns)} GenerateSessionVirtualId:988(849ns) PrepareNewSessionHandshake:477(2087ns) {SwitchToState:179(1076ns)}}}}}
2025-11-28T14:53:42.393250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789562499748918:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(458730294ns) {Configure:104(458725022ns) {ProcessConfigured:135(58897ns) {StartInitialHandshake:157(56419ns) {DropHandshakes:470(2599ns) {DropIncomingHandshake:442(637ns) DropOutgoingHandshake:457(529ns)} GenerateSessionVirtualId:988(578ns) PrepareNewSessionHandshake:477(1230ns) {SwitchToState:179(593ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.465934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789566273205553:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
2025-11-28T14:53:42.466483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789566273205558:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
2025-11-28T14:53:42.404472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789565960702444:2077] [node 6] ICP01 ready to work
2025-11-28T14:53:42.460440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789568194050767:2071] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.392427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577789566794716266:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.498383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789566273205564:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.429972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789561665735078:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(72412195ns) {Configure:104(72405932ns) {ProcessConfigured:135(66919ns) {StartInitialHandshake:157(64049ns) {DropHandshakes:470(2742ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(594ns)} GenerateSessionVirtualId:988(739ns) PrepareNewSessionHandshake:477(1678ns) {SwitchToState:179(962ns)}}}}}
2025-11-28T14:53:42.524574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205564:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.476790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789561545402931:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(525958604ns) {Configure:104(525952293ns) {ProcessConfigured:135(11566478ns) {StartInitialHandshake:157(11562414ns) {DropHandshakes:470(2295ns) {DropIncomingHandshake:442(472ns) DropOutgoingHandshake:457(368ns)} GenerateSessionVirtualId:988(862ns) PrepareNewSessionHandshake:477(4269ns) {SwitchToState:179(2060ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.548921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789566273205565:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
2025-11-28T14:53:42.549672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577789566794716248:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(155717318ns) {Configure:104(155712433ns) {ProcessConfigured:135(59475ns) {StartInitialHandshake:157(54730ns) {DropHandshakes:470(1577ns) {DropIncomingHandshake:442(406ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(650ns) PrepareNewSessionHandshake:477(969ns) {SwitchToState:179(431ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.515964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789569396393451:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12334
2025-11-28T14:53:42.495639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789561545402932:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17817386ns) {Configure:104(17812571ns) {ProcessConfigured:135(65822ns) {StartInitialHandshake:157(63165ns) {DropHandshakes:470(2685ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(393ns)} GenerateSessionVirtualId:988(887ns) PrepareNewSessionHandshake:477(1718ns) {SwitchToState:179(620ns)}}}}}
2025-11-28T14:53:42.454816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789565960702445:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55038
2025-11-28T14:53:42.455270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789565960702452:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55038
2025-11-28T14:53:42.516046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789569396393453:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12334
2025-11-28T14:53:42.516206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789569396393452:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12334
2025-11-28T14:53:42.516304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789569396393454:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12334
2025-11-28T14:53:42.516384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789569396393455:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12334
2025-11-28T14:53:42.545158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789569396393453:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.566838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789569396393455:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.568331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789565101426080:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.577474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789569396393441:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.577500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789569396393441:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.595256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789569396393441:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:42.595481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789569396393454:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.557091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789568194050803:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(69540452ns) {Configure:104(69534926ns) {ProcessConfigured:135(85418ns) {StartInitialHandshake:157(82507ns) {DropHandshakes:470(2907ns) {DropIncomingHandshake:442(596ns) DropOutgoingHandshake:457(465ns)} GenerateSessionVirtualId:988(1013ns) PrepareNewSessionHandshake:477(4151ns) {SwitchToState:179(993ns)}}}}}
2025-11-28T14:53:42.561584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205552:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.596675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789569396393410:2070] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.582063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789565960702445:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.617732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789569396393451:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.559379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789568194050817:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3592
2025-11-28T14:53:42.495977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789565840370297:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56119
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.561645Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789566273205552:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.510580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789565840370304:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56119
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.510649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789565840370305:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56119
2025-11-28T14:53:42.523591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789565840370305:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.525022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789561545402931:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.577016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789565840370297:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.577165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789561545402933:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.590303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577789565840370250:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(141878533ns) {Configure:104(141874587ns) {ProcessConfigured:135(89350ns) {StartInitialHandshake:157(87134ns) {DropHandshakes:470(2173ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(434ns)} GenerateSessionVirtualId:988(524ns) PrepareNewSessionHandshake:477(1879ns) {SwitchToState:179(775ns)}}}}}
2025-11-28T14:53:42.607695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789565840370310:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56119
2025-11-28T14:53:42.625894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789561545402933:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11824) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577789565840370297:2112] held: no
2025-11-28T14:53:42.631965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789565840370297:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.625350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789565960702452:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.627915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789569396393465:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12334
2025-11-28T14:53:42.656777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789565840370304:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.582397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205553:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.632558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789568194050817:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.651323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789566794716290:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:42.640959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:42.651986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789565960702430:2076] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(268411304ns) {Configure:104(268405302ns) {ProcessConfigured:135(18222431ns) {StartInitialHandshake:157(18219750ns) {DropHandshakes:470(1627ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(332ns) PrepareNewSessionHandshake:477(4678ns) {SwitchToState:179(1886ns)}}}}}
2025-11-28T14:53:42.608966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205557:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.652686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789569396393410:2070] [node 5] ICP22 created new session: [3:7577789569396393466:2070]
2025-11-28T14:53:42.656546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789569396393466:2070] [node 5] ICS09 handshake done sender: [3:7577789569396393441:2110] self: [3:7577789569396393440:0] peer: [5:7577789565840370309:0] socket: 413 qp: -1
2025-11-28T14:53:42.656613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789569396393466:2070] [node 5] ICS10 traffic start
2025-11-28T14:53:42.656687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:42.657167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789569396393452:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.657304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789569396393453:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:42.668646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577789569396393447:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(89503478ns) {Configure:104(89497726ns) {ProcessConfigured:135(12225410ns) {StartInitialHandshake:157(12221009ns) {DropHandshakes:470(2695ns) {DropIncomingHandshake:442(632ns) DropOutgoingHandshake:457(512ns)} GenerateSessionVirtualId:988(2546ns) PrepareNewSessionHandshake:477(3711ns) {SwitchToState:179(1682ns)}}}}}
2025-11-28T14:53:42.687835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789569396393457:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.687865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789569396393457:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.688283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789565101426080:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.688389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:42.688444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789565101426080:2048] [node 1] ICP22 created new session: [3:7577789569396393468:2048]
2025-11-28T14:53:42.696644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789569396393468:2048] [node 1] ICS09 handshake done sender: [3:7577789569396393453:2115] self: [3:7577789569396393458:0] peer: [1:7577789566273205556:0] socket: 434 qp: -1
2025-11-28T14:53:42.696709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789569396393468:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:42.696761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:42.660755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789565840370311:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56119
2025-11-28T14:53:42.633069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789568194050824:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3592
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.698475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789561545402933:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.698534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:42.708478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789561545402933:2050] [node 3] ICP22 created new session: [5:7577789565840370312:2050]
2025-11-28T14:53:42.664504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789566794716290:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.664730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789566794716266:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.688837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789566794716292:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:42.715319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789566794716293:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:42.657796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789568194050824:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.737112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789565960702462:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55038
2025-11-28T14:53:42.737355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789565960702463:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55038
2025-11-28T14:53:42.712875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577789565101426081:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.609007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789566273205557:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.718791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789566794716266:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789566794716290:2115] held: no
2025-11-28T14:53:42.716060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789565840370305:2116] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:42.740733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789565960702451:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.657924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789563899083450:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.616323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205565:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.725026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789566794716266:2061] [node 4] ICP35 event processing took too much time PendingConnection:279(38883556ns) {HandleHandshakeStatus:395(20889811ns) {LogHandshakeFail:472(9964063ns) {UpdateErrorStateLog:545(11023ns)} DropOutgoingHandshake:457(3447ns) ProcessPendingSessionEvents:495(642833ns) {unordered DropSessionEvent:512 num# 100 sum# 545928ns max# 16044ns}}}
2025-11-28T14:53:42.720803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789569396393457:2118] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:42.718721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789565840370312:2050] [node 3] ICS09 handshake done sender: [5:7577789565840370297:2112] self: [5:7577789565840370309:0] peer: [3:7577789569396393440:0] socket: 415 qp: -1
2025-11-28T14:53:42.628184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789566273205552:2110] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:42.740782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789565960702451:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.665372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789568194050820:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.742321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789565960702449:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.783757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789569396393445:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.725666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789561545402932:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.628519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789566273205546:2092] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.725417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789566794716290:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.726246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789565840370312:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:42.742352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789565960702449:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.804708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577789569396393463:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.665402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789568194050820:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.636853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789566273205557:2113] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:42.740971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789566794716266:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.668716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789568194050818:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3592
2025-11-28T14:53:42.677085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789568194050820:2116] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:42.677448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789568194050803:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.677542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:42.677610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789568194050803:2065] [node 6] ICP22 created new session: [4:7577789568194050828:2065]
2025-11-28T14:53:42.677667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789568194050828:2065] [node 6] ICS09 handshake done sender: [4:7577789568194050820:2116] self: [4:7577789568194050819:0] peer: [6:7577789566794716291:0] socket: 448 qp: -1
2025-11-28T14:53:42.677709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789568194050828:2065] [node 6] ICS10 traffic start
2025-11-28T14:53:42.677756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:42.759600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789568194050831:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3592
2025-11-28T14:53:42.759838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789568194050818:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.780793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789563899083448:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.780975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789568194050824:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:42.790396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577789568194050813:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(220909054ns) {Configure:104(220904540ns) {ProcessConfigured:135(65377ns) {StartInitialHandshake:157(62460ns) {DropHandshakes:470(1944ns) {DropIncomingHandshake:442(553ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(538ns) PrepareNewSessionHandshake:477(1283ns) {SwitchToState:179(266ns)}}}}}
2025-11-28T14:53:42.796520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789563899083450:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.800922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:42.808828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789563899083450:2050] [node 3] ICP22 created new session: [4:7577789568194050836:2050]
2025-11-28T14:53:42.821100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789568194050836:2050] [node 3] ICS09 handshake done sender: [4:7577789568194050824:2118] self: [4:7577789568194050826:0] peer: [3:7577789569396393456:0] socket: 455 qp: -1
2025-11-28T14:53:42.823352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789568194050838:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3592
2025-11-28T14:53:42.828469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789568194050836:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:42.640897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:42.804740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577789569396393463:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.848610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:42.759386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789566794716295:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:42.742890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789565960702462:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.740872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:42.808975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:42.760750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789565960702449:2117] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:42.764818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:42.876267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789568194050840:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3592
2025-11-28T14:53:42.743271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789561545402931:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.878114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789563899083450:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(90146622ns) {HandleHandshakeStatus:329(90143640ns) {DropIncomingHandshake:442(1985ns) DropOutgoingHandshake:457(21039ns) SwitchToState:179(687ns) ProcessPendingSessionEvents:495(520600ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 470286ns max# 63902ns}}}
2025-11-28T14:53:42.654419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789566273205546:2092] [node 5] ICP22 created new session: [1:7577789566273205573:2092]
2025-11-28T14:53:42.810731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789569396393445:2071] [node 4] ICP22 created new session: [3:7577789569396393476:2071]
2025-11-28T14:53:42.780450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789566794716293:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.663654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577789561978238189:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(200027716ns) {Configure:104(200021943ns) {ProcessConfigured:135(51185446ns) {StartInitialHandshake:157(51182272ns) {DropHandshakes:470(2679ns) {DropIncomingHandshake:442(443ns) DropOutgoingHandshake:457(408ns)} GenerateSessionVirtualId:988(826ns) PrepareNewSessionHandshake:477(2188ns) {SwitchToState:179(796ns)}}}}}
2025-11-28T14:53:42.762223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789565960702466:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55038
2025-11-28T14:53:42.684750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789561978238190:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.684834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:42.684894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789561978238190:2049] [node 3] ICP22 created new session: [1:7577789566273205574:2049]
2025-11-28T14:53:42.684939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789566273205574:2049] [node 3] ICS09 handshake done sender: [1:7577789566273205557:2113] self: [1:7577789566273205556:0] peer: [3:7577789569396393458:0] socket: 429 qp: -1
2025-11-28T14:53:42.684982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789566273205574:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:42.685019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:42.706111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789566273205573:2092] [node 5] ICS09 handshake done sender: [1:7577789566273205552:2110] self: [1:7577789566273205551:0] peer: [5:7577789565840370306:0] socket: 427 qp: -1
2025-11-28T14:53:42.712660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789566273205573:2092] [node 5] ICS10 traffic start
2025-11-28T14:53:42.712727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:42.757114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205558:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.781469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577789566273205577:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
2025-11-28T14:53:42.808878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205577:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.816469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205555:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.816535Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789566273205555:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.853718Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789566273205555:2112] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:42.865458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789566273205546:2092] [node 5] ICP35 event processing took too much time PendingConnection:279(235953389ns) {HandleHandshakeStatus:329(235950038ns) {DropIncomingHandshake:442(15636ns) DropOutgoingHandshake:457(487ns) SwitchToState:179(494ns) ProcessPendingSessionEvents:495(99885470ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 99766186ns max# 19003090ns}}}
2025-11-28T14:53:42.872635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789566273205549:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.872702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:42.881131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789566273205549:2090] [node 4] ICP22 created new session: [1:7577789566273205583:2090]
2025-11-28T14:53:42.905544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789566273205583:2090] [node 4] ICS09 handshake done sender: [1:7577789566273205555:2112] self: [1:7577789566273205554:0] peer: [4:7577789568194050832:0] socket: 428 qp: -1
2025-11-28T14:53:42.905585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789566273205583:2090] [node 4] ICS10 traffic start
2025-11-28T14:53:42.908658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205568:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.908710Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789566273205568:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.743365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.909369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789568194050838:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.796618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789566794716266:2061] [node 4] ICP22 created new session: [6:7577789566794716296:2061]
2025-11-28T14:53:42.923803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:42.923900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789566273205568:2118] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:42.812451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577789569396393474:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12334
2025-11-28T14:53:42.762916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789565960702443:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.929086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789561978238189:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.929167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:42.929218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789561978238189:2048] [node 2] ICP22 created new session: [1:7577789566273205586:2048]
2025-11-28T14:53:42.929263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789566273205586:2048] [node 2] ICS09 handshake done sender: [1:7577789566273205568:2118] self: [1:7577789566273205566:0] peer: [2:7577789565960702474:0] socket: 466 qp: -1
2025-11-28T14:53:42.929306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789566273205586:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:42.929344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:42.743425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789561545402931:2048] [node 1] ICP22 created new session: [5:7577789565840370316:2048]
2025-11-28T14:53:42.826113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789569396393463:2119] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:42.800812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789562499748918:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.809144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789566794716296:2061] [node 4] ICS09 handshake done sender: [6:7577789566794716290:2115] self: [6:7577789566794716291:0] peer: [4:7577789568194050819:0] socket: 451 qp: -1
2025-11-28T14:53:42.809193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789566794716296:2061] [node 4] ICS10 traffic start
2025-11-28T14:53:42.809815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789566794716298:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:42.809871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789566794716299:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:42.809916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789566794716300:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:42.810622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789562499748918:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17226352ns) {IncomingHandshake:261(17224322ns) {DropIncomingHandshake:442(703ns) PrepareNewSessionHandshake:477(1083ns) {SwitchToState:179(558ns)} GenerateSessionVirtualId:988(413ns) IssueIncomingHandshakeReply:193(4409718ns) {SwitchToState:179(878ns)}}}
2025-11-28T14:53:42.810785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:42.859320Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789566794716266:2061] [node 4] ICP35 event processing took too much time PendingConnection:279(103683115ns) {HandleHandshakeStatus:329(103678795ns) {DropIncomingHandshake:442(733ns) DropOutgoingHandshake:457(346ns) SwitchToState:179(591ns) ProcessPendingSessionEvents:495(1152ns)}}
2025-11-28T14:53:42.876346Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789566794716293:2117] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:42.912576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789566794716298:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.923221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789566794716295:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.928434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577789566794716299:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.931231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789562499748918:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.762973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:42.743469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789565840370316:2048] [node 1] ICS09 handshake done sender: [5:7577789565840370305:2116] self: [5:7577789565840370306:0] peer: [1:7577789566273205551:0] socket: 440 qp: -1
2025-11-28T14:53:42.909521Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577789563899083448:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577789568194050818:2115] held: no
2025-11-28T14:53:42.828555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789569396393476:2071] [node 4] ICS09 handshake done sender: [3:7577789569396393457:2118] self: [3:7577789569396393456:0] peer: [4:7577789568194050826:0] socket: 442 qp: -1
2025-11-28T14:53:42.938779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577789566273205549:2090] [node 4] ICP35 event processing took too much time PendingConnection:279(65089724ns) {HandleHandshakeStatus:329(65085832ns) {DropIncomingHandshake:442(16347ns) DropOutgoingHandshake:457(430ns) SwitchToState:179(410ns) ProcessPendingSessionEvents:495(8826734ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8764399ns max# 8264750ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.939730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789562499748917:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.910241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789568194050818:2115] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:42.844434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789569396393476:2071] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.939824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789562499748916:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.743505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789565840370316:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.944791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789566273205563:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.763029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789565960702443:2080] [node 5] ICP22 created new session: [2:7577789565960702467:2080]
2025-11-28T14:53:42.743551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:42.763099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789565960702467:2080] [node 5] ICS09 handshake done sender: [2:7577789565960702449:2117] self: [2:7577789565960702448:0] peer: [5:7577789565840370314:0] socket: 422 qp: -1
2025-11-28T14:53:42.844536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:42.910839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577789563899083449:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.939902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577789566794716248:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.944824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577789566273205563:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.783316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789565840370318:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56119
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.958089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:42.763141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789565960702467:2080] [node 5] ICS10 traffic start
2025-11-28T14:53:42.874015Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789569396393410:2070] [node 5] ICP35 event processing took too much time PendingConnection:279(276372099ns) {HandleHandshakeStatus:329(276369142ns) {DropIncomingHandshake:442(14362ns) DropOutgoingHandshake:457(4465ns) SwitchToState:179(590ns) ProcessPendingSessionEvents:495(176492584ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 166155287ns max# 23874927ns}}}
2025-11-28T14:53:42.913265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789563899083448:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.763197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:42.945857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789566273205563:2115] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:42.803700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789561545402933:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(120990921ns) {HandleHandshakeStatus:329(120987706ns) {DropIncomingHandshake:442(3043ns) DropOutgoingHandshake:457(415ns) SwitchToState:179(623ns) ProcessPendingSessionEvents:495(2038ns)}}
2025-11-28T14:53:42.931268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:42.962912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789562499748918:2050] [node 3] ICP22 created new session: [6:7577789566794716306:2050]
2025-11-28T14:53:42.874124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577789565101426081:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65238) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577789569396393452:2114] held: no
2025-11-28T14:53:42.946111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789566273205550:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.946179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:42.946238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789566273205550:2091] [node 6] ICP22 created new session: [1:7577789566273205589:2091]
2025-11-28T14:53:42.946284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789566273205589:2091] [node 6] ICS09 handshake done sender: [1:7577789566273205563:2115] self: [1:7577789566273205562:0] peer: [6:7577789566794716303:0] socket: 465 qp: -1
2025-11-28T14:53:42.946322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789566273205589:2091] [node 6] ICS10 traffic start
2025-11-28T14:53:42.946389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:42.804322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789565840370304:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:42.963506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789566794716306:2050] [node 3] ICS09 handshake done sender: [6:7577789566794716293:2117] self: [6:7577789566794716297:0] peer: [3:7577789569396393461:0] socket: 460 qp: -1
2025-11-28T14:53:42.963550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789566794716306:2050] [node 3] ICS10 traffic start
2025-11-28T14:53:42.963598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:43.005229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577789562499748918:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(51356059ns) {HandleHandshakeStatus:329(51353719ns) {DropIncomingHandshake:442(1100ns) DropOutgoingHandshake:457(6155ns) SwitchToState:179(695ns) ProcessPendingSessionEvents:495(546048ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 483781ns max# 56627ns}}}
2025-11-28T14:53:43.006647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789562499748916:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789566794716295:2118] held: no
2025-11-28T14:53:43.007431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577789562499748918:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:43.042999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789566794716308:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:43.064478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789562499748917:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65238) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577789566794716298:2119] held: no
2025-11-28T14:53:42.810120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789565960702463:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.874592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577789569396393447:2069] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:42.823626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789561545402932:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.823722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:42.823779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789561545402932:2049] [node 2] ICP22 created new session: [5:7577789565840370320:2049]
2025-11-28T14:53:42.823827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789565840370320:2049] [node 2] ICS09 handshake done sender: [5:7577789565840370304:2115] self: [5:7577789565840370314:0] peer: [2:7577789565960702448:0] socket: 424 qp: -1
2025-11-28T14:53:42.823861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789565840370320:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:42.823903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:42.928155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789565840370322:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56119
2025-11-28T14:53:42.940878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789565840370322:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.957594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577789561545402948:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.957689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789565840370323:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56119
2025-11-28T14:53:42.958322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789565840370308:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.958359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577789565840370308:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.967228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577789565840370325:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56119
2025-11-28T14:53:42.977548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789565840370308:2117] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:42.980889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577789565840370322:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:42.981106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577789561545402948:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.981182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:42.981233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789561545402948:2063] [node 4] ICP22 created new session: [5:7577789565840370327:2063]
2025-11-28T14:53:42.981299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789565840370327:2063] [node 4] ICS09 handshake done sender: [5:7577789565840370322:2121] self: [5:7577789565840370324:0] peer: [4:7577789568194050833:0] socket: 436 qp: -1
2025-11-28T14:53:42.981345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789565840370327:2063] [node 4] ICS10 traffic start
2025-11-28T14:53:42.981382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:42.988925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577789565840370250:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.989002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:42.989058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577789565840370250:2059] [node 6] ICP22 created new session: [5:7577789565840370329:2059]
2025-11-28T14:53:42.989106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577789565840370329:2059] [node 6] ICS09 handshake done sender: [5:7577789565840370308:2117] self: [5:7577789565840370307:0] peer: [6:7577789566794716304:0] socket: 470 qp: -1
2025-11-28T14:53:42.989137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577789565840370329:2059] [node 6] ICS10 traffic start
2025-11-28T14:53:42.989194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:43.089362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577789561545402931:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(271036855ns) {HandleHandshakeStatus:329(271034797ns) {DropIncomingHandshake:442(547ns) DropOutgoingHandshake:457(4952ns) SwitchToState:179(450ns) ProcessPendingSessionEvents:495(270599119ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 249892553ns max# 19745428ns}}}
2025-11-28T14:53:43.089455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577789561545402931:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-28T14:53:42.810833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789565960702451:2118] [node 3] ICH04 handshake succeeded
2025-11-28T14:53:42.817831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789561665735079:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.817904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-28T14:53:42.817956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789561665735079:2049] [node 3] ICP22 created new session: [2:7577789565960702471:2049]
2025-11-28T14:53:42.818000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789565960702471:2049] [node 3] ICS09 handshake done sender: [2:7577789565960702451:2118] self: [2:7577789565960702450:0] peer: [3:7577789569396393472:0] socket: 423 qp: -1
2025-11-28T14:53:42.818032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789565960702471:2049] [node 3] ICS10 traffic start
2025-11-28T14:53:42.818072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-28T14:53:42.842268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577789565960702443:2080] [node 5] ICP18 handshake (actor [2:7577789565960702463:2123]) is too old
2025-11-28T14:53:42.848594Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [2:7577789565960702443:2080] [node 5] ICP35 event processing took too much time StateWork:294(21327871ns) {IncomingHandshake:261(21322098ns)}
2025-11-28T14:53:42.869218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789565960702466:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.870232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789561665735078:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:42.923836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789565960702461:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.923874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789565960702461:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.947568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789561665735078:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789565960702466:2124] held: no
2025-11-28T14:53:42.953880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789565960702457:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.953910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577789565960702457:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.954892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577789565960702475:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55038
2025-11-28T14:53:42.956936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789565960702466:2124] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:42.957136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789561665735078:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.958585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:42.958636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789561665735078:2048] [node 1] ICP22 created new session: [2:7577789565960702478:2048]
2025-11-28T14:53:42.958700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789565960702478:2048] [node 1] ICS09 handshake done sender: [2:7577789565960702466:2124] self: [2:7577789565960702474:0] peer: [1:7577789566273205566:0] socket: 419 qp: -1
2025-11-28T14:53:42.958735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789565960702478:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:42.958787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:42.959818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789565960702461:2121] [node 4] ICH04 handshake succeeded
2025-11-28T14:53:42.962975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789565960702457:2120] [node 6] ICH04 handshake succeeded
2025-11-28T14:53:42.984731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789565960702444:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.984790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:42.984834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789565960702444:2077] [node 6] ICP22 created new session: [2:7577789565960702480:2077]
2025-11-28T14:53:42.984883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789565960702480:2077] [node 6] ICS09 handshake done sender: [2:7577789565960702457:2120] self: [2:7577789565960702456:0] peer: [6:7577789566794716302:0] socket: 469 qp: -1
2025-11-28T14:53:42.984924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789565960702480:2077] [node 6] ICS10 traffic start
2025-11-28T14:53:42.984966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:42.994542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789565960702430:2076] [node 4] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.994622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-28T14:53:43.005543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789565960702430:2076] [node 4] ICP22 created new session: [2:7577789565960702482:2076]
2025-11-28T14:53:43.005604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789565960702482:2076] [node 4] ICS09 handshake done sender: [2:7577789565960702461:2121] self: [2:7577789565960702460:0] peer: [4:7577789568194050841:0] socket: 412 qp: -1
2025-11-28T14:53:43.005645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789565960702482:2076] [node 4] ICS10 traffic start
2025-11-28T14:53:43.005714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-28T14:53:43.098127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577789565960702444:2077] [node 6] ICP35 event processing took too much time PendingConnection:279(112374184ns) {HandleHandshakeStatus:329(112369260ns) {DropIncomingHandshake:442(14220ns) DropOutgoingHandshake:457(399ns) SwitchToState:179(652ns) ProcessPendingSessionEvents:495(111998975ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 111549623ns max# 18309161ns}}}
2025-11-28T14:53:42.874650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-28T14:53:42.874688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789569396393447:2069] [node 6] ICP22 created new session: [3:7577789569396393478:2069]
2025-11-28T14:53:43.096618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789571089683606:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:42.874719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789569396393478:2069] [node 6] ICS09 handshake done sender: [3:7577789569396393463:2119] self: [3:7577789569396393461:0] peer: [6:7577789566794716297:0] socket: 459 qp: -1
2025-11-28T14:53:42.874751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789569396393478:2069] [node 6] ICS10 traffic start
2025-11-28T14:53:42.874787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-28T14:53:42.875867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577789569396393452:2114] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:42.889546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577789565101426081:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.889632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:42.889695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577789565101426081:2049] [node 2] ICP22 created new session: [3:7577789569396393480:2049]
2025-11-28T14:53:42.940665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789563899083448:2048] [node 1] ICP22 created new session: [4:7577789568194050842:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-28T14:53:43.105119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789566794716295:2118] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:42.889752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577789569396393480:2049] [node 2] ICS09 handshake done sender: [3:7577789569396393452:2114] self: [3:7577789569396393472:0] peer: [2:7577789565960702450:0] socket: 433 qp: -1
2025-11-28T14:53:42.889793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577789569396393480:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:42.889850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:42.925858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577789569396393445:2071] [node 4] ICP35 event processing took too much time PendingConnection:279(140826486ns) {HandleHandshakeStatus:329(140824308ns) {DropIncomingHandshake:442(37202ns) DropOutgoingHandshake:457(430ns) SwitchToState:179(508ns) ProcessPendingSessionEvents:495(63393739ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 63305249ns max# 14947366ns}}}
2025-11-28T14:53:43.113009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577789565840370323:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:43.113190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577789565840370250:2059] [node 6] ICP18 handshake (actor [5:7577789565840370323:2122]) is too old
2025-11-28T14:53:42.947684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789568194050842:2048] [node 1] ICS09 handshake done sender: [4:7577789568194050818:2115] self: [4:7577789568194050832:0] peer: [1:7577789566273205554:0] socket: 435 qp: -1
2025-11-28T14:53:42.947733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789568194050842:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:42.947792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:42.948053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577789563899083448:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(34709380ns) {HandleHandshakeStatus:329(34706130ns) {DropIncomingHandshake:442(970ns) DropOutgoingHandshake:457(388ns) SwitchToState:179(1029ns) ProcessPendingSessionEvents:495(1615ns)}}
2025-11-28T14:53:42.963549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577789568194050834:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:42.963578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577789568194050834:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-28T14:53:42.972879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577789568194050845:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3592
2025-11-28T14:53:42.977375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789568194050834:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:42.978636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577789568194050838:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:42.990533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577789563899083449:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:42.990594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:42.990645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789563899083449:2049] [node 2] ICP22 created new session: [4:7577789568194050846:2049]
2025-11-28T14:53:42.990691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789568194050846:2049] [node 2] ICS09 handshake done sender: [4:7577789568194050838:2121] self: [4:7577789568194050841:0] peer: [2:7577789565960702460:0] socket: 476 qp: -1
2025-11-28T14:53:42.990729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789568194050846:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:42.990760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:42.991698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577789568194050813:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-28T14:53:42.991748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:42.991790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577789568194050813:2066] [node 5] ICP22 created new session: [4:7577789568194050848:2066]
2025-11-28T14:53:42.991855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577789568194050848:2066] [node 5] ICS09 handshake done sender: [4:7577789568194050834:2120] self: [4:7577789568194050833:0] peer: [5:7577789565840370324:0] socket: 475 qp: -1
2025-11-28T14:53:42.991897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577789568194050848:2066] [node 5] ICS10 traffic start
2025-11-28T14:53:42.991948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:43.114776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577789566794716309:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-28T14:53:43.116001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789566794716299:2120] [node 5] ICH04 handshake succeeded
2025-11-28T14:53:43.116217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789562499748916:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:43.116298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:43.116388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789562499748916:2048] [node 1] ICP22 created new session: [6:7577789571089683607:2048]
2025-11-28T14:53:43.116439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789571089683607:2048] [node 1] ICS09 handshake done sender: [6:7577789566794716295:2118] self: [6:7577789566794716303:0] peer: [1:7577789566273205562:0] socket: 467 qp: -1
2025-11-28T14:53:43.116479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789571089683607:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:43.116559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:43.118803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789566794716248:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-28T14:53:43.118880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-28T14:53:43.118951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789566794716248:2064] [node 5] ICP22 created new session: [6:7577789571089683609:2064]
2025-11-28T14:53:43.118994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789571089683609:2064] [node 5] ICS09 handshake done sender: [6:7577789566794716299:2120] self: [6:7577789566794716304:0] peer: [5:7577789565840370307:0] socket: 473 qp: -1
2025-11-28T14:53:43.119672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789571089683609:2064] [node 5] ICS10 traffic start
2025-11-28T14:53:43.119746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-28T14:53:43.125467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577789566794716298:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:43.134317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577789562499748917:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:43.134431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:43.134513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577789562499748917:2049] [node 2] ICP22 created new session: [6:7577789571089683611:2049]
2025-11-28T14:53:43.134574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577789571089683611:2049] [node 2] ICS09 handshake done sender: [6:7577789566794716298:2119] self: [6:7577789566794716302:0] peer: [2:7577789565960702456:0] socket: 472 qp: -1
2025-11-28T14:53:43.134626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577789571089683611:2049] [node 2] ICS10 traffic start
2025-11-28T14:53:43.134676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:43.207699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:43.207950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789565960702478:2048] [node 1] ICS01 socket: 419 reason# EndOfStream
2025-11-28T14:53:43.208003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789561665735078:2048] [node 1] ICP30 unregister session Session# [2:7577789565960702478:2048] VirtualId# [2:7577789565960702474:0]
2025-11-28T14:53:43.208038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789565960702478:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.208125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:43.208165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:43.211262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-28T14:53:43.211490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789569396393468:2048] [node 1] ICS01 socket: 434 reason# EndOfStream
2025-11-28T14:53:43.211542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789565101426080:2048] [node 1] ICP30 unregister session Session# [3:7577789569396393468:2048] VirtualId# [3:7577789569396393458:0]
2025-11-28T14:53:43.211577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789569396393468:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.211650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:43.211692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:43.214995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-28T14:53:43.215220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789568194050842:2048] [node 1] ICS01 socket: 435 reason# EndOfStream
2025-11-28T14:53:43.215262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789563899083448:2048] [node 1] ICP30 unregister session Session# [4:7577789568194050842:2048] VirtualId# [4:7577789568194050832:0]
2025-11-28T14:53:43.215297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789568194050842:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.215384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:43.215424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:43.220426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-28T14:53:43.220660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789571089683607:2048] [node 1] ICS01 socket: 467 reason# EndOfStream
2025-11-28T14:53:43.220715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789562499748916:2048] [node 1] ICP30 unregister session Session# [6:7577789571089683607:2048] VirtualId# [6:7577789566794716303:0]
2025-11-28T14:53:43.220755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789571089683607:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.220836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:43.220880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:43.220340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-28T14:53:43.220614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789565840370316:2048] [node 1] ICS07 socket disconnect 440 reason# EndOfStream
2025-11-28T14:53:43.220664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789565840370316:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.220766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:43.220819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789565840370316:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:43.244779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789561545402931:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:43.244865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789565840370316:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:43.244905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789561545402931:2048] [node 1] ICP30 unregister session Session# [5:7577789565840370316:2048] VirtualId# [5:7577789565840370306:0]
2025-11-28T14:53:43.245028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:43.245063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789561545402931:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:43.245085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:43.256525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789561545402931:2048] [node 1] ICP33 wake up from error state
2025-11-28T14:53:43.351672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-28T14:53:43.351969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577789569396393480:2049] [node 2] ICS01 socket: 433 reason# EndOfStream
2025-11-28T14:53:43.352028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577789565101426081:2049] [node 2] ICP30 unregister session Session# [3:7577789569396393480:2049] VirtualId# [3:7577789569396393472:0]
2025-11-28T14:53:43.352074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577789569396393480:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.352166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:43.352216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:43.356631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-28T14:53:43.356876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789571089683611:2049] [node 2] ICS01 socket: 472 reason# EndOfStream
2025-11-28T14:53:43.356919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789562499748917:2049] [node 2] ICP30 unregister session Session# [6:7577789571089683611:2049] VirtualId# [6:7577789566794716302:0]
2025-11-28T14:53:43.356955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789571089683611:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.357051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:43.357106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:43.359652Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-28T14:53:43.359899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577789565840370320:2049] [node 2] ICS07 socket disconnect 424 reason# EndOfStream
2025-11-28T14:53:43.359938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789565840370320:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.360024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:43.360062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577789565840370320:2049] [node 2] ICS15 start handshake
2025-11-28T14:53:43.360505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-28T14:53:43.360717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789568194050846:2049] [node 2] ICS01 socket: 476 reason# EndOfStream
2025-11-28T14:53:43.360768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789563899083449:2049] [node 2] ICP30 unregister session Session# [4:7577789568194050846:2049] VirtualId# [4:7577789568194050841:0]
2025-11-28T14:53:43.360802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789568194050846:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.360907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:43.360956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:43.364978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577789561545402932:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65238) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:43.365049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789565840370320:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:43.365082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789561545402932:2049] [node 2] ICP30 unregister session Session# [5:7577789565840370320:2049] VirtualId# [5:7577789565840370314:0]
2025-11-28T14:53:43.365130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:43.365176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577789561545402932:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65238) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:43.365189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65238) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:43.376694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577789561545402932:2049] [node 2] ICP33 wake up from error state
2025-11-28T14:53:43.456618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-28T14:53:43.456864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577789568194050836:2050] [node 3] ICS01 socket: 455 reason# EndOfStream
2025-11-28T14:53:43.456911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577789563899083450:2050] [node 3] ICP30 unregister session Session# [4:7577789568194050836:2050] VirtualId# [4:7577789568194050826:0]
2025-11-28T14:53:43.456949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577789568194050836:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.457026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:43.457075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:43.456605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-28T14:53:43.456865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789566794716306:2050] [node 3] ICS07 socket disconnect 460 reason# EndOfStream
2025-11-28T14:53:43.456898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789566794716306:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.457002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:43.457046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789566794716306:2050] [node 3] ICS15 start handshake
2025-11-28T14:53:43.457932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577789562499748918:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11824) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:53:43.458006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789566794716306:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-28T14:53:43.458043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789562499748918:2050] [node 3] ICP30 unregister session Session# [6:7577789566794716306:2050] VirtualId# [6:7577789566794716297:0]
2025-11-28T14:53:43.458126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:43.458158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577789562499748918:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11824) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:43.458173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11824) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:53:43.460509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-28T14:53:43.460759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789565840370312:2050] [node 3] ICS01 socket: 415 reason# EndOfStream
2025-11-28T14:53:43.460801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789561545402933:2050] [node 3] ICP30 unregister session Session# [5:7577789565840370312:2050] VirtualId# [5:7577789565840370309:0]
2025-11-28T14:53:43.460834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789565840370312:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.460937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-28T14:53:43.460982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-28T14:53:43.472527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577789562499748918:2050] [node 3] ICP33 wake up from error state
2025-11-28T14:53:43.558117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-28T14:53:43.558407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577789565840370327:2063] [node 4] ICS01 socket: 436 reason# EndOfStream
2025-11-28T14:53:43.558502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577789561545402948:2063] [node 4] ICP30 unregister session Session# [5:7577789565840370327:2063] VirtualId# [5:7577789565840370324:0]
2025-11-28T14:53:43.558584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577789565840370327:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.558763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:43.558860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:43.562335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-28T14:53:43.562606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577789566794716296:2061] [node 4] ICS01 socket: 451 reason# EndOfStream
2025-11-28T14:53:43.562648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577789566794716266:2061] [node 4] ICP30 unregister session Session# [6:7577789566794716296:2061] VirtualId# [6:7577789566794716291:0]
2025-11-28T14:53:43.562691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789566794716296:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.562787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-28T14:53:43.562832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-28T14:53:43.638564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-28T14:53:43.638798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577789571089683609:2064] [node 5] ICS07 socket disconnect 473 reason# EndOfStream
2025-11-28T14:53:43.638828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577789571089683609:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:43.638924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-28T14:53:43.638968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577789571089683609:2064] [node 5] ICS15 start handshake
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
VERIFY failed (2025-11-28T14:53:43.739179Z): 
  ydb/library/actors/interconnect/ut/dynamic_proxy_ut.cpp:53
  Check(): requirement index == expected failed
NPrivate::InternalPanicImpl(int, char const*, char const*, int, int, int, TBasicStringBuf<char, std::__y1::char_traits<char>>, char const*, unsigned long)+888 (0xF9E668)
NPrivate::Panic(NPrivate::TStaticBuf const&, int, char const*, char const*, char const*, ...)+571 (0xF8C32B)
TArriveQueue::Check()+2174 (0xB9A3DE)
RaceTestIter(unsigned int, unsigned int)+7364 (0xB98F14)
NTestSuiteDynamicProxy::TTestCaseRaceCheck10::Execute_(NUnitTest::TTestContext&)+67 (0xB9A9D3)
std::__y1::__function::__func<NTestSuiteDynamicProxy::TCurrentTest::Execute()::'lambda'(), std::__y1::allocator<NTestSuiteDynamicProxy::TCurrentTest::Execute()::'lambda'()>, void ()>::operator()()+280 (0xB9EA08)
TColoredProcessor::Run(std::__y1::function<void ()>, TBasicString<char, std::__y1::char_traits<char>> const&, char const*, bool)+538 (0x101F80A)
NUnitTest::TTestBase::Run(std::__y1::function<void ()>, TBasicString<char, std::__y1::char_traits<char>> const&, char const*, bool)+504 (0xFEDDE8)
NTestSuiteDynamicProxy::TCurrentTest::Execute()+1229 (0xB9DD5D)
NUnitTest::TTestFactory::Execute()+2176 (0xFEF5A0)
NUnitTest::RunMain(int, char**)+5805 (0x10195CD)
??+0 (0x7F0E65F45D90)
__libc_start_main+128 (0x7F0E65F45E40)
_start+41 (0xAB4029)
2025-11-28T14:53:46.069425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789583676741341:2048] [node 2] ICP01 ready to work
nodeId# 1 peerNodeId# 2 TEvPoisonSession
Subscribe
2025-11-28T14:53:46.109620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789586208987432:2048] [node 1] ICP01 ready to work
2025-11-28T14:53:46.133129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577789586208987432:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21992984ns) {Configure:104(21988697ns) {ProcessConfigured:135(88779ns) {StartInitialHandshake:157(86367ns) {DropHandshakes:470(16583ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(754ns)} GenerateSessionVirtualId:988(1049ns) PrepareNewSessionHandshake:477(1997ns) {SwitchToState:179(592ns)}}}}}
2025-11-28T14:53:46.151842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789586208987442:2056] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:53:46.153158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789583676741350:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:46.154708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:46.156947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789586208987442:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:46.157663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789586208987442:2056] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:46.161939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:46.162221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
TEvNodeConnected
2025-11-28T14:53:46.162480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789586208987445:2048]
2025-11-28T14:53:46.162543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789586208987445:2048] [node 1] ICS09 handshake done sender: [2:7577789586208987442:2056] self: [2:7577789586208987441:0] peer: [1:7577789583676741351:0] socket: 16 qp: -1
2025-11-28T14:53:46.162582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789586208987445:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:46.162659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:46.164692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789583676741350:2056] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:46.166951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:46.167055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:46.167102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789583676741353:2048]
2025-11-28T14:53:46.167158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789583676741353:2048] [node 2] ICS09 handshake done sender: [1:7577789583676741350:2056] self: [1:7577789583676741351:0] peer: [2:7577789586208987441:0] socket: 18 qp: 4307
2025-11-28T14:53:46.167185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789583676741353:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:46.167229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:53:46.469709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:53:46.471419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:46.471471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789586208987445:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:46.471500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789586208987445:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:46.471525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:46.474483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:46.474673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789586208987445:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:46.474691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789586208987445:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:46.471887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-11-28T14:53:46.472135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789583676741353:2048] [node 2] ICS07 socket disconnect 18 reason# ECONNRESET
2025-11-28T14:53:46.472154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789583676741353:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:46.472191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:46.472237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789583676741353:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:46.479314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789586208987445:2048] [node 1] ICS08 incoming handshake Self# [1:7577789583676741351:0] Peer# [2:7577789586208987441:0] Counter# 1753 LastInputSerial# 44
2025-11-28T14:53:46.479954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789583676741355:2058] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:46.480160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:46.480204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789583676741353:2048] [node 2] ICS09 handshake done sender: [1:7577789583676741355:2058] self: [1:7577789583676741351:0] peer: [2:7577789586208987441:0] socket: 22 qp: 4308
2025-11-28T14:53:46.480664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789583676741353:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:46.480713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:46.483825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789583676741353:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:46.483851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789583676741353:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:46.483890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:46.483927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789583676741353:2048] [node 2] ICS08 incoming handshake Self# [2:7577789586208987441:0] Peer# [1:7577789583676741351:0] Counter# 44 LastInputSerial# 1753
2025-11-28T14:53:46.484037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:46.484231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789583676741353:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:46.484304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789583676741353:2048] VirtualId# [1:7577789583676741351:0]
2025-11-28T14:53:46.484340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:46.487481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789586208987448:2058] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:46.492745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:46.492821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789586208987445:2048] [node 1] ICS09 handshake done sender: [2:7577789586208987448:2058] self: [2:7577789586208987441:0] peer: [1:7577789583676741351:0] socket: 23 qp: 4309
2025-11-28T14:53:46.492849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789586208987445:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:46.492896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:46.493174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:46.493193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789586208987445:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:46.493221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789586208987445:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:46.493243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:46.493338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:46.493498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789586208987445:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:46.493513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789586208987445:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:46.500566Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789583676741361:2061]) peer tries to resume nonexistent session Self# [2:7577789586208987441:0] Peer# [1:7577789583676741351:0]
2025-11-28T14:53:46.500627Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789583676741361:2061] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:46.501284Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789586208987453:2060] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:46.502124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:46.502184Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:46.502208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789586208987445:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:46.502262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789586208987445:2048] VirtualId# [2:7577789586208987441:0]
2025-11-28T14:53:46.502362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 422
TEvUndelivered Cookie# 423
TEvUndelivered Cookie# 424
TEvUndelivered Cookie# 425
TEvUndelivered Cookie# 426
TEvUndelivered Cookie# 427
2025-11-28T14:53:46.517732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789586208987456:2061] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:53:46.518552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789583676741362:2062] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:46.518702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:46.520027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789586208987456:2061] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:46.522329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789586208987456:2061] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:46.522596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:46.522662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:46.522717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789586208987459:2048]
2025-11-28T14:53:46.522751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789586208987459:2048] [node 1] ICS09 handshake done sender: [2:7577789586208987456:2061] self: [2:7577789586208987455:0] peer: [1:7577789583676741363:0] socket: 16 qp: -1
2025-11-28T14:53:46.522793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789586208987459:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:46.522824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:46.523996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789583676741362:2062] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:46.524210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:46.524264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:46.524311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789583676741365:2048]
2025-11-28T14:53:46.524347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789583676741365:2048] [node 2] ICS09 handshake done sender: [1:7577789583676741362:2062] self: [1:7577789583676741363:0] peer: [2:7577789586208987455:0] socket: 18 qp: 4313
2025-11-28T14:53:46.524385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789583676741365:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:46.524413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:53:46.725076Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:53:46.725291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789583676741365:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-28T14:53:46.725314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789583676741365:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:46.725343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:46.725375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789583676741365:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:46.730396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789586208987459:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:46.730428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789586208987459:2048] [node 1] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:46.730472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:46.730531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789586208987459:2048] [node 1] ICS08 incoming handshake Self# [1:7577789583676741363:0] Peer# [2:7577789586208987455:0] Counter# 809 LastInputSerial# 19
2025-11-28T14:53:46.731092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789583676741367:2064] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:46.731351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:46.731413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789583676741365:2048] [node 2] ICS09 handshake done sender: [1:7577789583676741367:2064] self: [1:7577789583676741363:0] peer: [2:7577789586208987455:0] socket: 22 qp: 4314
2025-11-28T14:53:46.731727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789583676741365:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:46.731766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:46.732278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:46.732474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789586208987459:2048] [node 1] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:46.732533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789586208987459:2048] VirtualId# [2:7577789586208987455:0]
2025-11-28T14:53:46.732576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:53:46.734999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789586208987461:2062] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:46.735226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:46.735274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789586208987432:2048] [node 1] ICP32 transit to hold-by-error state Explanation# Session continuation race
2025-11-28T14:53:46.735286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: Session continuation race
2025-11-28T14:53:46.735893Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-11-28T14:53:46.736091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789583676741365:2048] [node 2] ICS07 socket disconnect 22 reason# ECONNRESET
2025-11-28T14:53:46.736138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789583676741365:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:46.736170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:46.736231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789583676741365:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:46.739468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789586208987463:2064]) peer tries to resume nonexistent session Self# [1:7577789583676741363:0] Peer# [2:7577789586208987455:0]
2025-11-28T14:53:46.739522Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789586208987463:2064] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:46.740103Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789583676741371:2065] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:46.740870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:46.740951Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:46.740975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789583676741365:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:46.741032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789583676741365:2048] VirtualId# [1:7577789583676741363:0]
2025-11-28T14:53:46.741085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:46.755901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789583676741374:2066] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:53:46.756875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789586208987464:2065] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:46.757013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:46.757388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789583676741374:2066] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:46.759214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789583676741374:2066] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:46.759994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789586208987464:2065] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:46.760237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:46.760292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:46.760329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789583676741377:2048]
2025-11-28T14:53:46.760378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789583676741377:2048] [node 2] ICS09 handshake done sender: [1:7577789583676741374:2066] self: [1:7577789583676741373:0] peer: [2:7577789586208987465:0] socket: 16 qp: -1
2025-11-28T14:53:46.760408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789583676741377:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:46.760461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:46.761031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:46.761083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:46.761110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789586208987467:2048]
2025-11-28T14:53:46.761148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789586208987467:2048] [node 1] ICS09 handshake done sender: [2:7577789586208987464:2065] self: [2:7577789586208987465:0] peer: [1:7577789583676741373:0] socket: 18 qp: 4317
2025-11-28T14:53:46.761172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789586208987467:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:46.761196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:53:47.028983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:53:47.029218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789583676741377:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:53:47.029239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789583676741377:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:47.036236Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Connection reset by peer
2025-11-28T14:53:47.036276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789586208987467:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:47.036299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789586208987467:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:47.036339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:47.033495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:47.033580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789583676741377:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:47.037770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789586208987467:2048] [node 1] ICS08 incoming handshake Self# [1:7577789583676741373:0] Peer# [2:7577789586208987465:0] Counter# 1250 LastInputSerial# 31
2025-11-28T14:53:47.048171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789587971708675:2067] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:47.048458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:47.048530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789583676741377:2048] [node 2] ICS09 handshake done sender: [1:7577789587971708675:2067] self: [1:7577789583676741373:0] peer: [2:7577789586208987465:0] socket: 22 qp: 4318
2025-11-28T14:53:47.048554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789583676741377:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:47.048593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:47.048983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:47.049184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789586208987467:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:47.049200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789586208987467:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:47.053066Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789590503954765:2067] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:47.054419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:47.054482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789586208987467:2048] [node 1] ICS09 handshake done sender: [2:7577789590503954765:2067] self: [2:7577789586208987465:0] peer: [1:7577789583676741373:0] socket: 23 qp: 4319
2025-11-28T14:53:47.054909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789586208987467:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:47.054962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:53:47.487669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:53:47.489670Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:47.489904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789583676741377:2048] [node 2] ICS01 socket: 22 reason# EndOfStream
2025-11-28T14:53:47.489968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789583676741377:2048] VirtualId# [1:7577789583676741373:0]
2025-11-28T14:53:47.490012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789583676741377:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:47.490051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:47.490097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:47.491182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] socket: Connection reset by peer
2025-11-28T14:53:47.491412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789586208987467:2048] [node 1] ICS07 socket disconnect 23 reason# ECONNRESET
2025-11-28T14:53:47.491445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789586208987467:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:47.491472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:47.491531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789586208987467:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:47.494770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789587971708680:2069]) peer tries to resume nonexistent session Self# [2:7577789586208987465:0] Peer# [1:7577789583676741373:0]
2025-11-28T14:53:47.494967Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789587971708680:2069] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:47.495561Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789590503954770:2070] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:47.497236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:47.497301Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:47.497323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789586208987467:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:47.497384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789586208987467:2048] VirtualId# [2:7577789586208987465:0]
2025-11-28T14:53:47.497448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:53:47.516827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [1:7577789587971708682:2070] [node 2] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:53:47.519282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789590503954772:2071] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:47.519410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:47.523354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789587971708682:2070] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:47.524012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789587971708682:2070] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:47.528748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:47.528823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:47.528876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789587971708686:2048]
2025-11-28T14:53:47.528921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789587971708686:2048] [node 2] ICS09 handshake done sender: [1:7577789587971708682:2070] self: [1:7577789587971708681:0] peer: [2:7577789590503954776:0] socket: 21 qp: -1
2025-11-28T14:53:47.528960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789587971708686:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:47.528998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:47.535152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789590503954774:2072] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:53:47.537943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789587971708684:2071] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:47.538076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577789583676741341:2048] [node 2] ICP18 handshake (actor [1:7577789587971708684:2071]) is too old
2025-11-28T14:53:47.540131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577789590503954772:2071] held: no
2025-11-28T14:53:47.540201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789590503954772:2071] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:47.540534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:47.540583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:47.540640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789590503954778:2048]
2025-11-28T14:53:47.540678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789590503954778:2048] [node 1] ICS09 handshake done sender: [2:7577789590503954772:2071] self: [2:7577789590503954776:0] peer: [1:7577789587971708681:0] socket: 22 qp: 4323
2025-11-28T14:53:47.540711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789590503954778:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:47.540747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:53:47.751401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789590503954778:2048] [node 1] ICS01 socket: 22 reason# Debug
2025-11-28T14:53:47.751623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789590503954778:2048] VirtualId# [2:7577789590503954776:0]
2025-11-28T14:53:47.751673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789590503954778:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:47.751736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:47.751806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:53:47.755467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:47.755742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789587971708686:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-28T14:53:47.755767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789587971708686:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:47.755833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:47.755872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789587971708686:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:47.778578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789590503954781:2074] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:53:47.779476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789587971708689:2073] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:47.779635Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:47.779663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789587971708686:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:47.779696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789587971708686:2048] VirtualId# [1:7577789587971708681:0]
2025-11-28T14:53:47.779747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:47.779244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789590503954783:2075]) peer tries to resume nonexistent session Self# [1:7577789587971708681:0] Peer# [2:7577789590503954776:0]
2025-11-28T14:53:47.779297Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789590503954783:2075] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:47.780316Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789587971708688:2072] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:47.781108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [1:7577789587971708689:2073] held: yes
2025-11-28T14:53:47.781548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789590503954781:2074] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:47.781876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789590503954781:2074] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:47.783367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:47.783407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:47.783449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789590503954785:2048]
2025-11-28T14:53:47.783488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789590503954785:2048] [node 1] ICS09 handshake done sender: [2:7577789590503954781:2074] self: [2:7577789590503954780:0] peer: [1:7577789587971708691:0] socket: 16 qp: -1
2025-11-28T14:53:47.783517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789590503954785:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:47.783548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:47.784689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789587971708689:2073] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:47.784968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:47.785004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:47.785041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789587971708693:2048]
2025-11-28T14:53:47.785078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789587971708693:2048] [node 2] ICS09 handshake done sender: [1:7577789587971708689:2073] self: [1:7577789587971708691:0] peer: [2:7577789590503954780:0] socket: 18 qp: 4326
2025-11-28T14:53:47.785106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789587971708693:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:47.785135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:53:47.929894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:53:47.930132Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:47.930340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789587971708693:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:47.930362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789587971708693:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:47.930392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:47.930434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789587971708693:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:47.936622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:47.936815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789590503954785:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:53:47.936837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789590503954785:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:47.936907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:47.936944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789590503954785:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:47.948261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789590503954785:2048] [node 1] ICS08 incoming handshake Self# [1:7577789587971708691:0] Peer# [2:7577789590503954780:0] Counter# 498 LastInputSerial# 13
2025-11-28T14:53:47.951961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789587971708695:2075] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:47.954945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:47.954997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789587971708693:2048] [node 2] ICS09 handshake done sender: [1:7577789587971708695:2075] self: [1:7577789587971708691:0] peer: [2:7577789590503954780:0] socket: 22 qp: 4329
2025-11-28T14:53:47.955507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789587971708693:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:47.955548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:47.955638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789587971708693:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:47.955659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789587971708693:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:47.955711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:47.955747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789587971708693:2048] [node 2] ICS08 incoming handshake Self# [2:7577789590503954780:0] Peer# [1:7577789587971708691:0] Counter# 13 LastInputSerial# 498
2025-11-28T14:53:47.955811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:47.955963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789587971708693:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:47.956013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789587971708693:2048] VirtualId# [1:7577789587971708691:0]
2025-11-28T14:53:47.956037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:47.961659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789590503954788:2077] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:47.965880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:47.965941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789590503954785:2048] [node 1] ICS09 handshake done sender: [2:7577789590503954788:2077] self: [2:7577789590503954780:0] peer: [1:7577789587971708691:0] socket: 23 qp: 4330
2025-11-28T14:53:47.965967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789590503954785:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:47.966003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:47.966285Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:47.966306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789590503954785:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:47.966330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789590503954785:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:47.966348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:47.966414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:47.972549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789590503954785:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:47.972584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789590503954785:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:47.977389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789587971708701:2078]) peer tries to resume nonexistent session Self# [2:7577789590503954780:0] Peer# [1:7577789587971708691:0]
2025-11-28T14:53:47.977437Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789587971708701:2078] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:47.984895Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789590503954793:2079] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:47.985736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:47.985805Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:47.985829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789590503954785:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:47.985883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789590503954785:2048] VirtualId# [2:7577789590503954780:0]
2025-11-28T14:53:47.985985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 1961
TEvUndelivered Cookie# 1962
TEvUndelivered Cookie# 1963
TEvUndelivered Cookie# 1964
TEvUndelivered Cookie# 1965
TEvUndelivered Cookie# 1966
TEvUndelivered Cookie# 1967
TEvUndelivered Cookie# 1968
2025-11-28T14:53:48.019529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789590503954796:2080] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:53:48.025138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789587971708702:2079] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:48.025262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:48.026338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789590503954796:2080] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:48.029388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789590503954796:2080] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:48.029660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:48.029703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:48.029740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789594798922095:2048]
2025-11-28T14:53:48.029776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789594798922095:2048] [node 1] ICS09 handshake done sender: [2:7577789590503954796:2080] self: [2:7577789590503954795:0] peer: [1:7577789592266675999:0] socket: 16 qp: -1
2025-11-28T14:53:48.029801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789594798922095:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:48.029833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:48.030922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789587971708702:2079] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:48.031279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:48.031325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:48.031358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789592266676001:2048]
2025-11-28T14:53:48.031409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789592266676001:2048] [node 2] ICS09 handshake done sender: [1:7577789587971708702:2079] self: [1:7577789592266675999:0] peer: [2:7577789590503954795:0] socket: 18 qp: 4334
2025-11-28T14:53:48.031432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789592266676001:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:48.031467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:53:48.502897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:53:48.503118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789592266676001:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-28T14:53:48.503142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789592266676001:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:48.503189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:48.503217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789592266676001:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:48.505297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:48.505498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789594798922095:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:53:48.505519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789594798922095:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:48.506024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:48.506084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789594798922095:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:48.507750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789594798922095:2048] [node 1] ICS08 incoming handshake Self# [1:7577789592266675999:0] Peer# [2:7577789590503954795:0] Counter# 3797 LastInputSerial# 97
2025-11-28T14:53:48.509952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789592266676003:2081] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:48.510245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:48.510293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789592266676001:2048] [node 2] ICS09 handshake done sender: [1:7577789592266676003:2081] self: [1:7577789592266675999:0] peer: [2:7577789590503954795:0] socket: 22 qp: 4335
2025-11-28T14:53:48.510604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789592266676001:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:48.510644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:48.512796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789592266676001:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:48.512829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789592266676001:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:48.512871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:48.512905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789592266676001:2048] [node 2] ICS08 incoming handshake Self# [2:7577789590503954795:0] Peer# [1:7577789592266675999:0] Counter# 97 LastInputSerial# 3797
2025-11-28T14:53:48.513147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:48.513324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789592266676001:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:48.513393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789592266676001:2048] VirtualId# [1:7577789592266675999:0]
2025-11-28T14:53:48.513430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:48.515399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789594798922098:2082] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:48.515824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:48.515876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789594798922095:2048] [node 1] ICS09 handshake done sender: [2:7577789594798922098:2082] self: [2:7577789590503954795:0] peer: [1:7577789592266675999:0] socket: 23 qp: 4336
2025-11-28T14:53:48.515899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789594798922095:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:48.515929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:48.516187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:48.516326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789594798922095:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-28T14:53:48.516350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789594798922095:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:48.518308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:48.518377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789594798922095:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:48.523559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789592266676009:2084]) peer tries to resume nonexistent session Self# [2:7577789590503954795:0] Peer# [1:7577789592266675999:0]
2025-11-28T14:53:48.523611Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789592266676009:2084] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:48.524279Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789594798922103:2084] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:48.525055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:48.525130Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:48.525154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789594798922095:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:48.525207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789594798922095:2048] VirtualId# [2:7577789590503954795:0]
2025-11-28T14:53:48.525256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:53:48.541980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789594798922106:2085] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:53:48.542918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789592266676010:2085] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:48.543038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:48.544057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789594798922106:2085] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:48.546047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789594798922106:2085] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:48.546315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:48.546357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:48.546396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789594798922109:2048]
2025-11-28T14:53:48.546429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789594798922109:2048] [node 1] ICS09 handshake done sender: [2:7577789594798922106:2085] self: [2:7577789594798922105:0] peer: [1:7577789592266676011:0] socket: 16 qp: -1
2025-11-28T14:53:48.546454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789594798922109:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:48.546480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:48.548009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789592266676010:2085] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:48.548282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:48.548320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:48.548355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789592266676013:2048]
2025-11-28T14:53:48.548405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789592266676013:2048] [node 2] ICS09 handshake done sender: [1:7577789592266676010:2085] self: [1:7577789592266676011:0] peer: [2:7577789594798922105:0] socket: 18 qp: 4340
2025-11-28T14:53:48.548429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789592266676013:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:48.548455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:53:48.994494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:53:48.994822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:48.995148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789592266676013:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:48.995195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789592266676013:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:48.995271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:48.995314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789592266676013:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:48.997609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:48.997662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789594798922109:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:48.997693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789594798922109:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:48.997728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:48.997794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:48.997988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789594798922109:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:48.998034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789594798922109:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:48.999782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789594798922109:2048] [node 1] ICS08 incoming handshake Self# [1:7577789592266676011:0] Peer# [2:7577789594798922105:0] Counter# 3247 LastInputSerial# 81
2025-11-28T14:53:49.002025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789592266676015:2087] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:49.002334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:49.002393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789592266676013:2048] [node 2] ICS09 handshake done sender: [1:7577789592266676015:2087] self: [1:7577789592266676011:0] peer: [2:7577789594798922105:0] socket: 22 qp: 4341
2025-11-28T14:53:49.002766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789592266676013:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:49.002838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:49.006214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789592266676013:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:49.006248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789592266676013:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:49.006315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:49.006350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789592266676013:2048] [node 2] ICS08 incoming handshake Self# [2:7577789594798922105:0] Peer# [1:7577789592266676011:0] Counter# 81 LastInputSerial# 3247
2025-11-28T14:53:49.006613Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:49.006820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789592266676013:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:49.006884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789592266676013:2048] VirtualId# [1:7577789592266676011:0]
2025-11-28T14:53:49.006919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:49.008914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789594798922112:2087] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:49.009267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:49.009326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789594798922109:2048] [node 1] ICS09 handshake done sender: [2:7577789594798922112:2087] self: [2:7577789594798922105:0] peer: [1:7577789592266676011:0] socket: 23 qp: 4342
2025-11-28T14:53:49.009349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789594798922109:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:49.009385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:49.009598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:49.009616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789594798922109:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:49.009635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789594798922109:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:49.009654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:49.009727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:49.009874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789594798922109:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:49.009888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789594798922109:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:49.016222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789596561643317:2090]) peer tries to resume nonexistent session Self# [2:7577789594798922105:0] Peer# [1:7577789592266676011:0]
2025-11-28T14:53:49.016283Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789596561643317:2090] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:49.016911Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789599093889413:2089] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:49.017649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:49.017724Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:49.017764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789594798922109:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:49.017821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789594798922109:2048] VirtualId# [2:7577789594798922105:0]
2025-11-28T14:53:49.017956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 3702
TEvUndelivered Cookie# 3703
TEvUndelivered Cookie# 3704
TEvUndelivered Cookie# 3705
TEvUndelivered Cookie# 3706
TEvUndelivered Cookie# 3707
TEvUndelivered Cookie# 3708
TEvUndelivered Cookie# 3709
2025-11-28T14:53:49.035645Z :INTERCONNECT ERROR: interconnect_handshake.cpp:727: Handshake [2:7577789599093889416:2090] [node 1] ICRDMA Unable to allocate memory region to perform rdma handshake
2025-11-28T14:53:49.036522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789596561643318:2091] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:49.036744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:49.037845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789599093889416:2090] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:49.039656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789599093889416:2090] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:49.040461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789596561643318:2091] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:49.040850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:49.040907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:49.040955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789599093889419:2048]
2025-11-28T14:53:49.040997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789599093889419:2048] [node 1] ICS09 handshake done sender: [2:7577789599093889416:2090] self: [2:7577789599093889415:0] peer: [1:7577789596561643319:0] socket: 16 qp: -1
2025-11-28T14:53:49.041031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789599093889419:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:49.041063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:49.041553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:49.041618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:49.041662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789596561643321:2048]
2025-11-28T14:53:49.041703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789596561643321:2048] [node 2] ICS09 handshake done sender: [1:7577789596561643318:2091] self: [1:7577789596561643319:0] peer: [2:7577789599093889415:0] socket: 18 qp: 4346
2025-11-28T14:53:49.041725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789596561643321:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:49.041750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:53:49.365086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:53:49.365178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:49.365425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:49.365447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789599093889419:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:49.365468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789599093889419:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:49.365493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:49.365540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789599093889419:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:49.365556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789599093889419:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:49.366729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:49.366940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789596561643321:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:49.366962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789596561643321:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:49.367017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:49.367046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789596561643321:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:49.374399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789599093889419:2048] [node 1] ICS08 incoming handshake Self# [1:7577789596561643319:0] Peer# [2:7577789599093889415:0] Counter# 1673 LastInputSerial# 42
2025-11-28T14:53:49.374862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789596561643321:2048] [node 2] ICS08 incoming handshake Self# [2:7577789599093889415:0] Peer# [1:7577789596561643319:0] Counter# 42 LastInputSerial# 1673
2025-11-28T14:53:49.375306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789596561643323:2093] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:49.375554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:49.375616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789596561643321:2048] [node 2] ICS09 handshake done sender: [1:7577789596561643323:2093] self: [1:7577789596561643319:0] peer: [2:7577789599093889415:0] socket: 24 qp: 4348
2025-11-28T14:53:49.375947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789596561643321:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:49.375992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:49.378074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789599093889423:2092] held: no
2025-11-28T14:53:49.379777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789599093889423:2092] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:49.379968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:49.380026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789599093889419:2048] [node 1] ICS09 handshake done sender: [2:7577789599093889423:2092] self: [2:7577789599093889415:0] peer: [1:7577789596561643319:0] socket: 25 qp: 4350
2025-11-28T14:53:49.380049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789599093889419:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:49.380082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:53:49.919851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789599093889419:2048] [node 1] ICS01 socket: 25 reason# Debug
2025-11-28T14:53:49.919943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789599093889419:2048] VirtualId# [2:7577789599093889415:0]
2025-11-28T14:53:49.919977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789599093889419:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:49.920034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:49.920097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:49.920328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:49.920606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789596561643321:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-28T14:53:49.920632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789596561643321:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:49.920701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:49.920744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789596561643321:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-28T14:53:49.929575Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789596561643328:2095] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:49.930407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:49.930495Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:49.930539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789596561643321:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:49.930613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789596561643321:2048] VirtualId# [1:7577789596561643319:0]
2025-11-28T14:53:49.930712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:49.928890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789599093889426:2094]) peer tries to resume nonexistent session Self# [1:7577789596561643319:0] Peer# [2:7577789599093889415:0]
2025-11-28T14:53:49.928958Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789599093889426:2094] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:49.951862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789599093889427:2095] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:49.952029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:49.954783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789596561643331:2096] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:49.976757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789599093889427:2095] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:49.977211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789596561643331:2096] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:49.977609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:49.977662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:49.977716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789596561643334:2048]
2025-11-28T14:53:49.977750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789596561643334:2048] [node 2] ICS09 handshake done sender: [1:7577789596561643331:2096] self: [1:7577789596561643330:0] peer: [2:7577789599093889428:0] socket: 16 qp: 4353
2025-11-28T14:53:49.977779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789596561643334:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:49.977808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:49.980516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:49.980568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:49.980608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789599093889430:2048]
2025-11-28T14:53:49.980655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789599093889430:2048] [node 1] ICS09 handshake done sender: [2:7577789599093889427:2095] self: [2:7577789599093889428:0] peer: [1:7577789596561643330:0] socket: 18 qp: 4352
2025-11-28T14:53:49.980687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789599093889430:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:49.980714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:53:50.059534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789599093889430:2048] [node 1] ICS01 socket: 18 reason# Debug
2025-11-28T14:53:50.059633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789599093889430:2048] VirtualId# [2:7577789599093889428:0]
2025-11-28T14:53:50.059659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789599093889430:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:50.059712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:50.059827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 5356
TEvUndelivered Cookie# 5357
TEvUndelivered Cookie# 5358
TEvUndelivered Cookie# 5359
TEvUndelivered Cookie# 5360
TEvUndelivered Cookie# 5361
TEvUndelivered Cookie# 5362
TEvUndelivered Cookie# 5363
TEvUndelivered Cookie# 5364
TEvUndelivered Cookie# 5365
2025-11-28T14:53:50.060745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:50.060985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789596561643334:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:53:50.061005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789596561643334:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:50.061062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:50.061090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789596561643334:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:50.064559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789603388856728:2097]) peer tries to resume nonexistent session Self# [1:7577789596561643330:0] Peer# [2:7577789599093889428:0]
2025-11-28T14:53:50.064621Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789603388856728:2097] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:50.065098Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789600856610632:2097] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:50.065758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:50.065818Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:50.065838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789596561643334:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:50.065899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789596561643334:2048] VirtualId# [1:7577789596561643330:0]
2025-11-28T14:53:50.065940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:50.069897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789603388856729:2098] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:50.070027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:50.070386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789600856610635:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:50.078898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789603388856729:2098] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:50.079161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789600856610635:2098] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:50.079435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:50.079483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:50.079521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789600856610638:2048]
2025-11-28T14:53:50.079554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789600856610638:2048] [node 2] ICS09 handshake done sender: [1:7577789600856610635:2098] self: [1:7577789600856610634:0] peer: [2:7577789603388856730:0] socket: 16 qp: 4355
2025-11-28T14:53:50.079581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789600856610638:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:50.079613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:50.080151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:50.080187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:50.080222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789603388856732:2048]
2025-11-28T14:53:50.080251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789603388856732:2048] [node 1] ICS09 handshake done sender: [2:7577789603388856729:2098] self: [2:7577789603388856730:0] peer: [1:7577789600856610634:0] socket: 18 qp: 4356
2025-11-28T14:53:50.080273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789603388856732:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:50.080293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:53:50.422814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:53:50.422951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:50.423145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789603388856732:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:50.423164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789603388856732:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:50.423221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:50.423250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789603388856732:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:50.422944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:53:50.422993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789600856610638:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:50.423021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789600856610638:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:50.423048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:50.424228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:50.424486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789600856610638:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:50.424503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789600856610638:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:50.438474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789600856610638:2048] [node 2] ICS08 incoming handshake Self# [2:7577789603388856730:0] Peer# [1:7577789600856610634:0] Counter# 46 LastInputSerial# 1885
2025-11-28T14:53:50.438862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789600856610640:2099] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:50.439155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:50.439204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789600856610638:2048] [node 2] ICS09 handshake done sender: [1:7577789600856610640:2099] self: [1:7577789600856610634:0] peer: [2:7577789603388856730:0] socket: 22 qp: 4359
2025-11-28T14:53:50.439554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789600856610638:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:50.439589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:50.438171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789603388856732:2048] [node 1] ICS08 incoming handshake Self# [1:7577789600856610634:0] Peer# [2:7577789603388856730:0] Counter# 1885 LastInputSerial# 46
2025-11-28T14:53:50.453789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789603388856735:2101] held: no
2025-11-28T14:53:50.453865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789603388856735:2101] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:50.454143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:50.454207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789603388856732:2048] [node 1] ICS09 handshake done sender: [2:7577789603388856735:2101] self: [2:7577789603388856730:0] peer: [1:7577789600856610634:0] socket: 23 qp: 4357
2025-11-28T14:53:50.454529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789603388856732:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:50.454581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:53:50.654048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789600856610638:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-28T14:53:50.654137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789600856610638:2048] VirtualId# [1:7577789600856610634:0]
2025-11-28T14:53:50.654175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789600856610638:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:50.654224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:50.654255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:50.656572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:50.656774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789603388856732:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-28T14:53:50.656803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789603388856732:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:50.656901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:50.656934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789603388856732:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:50.664102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789600856610645:2101]) peer tries to resume nonexistent session Self# [2:7577789603388856730:0] Peer# [1:7577789600856610634:0]
2025-11-28T14:53:50.665511Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789600856610645:2101] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:50.672717Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789603388856739:2103] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:50.673647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:50.673726Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:50.673748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789603388856732:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:50.673804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789603388856732:2048] VirtualId# [2:7577789603388856730:0]
2025-11-28T14:53:50.673865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:53:50.674783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789603388856741:2104] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:50.675056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:50.676715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789600856610647:2102] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:50.679905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789600856610649:2103] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:50.684597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789603388856741:2104] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:50.684845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789600856610647:2102] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:50.685135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:50.685219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:50.685260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789600856610652:2048]
2025-11-28T14:53:50.685294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789600856610652:2048] [node 2] ICS09 handshake done sender: [1:7577789600856610647:2102] self: [1:7577789600856610646:0] peer: [2:7577789603388856744:0] socket: 20 qp: 4364
2025-11-28T14:53:50.685321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789600856610652:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:50.685348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:50.689109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:50.689192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:50.689233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789603388856747:2048]
2025-11-28T14:53:50.689270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789603388856747:2048] [node 1] ICS09 handshake done sender: [2:7577789603388856741:2104] self: [2:7577789603388856744:0] peer: [1:7577789600856610646:0] socket: 21 qp: 4363
2025-11-28T14:53:50.689318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789603388856747:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:50.689351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:53:50.844584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:53:50.845069Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:50.845428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789600856610652:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:53:50.845450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789600856610652:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:50.845474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:50.845502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789600856610652:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:50.848534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] socket: Connection reset by peer
2025-11-28T14:53:50.848794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789603388856747:2048] [node 1] ICS07 socket disconnect 21 reason# ECONNRESET
2025-11-28T14:53:50.848826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789603388856747:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:50.848856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:50.848912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789603388856747:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:50.854749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789603388856747:2048] [node 1] ICS08 incoming handshake Self# [1:7577789600856610646:0] Peer# [2:7577789603388856744:0] Counter# 331 LastInputSerial# 8
2025-11-28T14:53:50.860241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789600856610652:2048] [node 2] ICS08 incoming handshake Self# [2:7577789603388856744:0] Peer# [1:7577789600856610646:0] Counter# 8 LastInputSerial# 331
2025-11-28T14:53:50.864652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789600856610654:2104] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:50.866240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:50.866306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789600856610652:2048] [node 2] ICS09 handshake done sender: [1:7577789600856610654:2104] self: [1:7577789600856610646:0] peer: [2:7577789603388856744:0] socket: 16 qp: 4368
2025-11-28T14:53:50.866642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789600856610652:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:50.866689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:50.869862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789603388856750:2108] held: no
2025-11-28T14:53:50.869929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789603388856750:2108] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:50.870117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:50.870161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789603388856747:2048] [node 1] ICS09 handshake done sender: [2:7577789603388856750:2108] self: [2:7577789603388856744:0] peer: [1:7577789600856610646:0] socket: 18 qp: 4367
2025-11-28T14:53:50.870523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789603388856747:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:50.870557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:53:51.266259Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:53:51.266450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789603388856747:2048] [node 1] ICS07 socket disconnect 18 reason# Debug
2025-11-28T14:53:51.266472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789603388856747:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:51.266513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:51.266543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789603388856747:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:51.267777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:51.267973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789600856610652:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:53:51.267993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789600856610652:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:51.268038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:51.268066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789600856610652:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:51.274904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789600856610652:2048] [node 2] ICS08 incoming handshake Self# [2:7577789603388856744:0] Peer# [1:7577789600856610646:0] Counter# 96 LastInputSerial# 3600
2025-11-28T14:53:51.275112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789603388856747:2048] [node 1] ICS08 incoming handshake Self# [1:7577789600856610646:0] Peer# [2:7577789603388856744:0] Counter# 3600 LastInputSerial# 96
2025-11-28T14:53:51.275551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789605151577955:2106] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:51.275768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:51.275819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789600856610652:2048] [node 2] ICS09 handshake done sender: [1:7577789605151577955:2106] self: [1:7577789600856610646:0] peer: [2:7577789603388856744:0] socket: 22 qp: 4372
2025-11-28T14:53:51.276106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789600856610652:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:51.276143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:51.279662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789607683824052:2111] held: no
2025-11-28T14:53:51.279727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789607683824052:2111] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:51.279988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:51.280037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789603388856747:2048] [node 1] ICS09 handshake done sender: [2:7577789607683824052:2111] self: [2:7577789603388856744:0] peer: [1:7577789600856610646:0] socket: 23 qp: 4374
2025-11-28T14:53:51.281087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789603388856747:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:51.281128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:53:51.859394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:53:51.859646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789603388856747:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-28T14:53:51.859669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789603388856747:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:51.859713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:51.859741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789603388856747:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:51.862611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:51.862852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789600856610652:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-28T14:53:51.862873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789600856610652:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:51.862918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:51.862947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789600856610652:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:51.868834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789603388856747:2048] [node 1] ICS08 incoming handshake Self# [1:7577789600856610646:0] Peer# [2:7577789603388856744:0] Counter# 8852 LastInputSerial# 232
2025-11-28T14:53:51.869309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789605151577960:2108] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:51.869516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:51.869557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789600856610652:2048] [node 2] ICS09 handshake done sender: [1:7577789605151577960:2108] self: [1:7577789600856610646:0] peer: [2:7577789603388856744:0] socket: 20 qp: 4376
2025-11-28T14:53:51.870043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789600856610652:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:51.870085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:51.873456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789600856610652:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:51.873488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789600856610652:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:51.873531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:51.873570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789600856610652:2048] [node 2] ICS08 incoming handshake Self# [2:7577789603388856744:0] Peer# [1:7577789600856610646:0] Counter# 232 LastInputSerial# 8852
2025-11-28T14:53:51.873962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:51.874158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789600856610652:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:51.874214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789600856610652:2048] VirtualId# [1:7577789600856610646:0]
2025-11-28T14:53:51.874243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:51.875797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789607683824057:2114] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:51.876158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:51.876209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789603388856747:2048] [node 1] ICS09 handshake done sender: [2:7577789607683824057:2114] self: [2:7577789603388856744:0] peer: [1:7577789600856610646:0] socket: 21 qp: 4377
2025-11-28T14:53:51.876936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789603388856747:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:51.876973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:51.877192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:51.877211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789603388856747:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:51.877230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789603388856747:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:51.877256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:51.877327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:51.877473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789603388856747:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:51.877488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789603388856747:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:51.884045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789605151577966:2111]) peer tries to resume nonexistent session Self# [2:7577789603388856744:0] Peer# [1:7577789600856610646:0]
2025-11-28T14:53:51.884099Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789605151577966:2111] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:51.884802Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789607683824061:2116] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:51.885605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:51.885674Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:51.885697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789603388856747:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:51.885763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789603388856747:2048] VirtualId# [2:7577789603388856744:0]
2025-11-28T14:53:51.885879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 8243
TEvUndelivered Cookie# 8244
TEvUndelivered Cookie# 8245
TEvUndelivered Cookie# 8246
TEvUndelivered Cookie# 8247
TEvUndelivered Cookie# 8248
TEvUndelivered Cookie# 8249
TEvUndelivered Cookie# 8250
TEvUndelivered Cookie# 8251
2025-11-28T14:53:51.890575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789605151577967:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:51.890688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:51.891786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789607683824064:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:51.896543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789605151577967:2112] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:51.896745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789607683824064:2117] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:51.897037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:51.897081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:51.897120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789607683824067:2048]
2025-11-28T14:53:51.897156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789607683824067:2048] [node 1] ICS09 handshake done sender: [2:7577789607683824064:2117] self: [2:7577789607683824063:0] peer: [1:7577789605151577968:0] socket: 16 qp: 4381
2025-11-28T14:53:51.897190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789607683824067:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:51.897218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:51.897704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:51.897738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:51.897771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789605151577970:2048]
2025-11-28T14:53:51.897802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789605151577970:2048] [node 2] ICS09 handshake done sender: [1:7577789605151577967:2112] self: [1:7577789605151577968:0] peer: [2:7577789607683824063:0] socket: 20 qp: 4380
2025-11-28T14:53:51.897825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789605151577970:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:51.897848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:53:52.352494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789607683824067:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-28T14:53:52.353808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:52.354033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789605151577970:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:53:52.352917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789607683824067:2048] VirtualId# [2:7577789607683824063:0]
2025-11-28T14:53:52.352946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789607683824067:2048] [node 1] ICS25 shutdown socket, reason# Debug
TEvNodeDisconnected
2025-11-28T14:53:52.352989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:52.353081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvUndelivered Cookie# 9001
TEvUndelivered Cookie# 9002
TEvUndelivered Cookie# 9003
TEvUndelivered Cookie# 9004
TEvUndelivered Cookie# 9005
TEvUndelivered Cookie# 9006
TEvUndelivered Cookie# 9007
TEvUndelivered Cookie# 9008
TEvUndelivered Cookie# 9009
2025-11-28T14:53:52.354053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789605151577970:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:52.354109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:52.354137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789605151577970:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:52.357907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789611978791365:2118]) peer tries to resume nonexistent session Self# [1:7577789605151577968:0] Peer# [2:7577789607683824063:0]
2025-11-28T14:53:52.357971Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789611978791365:2118] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:52.359366Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789609446545268:2114] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:52.360121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:52.360192Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:52.360215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789605151577970:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:52.363086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789605151577970:2048] VirtualId# [1:7577789605151577968:0]
2025-11-28T14:53:52.363153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:52.376903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789611978791366:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:52.377066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:52.379712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789609446545271:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:52.398404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789611978791366:2119] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:52.398680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789609446545271:2115] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:52.398979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:52.399023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:52.399065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789609446545274:2048]
2025-11-28T14:53:52.399103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789609446545274:2048] [node 2] ICS09 handshake done sender: [1:7577789609446545271:2115] self: [1:7577789609446545270:0] peer: [2:7577789611978791367:0] socket: 18 qp: 4384
2025-11-28T14:53:52.399147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789609446545274:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:52.399183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:52.399881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:52.399921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:52.399954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789611978791369:2048]
2025-11-28T14:53:52.399984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789611978791369:2048] [node 1] ICS09 handshake done sender: [2:7577789611978791366:2119] self: [2:7577789611978791367:0] peer: [1:7577789609446545270:0] socket: 16 qp: 4383
2025-11-28T14:53:52.400006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789611978791369:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:52.400030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:53:52.553162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789611978791369:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-28T14:53:52.553326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789611978791369:2048] VirtualId# [2:7577789611978791367:0]
2025-11-28T14:53:52.553360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789611978791369:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:52.553408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:52.553452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:52.554920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:52.555159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789609446545274:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:52.555178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789609446545274:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:52.555247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:52.555279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789609446545274:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-28T14:53:52.559137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789611978791371:2121]) peer tries to resume nonexistent session Self# [1:7577789609446545270:0] Peer# [2:7577789611978791367:0]
2025-11-28T14:53:52.559195Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789611978791371:2121] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:52.559766Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789609446545276:2116] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:52.560664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:52.560727Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:52.560750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789609446545274:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:52.561040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789609446545274:2048] VirtualId# [1:7577789609446545270:0]
2025-11-28T14:53:52.561093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:52.565841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789611978791372:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:52.565972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:52.567949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789609446545279:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:52.575828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789611978791372:2122] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:52.576098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789609446545279:2117] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:52.576348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:52.576450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:52.576507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789609446545282:2048]
2025-11-28T14:53:52.576544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789609446545282:2048] [node 2] ICS09 handshake done sender: [1:7577789609446545279:2117] self: [1:7577789609446545278:0] peer: [2:7577789611978791373:0] socket: 16 qp: 4387
2025-11-28T14:53:52.576574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789609446545282:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:52.576602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:52.577041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:52.577092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:52.577125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789611978791375:2048]
2025-11-28T14:53:52.577154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789611978791375:2048] [node 1] ICS09 handshake done sender: [2:7577789611978791372:2122] self: [2:7577789611978791373:0] peer: [1:7577789609446545278:0] socket: 18 qp: 4386
2025-11-28T14:53:52.577174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789611978791375:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:52.577195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:53:52.853277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789609446545282:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-28T14:53:52.853474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789609446545282:2048] VirtualId# [1:7577789609446545278:0]
2025-11-28T14:53:52.853507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789609446545282:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:52.853556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:52.853586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:52.853856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:52.853892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789611978791375:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:52.853950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789611978791375:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:52.853987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:52.854060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:52.854263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789611978791375:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:52.854290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789611978791375:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:52.859028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789609446545284:2118]) peer tries to resume nonexistent session Self# [2:7577789611978791373:0] Peer# [1:7577789609446545278:0]
2025-11-28T14:53:52.859097Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789609446545284:2118] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:52.859831Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789611978791377:2124] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:52.860795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:52.860856Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:52.860877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789611978791375:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:52.861030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789611978791375:2048] VirtualId# [2:7577789611978791373:0]
2025-11-28T14:53:52.861113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 9588
TEvUndelivered Cookie# 9589
TEvUndelivered Cookie# 9590
TEvUndelivered Cookie# 9591
2025-11-28T14:53:52.867185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789611978791379:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:52.868379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:52.869093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789609446545286:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:52.869304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789609446545288:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:52.877270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789609446545286:2119] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:52.877645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:52.877730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:52.877767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789609446545291:2048]
2025-11-28T14:53:52.877800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789609446545291:2048] [node 2] ICS09 handshake done sender: [1:7577789609446545286:2119] self: [1:7577789609446545285:0] peer: [2:7577789611978791383:0] socket: 20 qp: 4389
2025-11-28T14:53:52.877824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789609446545291:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:52.877861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:52.877144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789611978791379:2125] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:52.877440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:52.877587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:52.877637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789611978791385:2048]
2025-11-28T14:53:52.877675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789611978791385:2048] [node 1] ICS09 handshake done sender: [2:7577789611978791379:2125] self: [2:7577789611978791383:0] peer: [1:7577789609446545285:0] socket: 23 qp: 4390
2025-11-28T14:53:52.877711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789611978791385:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:52.877756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:53:53.048571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:53:53.048870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789611978791385:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-28T14:53:53.048893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789611978791385:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:53.048930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:53.048956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789611978791385:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:53.049783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:53.050114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789609446545291:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:53:53.050138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789609446545291:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:53.050165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:53.050198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789609446545291:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:53.053683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789611978791385:2048] [node 1] ICS08 incoming handshake Self# [1:7577789609446545285:0] Peer# [2:7577789611978791383:0] Counter# 823 LastInputSerial# 20
2025-11-28T14:53:53.053512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789609446545291:2048] [node 2] ICS08 incoming handshake Self# [2:7577789611978791383:0] Peer# [1:7577789609446545285:0] Counter# 20 LastInputSerial# 823
2025-11-28T14:53:53.054186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789613741512589:2121] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:53.054414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:53.054464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789609446545291:2048] [node 2] ICS09 handshake done sender: [1:7577789613741512589:2121] self: [1:7577789609446545285:0] peer: [2:7577789611978791383:0] socket: 24 qp: 4395
2025-11-28T14:53:53.056108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789616273758685:2129] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:53.056562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789609446545291:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:53.056604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:53.057649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:53.057699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789611978791385:2048] [node 1] ICS09 handshake done sender: [2:7577789616273758685:2129] self: [2:7577789611978791383:0] peer: [1:7577789609446545285:0] socket: 25 qp: 4396
2025-11-28T14:53:53.058261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789611978791385:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:53.058297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:53:53.447561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:53:53.447703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:53.447899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789611978791385:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-28T14:53:53.447919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789611978791385:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:53.447942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:53.447978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789611978791385:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:53.447750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:53.447984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789609446545291:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-28T14:53:53.448005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789609446545291:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:53.448057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:53.448096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789609446545291:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:53.453838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789609446545291:2048] [node 2] ICS08 incoming handshake Self# [2:7577789611978791383:0] Peer# [1:7577789609446545285:0] Counter# 109 LastInputSerial# 4403
2025-11-28T14:53:53.454851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789611978791385:2048] [node 1] ICS08 incoming handshake Self# [1:7577789609446545285:0] Peer# [2:7577789611978791383:0] Counter# 4403 LastInputSerial# 109
2025-11-28T14:53:53.456839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789613741512594:2123] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:53.457089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:53.457144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789609446545291:2048] [node 2] ICS09 handshake done sender: [1:7577789613741512594:2123] self: [1:7577789609446545285:0] peer: [2:7577789611978791383:0] socket: 20 qp: 4398
2025-11-28T14:53:53.457426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789609446545291:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:53.457468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:53.459763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789616273758690:2132] held: no
2025-11-28T14:53:53.459829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789616273758690:2132] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:53.459951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:53.459998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789611978791385:2048] [node 1] ICS09 handshake done sender: [2:7577789616273758690:2132] self: [2:7577789611978791383:0] peer: [1:7577789609446545285:0] socket: 21 qp: 4400
2025-11-28T14:53:53.460341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789611978791385:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:53.460408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:53:53.902126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789611978791385:2048] [node 1] ICS01 socket: 21 reason# Debug
2025-11-28T14:53:53.902213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789611978791385:2048] VirtualId# [2:7577789611978791383:0]
2025-11-28T14:53:53.902241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789611978791385:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:53.902289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:53.902334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:53:53.903246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:53.903495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789609446545291:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:53:53.903521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789609446545291:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:53.903543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:53.903594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789609446545291:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:53.906274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789616273758693:2134]) peer tries to resume nonexistent session Self# [1:7577789609446545285:0] Peer# [2:7577789611978791383:0]
2025-11-28T14:53:53.906336Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789616273758693:2134] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:53.906505Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789613741512599:2125] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:53.908456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:53.908544Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:53.908566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789609446545291:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:53.908618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789609446545291:2048] VirtualId# [1:7577789609446545285:0]
2025-11-28T14:53:53.908667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:53.912512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789616273758694:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:53.912656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:53.912980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789613741512602:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:53.921309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789613741512602:2126] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:53.921575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:53.921616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:53.921654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789613741512605:2048]
2025-11-28T14:53:53.921688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789613741512605:2048] [node 2] ICS09 handshake done sender: [1:7577789613741512602:2126] self: [1:7577789613741512601:0] peer: [2:7577789616273758695:0] socket: 16 qp: 4403
2025-11-28T14:53:53.921715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789613741512605:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:53.921744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:53.920641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789616273758694:2135] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:53.922223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:53.922253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:53.922284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789616273758697:2048]
2025-11-28T14:53:53.922314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789616273758697:2048] [node 1] ICS09 handshake done sender: [2:7577789616273758694:2135] self: [2:7577789616273758695:0] peer: [1:7577789613741512601:0] socket: 18 qp: 4402
2025-11-28T14:53:53.922344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789616273758697:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:53.922368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:53:54.339556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:53:54.339785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789613741512605:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:53:54.339825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789613741512605:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:54.339870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:54.339897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789613741512605:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:54.342298Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:54.342496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789616273758697:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:54.342517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789616273758697:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:54.342579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:54.342611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789616273758697:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:54.346341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789616273758697:2048] [node 1] ICS08 incoming handshake Self# [1:7577789613741512601:0] Peer# [2:7577789616273758695:0] Counter# 2785 LastInputSerial# 76
2025-11-28T14:53:54.346935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789618036479903:2127] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:54.347175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:54.347226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789613741512605:2048] [node 2] ICS09 handshake done sender: [1:7577789618036479903:2127] self: [1:7577789613741512601:0] peer: [2:7577789616273758695:0] socket: 22 qp: 4404
2025-11-28T14:53:54.347653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789613741512605:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:54.347694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:54.350988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789613741512605:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:54.351019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789613741512605:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:54.351066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:54.351105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789613741512605:2048] [node 2] ICS08 incoming handshake Self# [2:7577789616273758695:0] Peer# [1:7577789613741512601:0] Counter# 76 LastInputSerial# 2785
2025-11-28T14:53:54.351354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:54.351550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789613741512605:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:54.351603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789613741512605:2048] VirtualId# [1:7577789613741512601:0]
2025-11-28T14:53:54.351632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:54.354779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789620568725996:2138] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:54.355084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:54.355479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789616273758697:2048] [node 1] ICS09 handshake done sender: [2:7577789620568725996:2138] self: [2:7577789616273758695:0] peer: [1:7577789613741512601:0] socket: 23 qp: 4405
2025-11-28T14:53:54.355972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789616273758697:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:54.356021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:54.356293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:54.356314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789616273758697:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:54.356351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789616273758697:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:54.356393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:54.356485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:54.356667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789616273758697:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:54.356683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789616273758697:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:54.364753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789618036479909:2130]) peer tries to resume nonexistent session Self# [2:7577789616273758695:0] Peer# [1:7577789613741512601:0]
2025-11-28T14:53:54.364813Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789618036479909:2130] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:54.368420Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789620568726001:2140] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:54.369468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:54.369542Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:54.369569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789616273758697:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:54.369623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789616273758697:2048] VirtualId# [2:7577789616273758695:0]
2025-11-28T14:53:54.369673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:53:54.374527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789618036479910:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:54.374638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:54.375903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789620568726004:2141] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:54.380878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789618036479910:2131] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:54.381071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789620568726004:2141] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:54.381308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:54.381351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:54.381391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789620568726007:2048]
2025-11-28T14:53:54.381460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789620568726007:2048] [node 1] ICS09 handshake done sender: [2:7577789620568726004:2141] self: [2:7577789620568726003:0] peer: [1:7577789618036479911:0] socket: 16 qp: 4410
2025-11-28T14:53:54.381493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789620568726007:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:54.381525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:54.382009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:54.382044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:54.382077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789618036479913:2048]
2025-11-28T14:53:54.382107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789618036479913:2048] [node 2] ICS09 handshake done sender: [1:7577789618036479910:2131] self: [1:7577789618036479911:0] peer: [2:7577789620568726003:0] socket: 18 qp: 4409
2025-11-28T14:53:54.382128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789618036479913:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:54.382151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:53:54.761744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:54.761994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789618036479913:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:54.762020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789618036479913:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:54.762086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:54.762122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789618036479913:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 13065
TEvUndelivered Cookie# 13066
2025-11-28T14:53:54.760558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789620568726007:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-28T14:53:54.760654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789620568726007:2048] VirtualId# [2:7577789620568726003:0]
2025-11-28T14:53:54.760680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789620568726007:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:54.760743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:54.760803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:54.766103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789620568726009:2142]) peer tries to resume nonexistent session Self# [1:7577789618036479911:0] Peer# [2:7577789620568726003:0]
2025-11-28T14:53:54.766148Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789620568726009:2142] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:54.766560Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789618036479915:2133] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:54.767168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:54.767211Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:54.767226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789618036479913:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:54.767272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789618036479913:2048] VirtualId# [1:7577789618036479911:0]
2025-11-28T14:53:54.767295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:54.773159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789620568726013:2144] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:54.773258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:54.772883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789618036479917:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:54.776907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789618036479919:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:54.788908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789618036479919:2135] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:54.789266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:54.789335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:54.788695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789620568726013:2144] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:54.789399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789618036479923:2048]
2025-11-28T14:53:54.789443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789618036479923:2048] [node 2] ICS09 handshake done sender: [1:7577789618036479919:2135] self: [1:7577789618036479918:0] peer: [2:7577789620568726014:0] socket: 18 qp: 4413
2025-11-28T14:53:54.789479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789618036479923:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:54.789514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:54.795835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:54.795897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:54.795946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789620568726016:2048]
2025-11-28T14:53:54.795986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789620568726016:2048] [node 1] ICS09 handshake done sender: [2:7577789620568726013:2144] self: [2:7577789620568726014:0] peer: [1:7577789618036479918:0] socket: 16 qp: 4415
2025-11-28T14:53:54.796020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789620568726016:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:54.796053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:53:55.329581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:55.329833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789618036479923:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:55.329867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789618036479923:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:55.329926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:55.329958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789618036479923:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-28T14:53:55.328667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789620568726016:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-28T14:53:55.328762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789620568726016:2048] VirtualId# [2:7577789620568726014:0]
2025-11-28T14:53:55.328804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789620568726016:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:55.328857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:55.328904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:55.333945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789624863693314:2146]) peer tries to resume nonexistent session Self# [1:7577789618036479918:0] Peer# [2:7577789620568726014:0]
2025-11-28T14:53:55.334008Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789624863693314:2146] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:55.334269Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789622331447221:2136] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:55.334868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:55.334934Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:55.334955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789618036479923:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:55.335090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789618036479923:2048] VirtualId# [1:7577789618036479918:0]
2025-11-28T14:53:55.335134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:55.339856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789624863693315:2147] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:55.340004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:55.340493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789622331447224:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:55.347958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789624863693315:2147] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:55.348245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789622331447224:2137] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:55.348530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:55.348576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:55.348616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789622331447227:2048]
2025-11-28T14:53:55.348654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789622331447227:2048] [node 2] ICS09 handshake done sender: [1:7577789622331447224:2137] self: [1:7577789622331447223:0] peer: [2:7577789624863693316:0] socket: 18 qp: 4417
2025-11-28T14:53:55.348681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789622331447227:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:55.348713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:55.349177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:55.349212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:55.349245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789624863693318:2048]
2025-11-28T14:53:55.349282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789624863693318:2048] [node 1] ICS09 handshake done sender: [2:7577789624863693315:2147] self: [2:7577789624863693316:0] peer: [1:7577789622331447223:0] socket: 16 qp: 4418
2025-11-28T14:53:55.349308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789624863693318:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:55.349332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:53:55.643075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:53:55.643269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789624863693318:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:53:55.643292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789624863693318:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:55.643343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:55.643370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789624863693318:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:55.645490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:55.645699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789622331447227:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:55.645719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789622331447227:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:55.645771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:55.645793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789622331447227:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:55.649502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789624863693318:2048] [node 1] ICS08 incoming handshake Self# [1:7577789622331447223:0] Peer# [2:7577789624863693316:0] Counter# 1894 LastInputSerial# 48
2025-11-28T14:53:55.651130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789622331447229:2138] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:55.651385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:55.651420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789622331447227:2048] [node 2] ICS09 handshake done sender: [1:7577789622331447229:2138] self: [1:7577789622331447223:0] peer: [2:7577789624863693316:0] socket: 22 qp: 4419
2025-11-28T14:53:55.651898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789622331447227:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:55.651946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:55.655732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789624863693321:2150] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:55.655999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:55.656069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789624863693318:2048] [node 1] ICS09 handshake done sender: [2:7577789624863693321:2150] self: [2:7577789624863693316:0] peer: [1:7577789622331447223:0] socket: 23 qp: 4420
2025-11-28T14:53:55.656649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789624863693318:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:55.656687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:55.656921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:55.656938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789624863693318:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:55.656954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789624863693318:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:55.656969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:55.657015Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:55.657128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789624863693318:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:55.657143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789624863693318:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:55.656440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789622331447227:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:55.656481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789622331447227:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:55.656539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:55.656575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789622331447227:2048] [node 2] ICS08 incoming handshake Self# [2:7577789624863693316:0] Peer# [1:7577789622331447223:0] Counter# 48 LastInputSerial# 1894
2025-11-28T14:53:55.658057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-11-28T14:53:55.658207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789622331447227:2048] [node 2] ICS07 socket disconnect -1 reason# ECONNRESET
2025-11-28T14:53:55.660515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789622331447227:2048] [node 2] ICS08 incoming handshake Self# [2:7577789624863693316:0] Peer# [1:7577789622331447223:0] Counter# 48 LastInputSerial# 1894
2025-11-28T14:53:55.662787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789624863693325:2152] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:55.662985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:55.663026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789624863693318:2048] [node 1] ICS09 handshake done sender: [2:7577789624863693325:2152] self: [2:7577789624863693316:0] peer: [1:7577789622331447223:0] socket: 18 qp: 4423
2025-11-28T14:53:55.663327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789624863693318:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:55.663352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:55.664743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789622331447234:2140] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:55.665107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:55.665139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789622331447227:2048] [node 2] ICS09 handshake done sender: [1:7577789622331447234:2140] self: [1:7577789622331447223:0] peer: [2:7577789624863693316:0] socket: 20 qp: 4424
2025-11-28T14:53:55.665571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789622331447227:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:55.665609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:53:55.924456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:53:55.924594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:53:55.924618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789624863693318:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:55.924640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789624863693318:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:55.924659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:55.925247Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:55.925506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789622331447227:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:53:55.925526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789622331447227:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:55.925584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:55.925614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789622331447227:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:55.927996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:55.928197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789624863693318:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:55.928215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789624863693318:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:55.929866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789624863693318:2048] [node 1] ICS08 incoming handshake Self# [1:7577789622331447223:0] Peer# [2:7577789624863693316:0] Counter# 4233 LastInputSerial# 107
2025-11-28T14:53:55.931794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789622331447237:2142] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:55.935764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789624863693330:2154] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:55.941337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:55.941393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789622331447227:2048] [node 2] ICS09 handshake done sender: [1:7577789622331447237:2142] self: [1:7577789622331447223:0] peer: [2:7577789624863693316:0] socket: 16 qp: 4425
2025-11-28T14:53:55.941978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789622331447227:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:55.942024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:55.942235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789622331447227:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:55.942260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789622331447227:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:55.942284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:55.942330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789622331447227:2048] [node 2] ICS08 incoming handshake Self# [2:7577789624863693316:0] Peer# [1:7577789622331447223:0] Counter# 107 LastInputSerial# 4233
2025-11-28T14:53:55.942401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:55.942574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789622331447227:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:55.943151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:55.943212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789624863693318:2048] [node 1] ICS09 handshake done sender: [2:7577789624863693330:2154] self: [2:7577789624863693316:0] peer: [1:7577789622331447223:0] socket: 22 qp: 4426
2025-11-28T14:53:55.943798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789624863693318:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:55.943841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:55.944146Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Connection reset by peer
2025-11-28T14:53:55.944167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789624863693318:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:55.944191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789624863693318:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:55.944213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:55.944339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:55.944526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789624863693318:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:55.944540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789624863693318:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:55.950148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789622331447227:2048] [node 2] ICS08 incoming handshake Self# [2:7577789624863693316:0] Peer# [1:7577789622331447223:0] Counter# 108 LastInputSerial# 4233
2025-11-28T14:53:55.952853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789624863693334:2156] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:55.953096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:55.953145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789624863693318:2048] [node 1] ICS09 handshake done sender: [2:7577789624863693334:2156] self: [2:7577789624863693316:0] peer: [1:7577789622331447223:0] socket: 16 qp: 4430
2025-11-28T14:53:55.953478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789624863693318:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:55.953529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:55.955264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789622331447242:2144] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:55.955616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:55.955662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789622331447227:2048] [node 2] ICS09 handshake done sender: [1:7577789622331447242:2144] self: [1:7577789622331447223:0] peer: [2:7577789624863693316:0] socket: 21 qp: 4429
2025-11-28T14:53:55.956032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789622331447227:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:55.956070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:53:56.515967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789622331447227:2048] [node 2] ICS01 socket: 21 reason# Debug
2025-11-28T14:53:56.516062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789622331447227:2048] VirtualId# [1:7577789622331447223:0]
2025-11-28T14:53:56.516092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789622331447227:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:56.516141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:56.516183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:56.522011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:56.522203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789624863693318:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:53:56.522243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789624863693318:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:56.522304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:56.522337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789624863693318:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:56.536801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789626626414544:2147]) peer tries to resume nonexistent session Self# [2:7577789624863693316:0] Peer# [1:7577789622331447223:0]
2025-11-28T14:53:56.536849Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789626626414544:2147] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:56.537590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789629158660636:2158] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:56.537657Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789629158660635:2157] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:56.538357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:56.538425Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:56.538448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789624863693318:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:56.538501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789624863693318:2048] VirtualId# [2:7577789624863693316:0]
2025-11-28T14:53:56.538547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:53:56.539534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:56.540108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789626626414542:2146] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:56.554958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789626626414545:2148] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:56.555331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789629158660636:2158] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:56.555514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789626626414542:2146] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:56.555725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:56.555777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:56.555814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789626626414548:2048]
2025-11-28T14:53:56.555849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789626626414548:2048] [node 2] ICS09 handshake done sender: [1:7577789626626414542:2146] self: [1:7577789626626414541:0] peer: [2:7577789629158660640:0] socket: 18 qp: 4432
2025-11-28T14:53:56.555880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789626626414548:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:56.555911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:56.557441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:56.557481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:56.557514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789629158660643:2048]
2025-11-28T14:53:56.557555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789629158660643:2048] [node 1] ICS09 handshake done sender: [2:7577789629158660636:2158] self: [2:7577789629158660640:0] peer: [1:7577789626626414541:0] socket: 20 qp: 4431
2025-11-28T14:53:56.557579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789629158660643:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:56.557604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:53:56.979545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:53:56.980310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:56.980603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789626626414548:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:56.980625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789626626414548:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:56.980656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:56.980695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789626626414548:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:56.983631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:56.983817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789629158660643:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:53:56.983835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789629158660643:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:56.983881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:56.983908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789629158660643:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:56.985535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789629158660643:2048] [node 1] ICS08 incoming handshake Self# [1:7577789626626414541:0] Peer# [2:7577789629158660640:0] Counter# 3126 LastInputSerial# 75
2025-11-28T14:53:56.988571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789626626414550:2149] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:56.988821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:56.988866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789626626414548:2048] [node 2] ICS09 handshake done sender: [1:7577789626626414550:2149] self: [1:7577789626626414541:0] peer: [2:7577789629158660640:0] socket: 16 qp: 4436
2025-11-28T14:53:56.989273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789626626414548:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:56.989308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:56.991298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789626626414548:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:56.991326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789626626414548:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:56.991349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:56.991396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789626626414548:2048] [node 2] ICS08 incoming handshake Self# [2:7577789629158660640:0] Peer# [1:7577789626626414541:0] Counter# 75 LastInputSerial# 3126
2025-11-28T14:53:56.991601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:56.991774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789626626414548:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:56.992568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789629158660646:2162] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:56.992913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:56.992963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789629158660643:2048] [node 1] ICS09 handshake done sender: [2:7577789629158660646:2162] self: [2:7577789629158660640:0] peer: [1:7577789626626414541:0] socket: 21 qp: 4437
2025-11-28T14:53:56.993507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789629158660643:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:56.993541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:56.998898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Connection reset by peer
2025-11-28T14:53:56.998928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789629158660643:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:56.998951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789629158660643:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:56.998975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:56.999192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:56.999375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789629158660643:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:56.999392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789629158660643:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:57.002909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789626626414548:2048] [node 2] ICS08 incoming handshake Self# [2:7577789629158660640:0] Peer# [1:7577789626626414541:0] Counter# 76 LastInputSerial# 3126
2025-11-28T14:53:57.005358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789629158660651:2164] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:57.005586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:53:57.005633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789629158660643:2048] [node 1] ICS09 handshake done sender: [2:7577789629158660651:2164] self: [2:7577789629158660640:0] peer: [1:7577789626626414541:0] socket: 18 qp: 4441
2025-11-28T14:53:57.005979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789629158660643:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:57.006012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:57.007413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789626626414556:2152] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:57.007696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:53:57.007741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789626626414548:2048] [node 2] ICS09 handshake done sender: [1:7577789626626414556:2152] self: [1:7577789626626414541:0] peer: [2:7577789629158660640:0] socket: 20 qp: 4440
2025-11-28T14:53:57.008107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789626626414548:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:57.008142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:53:57.276395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:53:57.276649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789626626414548:2048] [node 2] ICS07 socket disconnect 20 reason# Debug
2025-11-28T14:53:57.276673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789626626414548:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:57.276700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:57.276736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789626626414548:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:57.280955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789629158660643:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:57.280985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789629158660643:2048] [node 1] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:57.281023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:57.281069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789629158660643:2048] [node 1] ICS08 incoming handshake Self# [1:7577789626626414541:0] Peer# [2:7577789629158660640:0] Counter# 5705 LastInputSerial# 139
2025-11-28T14:53:57.281314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:57.281499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789629158660643:2048] [node 1] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:57.281557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789629158660643:2048] VirtualId# [2:7577789629158660640:0]
2025-11-28T14:53:57.281592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:57.282403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789630921381855:2154] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:57.282623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:57.282669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789626626414548:2048] [node 2] ICS09 handshake done sender: [1:7577789630921381855:2154] self: [1:7577789626626414541:0] peer: [2:7577789629158660640:0] socket: 16 qp: 4443
2025-11-28T14:53:57.283204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789626626414548:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:57.283240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
TEvNodeDisconnected
2025-11-28T14:53:57.286063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789633453627951:2165] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:57.286267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:57.286308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789586208987432:2048] [node 1] ICP32 transit to hold-by-error state Explanation# Session continuation race
2025-11-28T14:53:57.286320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: Session continuation race
2025-11-28T14:53:57.286994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-11-28T14:53:57.287184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789626626414548:2048] [node 2] ICS07 socket disconnect 16 reason# ECONNRESET
2025-11-28T14:53:57.287206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789626626414548:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:57.287229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:57.287275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789626626414548:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:57.290387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789633453627953:2167]) peer tries to resume nonexistent session Self# [1:7577789626626414541:0] Peer# [2:7577789629158660640:0]
2025-11-28T14:53:57.290443Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789633453627953:2167] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:57.290971Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789630921381859:2155] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:57.291705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:57.291769Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:57.291790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789626626414548:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:57.291846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789626626414548:2048] VirtualId# [1:7577789626626414541:0]
2025-11-28T14:53:57.291889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:57.296235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789633453627954:2168] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:57.296394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:57.296769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789630921381862:2156] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:57.299332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789633453627954:2168] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:57.299484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789630921381862:2156] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:57.299682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:57.299722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:57.299761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789630921381865:2048]
2025-11-28T14:53:57.299798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789630921381865:2048] [node 2] ICS09 handshake done sender: [1:7577789630921381862:2156] self: [1:7577789630921381861:0] peer: [2:7577789633453627955:0] socket: 16 qp: 4445
2025-11-28T14:53:57.299826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789630921381865:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:57.299856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:57.300300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:57.300331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:57.300387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789633453627957:2048]
2025-11-28T14:53:57.300417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789633453627957:2048] [node 1] ICS09 handshake done sender: [2:7577789633453627954:2168] self: [2:7577789633453627955:0] peer: [1:7577789630921381861:0] socket: 18 qp: 4446
2025-11-28T14:53:57.300438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789633453627957:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:57.300461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:53:57.646556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:57.646758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789633453627957:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:57.646775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789633453627957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:57.646825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:57.646858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789633453627957:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:57.646231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789630921381865:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-28T14:53:57.646405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789630921381865:2048] VirtualId# [1:7577789630921381861:0]
2025-11-28T14:53:57.646427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789630921381865:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:57.646475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:57.646517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:57.650832Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789633453627959:2170] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:53:57.651976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:57.652034Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:57.652057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789633453627957:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:57.650449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789630921381867:2157]) peer tries to resume nonexistent session Self# [2:7577789633453627955:0] Peer# [1:7577789630921381861:0]
2025-11-28T14:53:57.650561Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789630921381867:2157] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:57.652230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789633453627957:2048] VirtualId# [2:7577789633453627955:0]
2025-11-28T14:53:57.652291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:53:57.655813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789633453627963:2172] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:57.657044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:57.657628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789630921381869:2158] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:57.657792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789630921381871:2159] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:57.660733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789633453627963:2172] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:57.661004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:57.661049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:57.661085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789633453627967:2048]
2025-11-28T14:53:57.661119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789633453627967:2048] [node 1] ICS09 handshake done sender: [2:7577789633453627963:2172] self: [2:7577789633453627965:0] peer: [1:7577789630921381868:0] socket: 18 qp: 4449
2025-11-28T14:53:57.661145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789633453627967:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:57.661188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:57.663837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789630921381869:2158] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:57.664096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:57.664153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:57.664189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789630921381874:2048]
2025-11-28T14:53:57.664223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789630921381874:2048] [node 2] ICS09 handshake done sender: [1:7577789630921381869:2158] self: [1:7577789630921381868:0] peer: [2:7577789633453627965:0] socket: 16 qp: 4448
2025-11-28T14:53:57.664262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789630921381874:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:57.664290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:53:57.823720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Connection reset by peer
2025-11-28T14:53:57.823760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789630921381874:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:57.823792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789630921381874:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:57.823826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:57.821217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:53:57.821519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789633453627967:2048] [node 1] ICS07 socket disconnect 18 reason# Debug
2025-11-28T14:53:57.821559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789633453627967:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:57.821607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:57.821642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789633453627967:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:57.825680Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:57.825925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789630921381874:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:57.825944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789630921381874:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:57.829989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789630921381874:2048] [node 2] ICS08 incoming handshake Self# [2:7577789633453627965:0] Peer# [1:7577789630921381868:0] Counter# 14 LastInputSerial# 526
2025-11-28T14:53:57.829989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789633453627967:2048] [node 1] ICS08 incoming handshake Self# [1:7577789630921381868:0] Peer# [2:7577789633453627965:0] Counter# 526 LastInputSerial# 14
2025-11-28T14:53:57.830470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789630921381877:2161] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:57.830892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:57.830957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789630921381874:2048] [node 2] ICS09 handshake done sender: [1:7577789630921381877:2161] self: [1:7577789630921381868:0] peer: [2:7577789633453627965:0] socket: 24 qp: 4454
2025-11-28T14:53:57.831592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789630921381874:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:57.831635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:57.833898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789633453627971:2175] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:57.834272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789633453627971:2175] held: no
2025-11-28T14:53:57.834303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:57.834342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789633453627967:2048] [node 1] ICS09 handshake done sender: [2:7577789633453627971:2175] self: [2:7577789633453627965:0] peer: [1:7577789630921381868:0] socket: 25 qp: 4455
2025-11-28T14:53:57.834706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789633453627967:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:57.834745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:53:58.392457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:53:58.393503Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:53:58.393529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789630921381874:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:53:58.393550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789630921381874:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:53:58.393579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:58.393790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:58.394149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789630921381874:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:58.394170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789630921381874:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:58.399141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789635216349177:2162] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:58.399355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:58.399397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789630921381874:2048] [node 2] ICS09 handshake done sender: [1:7577789635216349177:2162] self: [1:7577789630921381868:0] peer: [2:7577789633453627965:0] socket: 16 qp: 4456
2025-11-28T14:53:58.399960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789630921381874:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:58.399999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
TEvNodeDisconnected
2025-11-28T14:53:58.398530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789633453627967:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:58.398563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789633453627967:2048] [node 1] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:58.398614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:58.398665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789633453627967:2048] [node 1] ICS08 incoming handshake Self# [1:7577789630921381868:0] Peer# [2:7577789633453627965:0] Counter# 5522 LastInputSerial# 137
2025-11-28T14:53:58.400240Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:58.400441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789633453627967:2048] [node 1] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:58.400512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789633453627967:2048] VirtualId# [2:7577789633453627965:0]
2025-11-28T14:53:58.400546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:58.404347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789637748595270:2177] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:58.407821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-11-28T14:53:58.408024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789630921381874:2048] [node 2] ICS07 socket disconnect 16 reason# ECONNRESET
2025-11-28T14:53:58.408046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789630921381874:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:58.408070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:58.408113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789630921381874:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:58.407141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:58.407181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789586208987432:2048] [node 1] ICP32 transit to hold-by-error state Explanation# Session continuation race
2025-11-28T14:53:58.407194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: Session continuation race
2025-11-28T14:53:58.411973Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789635216349181:2163] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:58.412673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:58.412744Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:58.412770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789630921381874:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:58.412820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789630921381874:2048] VirtualId# [1:7577789630921381868:0]
2025-11-28T14:53:58.412863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:58.411381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789637748595272:2179]) peer tries to resume nonexistent session Self# [1:7577789630921381868:0] Peer# [2:7577789633453627965:0]
2025-11-28T14:53:58.411437Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789637748595272:2179] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:58.421203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789635216349184:2164] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:58.420614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789637748595273:2180] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:58.420895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:58.431404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789635216349184:2164] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:58.431628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:58.431677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:58.431718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789635216349187:2048]
2025-11-28T14:53:58.431754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789635216349187:2048] [node 2] ICS09 handshake done sender: [1:7577789635216349184:2164] self: [1:7577789635216349183:0] peer: [2:7577789637748595274:0] socket: 16 qp: 4460
2025-11-28T14:53:58.431784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789635216349187:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:58.431824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:58.431229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789637748595273:2180] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:58.432186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:58.432213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:58.432241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789637748595276:2048]
2025-11-28T14:53:58.432271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789637748595276:2048] [node 1] ICS09 handshake done sender: [2:7577789637748595273:2180] self: [2:7577789637748595274:0] peer: [1:7577789635216349183:0] socket: 18 qp: 4459
2025-11-28T14:53:58.432292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789637748595276:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:58.432314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:53:58.909245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:53:58.910087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:53:58.910342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789635216349187:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:53:58.910366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789635216349187:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:58.910397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:58.910440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789635216349187:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:58.910955Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:58.911168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789637748595276:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:53:58.911200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789637748595276:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:53:58.911264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:58.911293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789637748595276:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:58.919713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789635216349187:2048] [node 2] ICS08 incoming handshake Self# [2:7577789637748595274:0] Peer# [1:7577789635216349183:0] Counter# 83 LastInputSerial# 3567
2025-11-28T14:53:58.917786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789637748595276:2048] [node 1] ICS08 incoming handshake Self# [1:7577789635216349183:0] Peer# [2:7577789637748595274:0] Counter# 3567 LastInputSerial# 83
2025-11-28T14:53:58.922058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789635216349189:2165] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:58.922376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:58.922432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789635216349187:2048] [node 2] ICS09 handshake done sender: [1:7577789635216349189:2165] self: [1:7577789635216349183:0] peer: [2:7577789637748595274:0] socket: 22 qp: 4462
2025-11-28T14:53:58.922884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789635216349187:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:58.922924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:58.924569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789637748595279:2183] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:58.924746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:58.924793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789637748595276:2048] [node 1] ICS09 handshake done sender: [2:7577789637748595279:2183] self: [2:7577789637748595274:0] peer: [1:7577789635216349183:0] socket: 23 qp: 4461
2025-11-28T14:53:58.925317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789637748595276:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:58.925360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:53:59.055948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:53:59.056185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789635216349187:2048] [node 2] ICS07 socket disconnect 22 reason# Debug
2025-11-28T14:53:59.056208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789635216349187:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:59.056236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:59.056270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789635216349187:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:59.061544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789639511316490:2167] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:59.061827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:59.061870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789635216349187:2048] [node 2] ICS09 handshake done sender: [1:7577789639511316490:2167] self: [1:7577789635216349183:0] peer: [2:7577789637748595274:0] socket: 16 qp: 4465
2025-11-28T14:53:59.062248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789635216349187:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:59.062284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
TEvNodeDisconnected
2025-11-28T14:53:59.060938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789637748595276:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:59.060977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789637748595276:2048] [node 1] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:59.061027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:59.061086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789637748595276:2048] [node 1] ICS08 incoming handshake Self# [1:7577789635216349183:0] Peer# [2:7577789637748595274:0] Counter# 4784 LastInputSerial# 112
2025-11-28T14:53:59.062660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:59.063023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789637748595276:2048] [node 1] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:59.063079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789637748595276:2048] VirtualId# [2:7577789637748595274:0]
2025-11-28T14:53:59.063116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:59.066325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789642043562579:2185] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:59.066525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:59.066559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789586208987432:2048] [node 1] ICP32 transit to hold-by-error state Explanation# Session continuation race
2025-11-28T14:53:59.066571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: Session continuation race
2025-11-28T14:53:59.067166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-11-28T14:53:59.067380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789635216349187:2048] [node 2] ICS07 socket disconnect 16 reason# ECONNRESET
2025-11-28T14:53:59.067401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789635216349187:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:59.067426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:59.067473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789635216349187:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:59.070823Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789642043562581:2187]) peer tries to resume nonexistent session Self# [1:7577789635216349183:0] Peer# [2:7577789637748595274:0]
2025-11-28T14:53:59.070883Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789642043562581:2187] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:59.071462Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789639511316494:2168] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:59.072195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:59.072262Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:59.072286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789635216349187:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:59.072340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789635216349187:2048] VirtualId# [1:7577789635216349183:0]
2025-11-28T14:53:59.072795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:59.077135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789642043562582:2188] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:59.077306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:59.079201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789639511316497:2169] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:59.089717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789642043562582:2188] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:59.089944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:59.089986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:59.090026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789642043562585:2048]
2025-11-28T14:53:59.090063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789642043562585:2048] [node 1] ICS09 handshake done sender: [2:7577789642043562582:2188] self: [2:7577789642043562583:0] peer: [1:7577789639511316496:0] socket: 18 qp: 4468
2025-11-28T14:53:59.090093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789642043562585:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:59.090123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:59.090760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789639511316497:2169] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:59.091059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:59.091095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:59.091134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789639511316500:2048]
2025-11-28T14:53:59.091170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789639511316500:2048] [node 2] ICS09 handshake done sender: [1:7577789639511316497:2169] self: [1:7577789639511316496:0] peer: [2:7577789642043562583:0] socket: 16 qp: 4469
2025-11-28T14:53:59.091197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789639511316500:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:59.091226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:53:59.352652Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:53:59.352899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789639511316500:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:53:59.352925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789639511316500:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:59.352960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:59.352996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789639511316500:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:59.357386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789642043562585:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:59.357417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789642043562585:2048] [node 1] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:53:59.357463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:59.357510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789642043562585:2048] [node 1] ICS08 incoming handshake Self# [1:7577789639511316496:0] Peer# [2:7577789642043562583:0] Counter# 1264 LastInputSerial# 28
2025-11-28T14:53:59.357748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:59.357940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789642043562585:2048] [node 1] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:53:59.358308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789642043562585:2048] VirtualId# [2:7577789642043562583:0]
2025-11-28T14:53:59.358350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:53:59.358981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789639511316502:2170] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:59.359212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:59.359260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789639511316500:2048] [node 2] ICS09 handshake done sender: [1:7577789639511316502:2170] self: [1:7577789639511316496:0] peer: [2:7577789642043562583:0] socket: 22 qp: 4471
2025-11-28T14:53:59.359670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789639511316500:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:59.359707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
TEvNodeDisconnected
2025-11-28T14:53:59.362158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789642043562587:2190] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:59.362323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:59.362362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577789586208987432:2048] [node 1] ICP32 transit to hold-by-error state Explanation# Session continuation race
2025-11-28T14:53:59.362373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: Session continuation race
2025-11-28T14:53:59.363001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-11-28T14:53:59.363207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789639511316500:2048] [node 2] ICS07 socket disconnect 22 reason# ECONNRESET
2025-11-28T14:53:59.363227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789639511316500:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:59.363250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:59.363299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789639511316500:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:59.366382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789642043562589:2192]) peer tries to resume nonexistent session Self# [1:7577789639511316496:0] Peer# [2:7577789642043562583:0]
2025-11-28T14:53:59.366441Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789642043562589:2192] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:53:59.366965Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789639511316506:2171] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:53:59.367634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:53:59.367699Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:53:59.367722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789639511316500:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:53:59.367778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789639511316500:2048] VirtualId# [1:7577789639511316496:0]
2025-11-28T14:53:59.367818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:53:59.372102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789642043562590:2193] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:59.372233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:53:59.372647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789639511316509:2172] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:53:59.375196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789642043562590:2193] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:59.375364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789639511316509:2172] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:59.375587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:59.375630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:53:59.375670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789639511316512:2048]
2025-11-28T14:53:59.375707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789639511316512:2048] [node 2] ICS09 handshake done sender: [1:7577789639511316509:2172] self: [1:7577789639511316508:0] peer: [2:7577789642043562591:0] socket: 16 qp: 4473
2025-11-28T14:53:59.375737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789639511316512:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:59.375768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:59.376255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:59.376287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:53:59.376320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789642043562593:2048]
2025-11-28T14:53:59.376353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789642043562593:2048] [node 1] ICS09 handshake done sender: [2:7577789642043562590:2193] self: [2:7577789642043562591:0] peer: [1:7577789639511316508:0] socket: 18 qp: 4474
2025-11-28T14:53:59.376401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789642043562593:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:59.376423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:53:59.927926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:53:59.928189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789639511316512:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:53:59.928217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789639511316512:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:53:59.928254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:53:59.928292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789639511316512:2048] [node 2] ICS15 start handshake
2025-11-28T14:53:59.933686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Connection reset by peer
2025-11-28T14:53:59.933723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789642043562593:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:53:59.933745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789642043562593:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:53:59.933768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:53:59.943497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:53:59.943743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789642043562593:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:53:59.943767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789642043562593:2048] [node 1] ICS15 start handshake
2025-11-28T14:53:59.944086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789642043562593:2048] [node 1] ICS08 incoming handshake Self# [1:7577789639511316508:0] Peer# [2:7577789642043562591:0] Counter# 3858 LastInputSerial# 91
2025-11-28T14:53:59.948643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789639511316514:2173] [node 2] ICH04 handshake succeeded
2025-11-28T14:53:59.949560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789639511316512:2048] [node 2] ICS08 incoming handshake Self# [2:7577789642043562591:0] Peer# [1:7577789639511316508:0] Counter# 91 LastInputSerial# 3858
2025-11-28T14:53:59.949596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:53:59.949639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789639511316512:2048] [node 2] ICS09 handshake done sender: [1:7577789639511316514:2173] self: [1:7577789639511316508:0] peer: [2:7577789642043562591:0] socket: 22 qp: 4476
2025-11-28T14:53:59.949979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789639511316512:2048] [node 2] ICS10 traffic start
2025-11-28T14:53:59.950024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:53:59.958240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789642043562595:2195] [node 1] ICH04 handshake succeeded
2025-11-28T14:53:59.959431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:53:59.959492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789642043562593:2048] [node 1] ICS09 handshake done sender: [2:7577789642043562595:2195] self: [2:7577789642043562591:0] peer: [1:7577789639511316508:0] socket: 23 qp: 4475
2025-11-28T14:53:59.959933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789642043562593:2048] [node 1] ICS10 traffic start
2025-11-28T14:53:59.959971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:53:59.960161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:00.399035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789639511316512:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-28T14:54:00.399124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789639511316512:2048] VirtualId# [1:7577789639511316508:0]
2025-11-28T14:54:00.399153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789639511316512:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:00.399186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:00.399215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:00.400853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:00.401061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789642043562593:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-28T14:54:00.401084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789642043562593:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:00.401163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:00.401199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789642043562593:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:00.406945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789643806283818:2176]) peer tries to resume nonexistent session Self# [2:7577789642043562591:0] Peer# [1:7577789639511316508:0]
2025-11-28T14:54:00.407004Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789643806283818:2176] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:00.409017Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789646338529896:2198] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:00.411215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789646338529898:2199] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:00.411272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:00.411333Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:00.411355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789642043562593:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:00.411410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789642043562593:2048] VirtualId# [2:7577789642043562591:0]
2025-11-28T14:54:00.411459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:00.413123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:00.414532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789643806283816:2175] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:00.419780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789643806283819:2177] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:00.420588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789646338529898:2199] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:00.420652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789643806283816:2175] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:00.421001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:00.421066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:00.421108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789643806283822:2048]
2025-11-28T14:54:00.421090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:00.421125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:00.421154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789646338529904:2048]
2025-11-28T14:54:00.421192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789646338529904:2048] [node 1] ICS09 handshake done sender: [2:7577789646338529898:2199] self: [2:7577789646338529901:0] peer: [1:7577789643806283815:0] socket: 21 qp: 4482
2025-11-28T14:54:00.421212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789646338529904:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:00.421231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:00.421144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789643806283822:2048] [node 2] ICS09 handshake done sender: [1:7577789643806283816:2175] self: [1:7577789643806283815:0] peer: [2:7577789646338529901:0] socket: 18 qp: 4479
2025-11-28T14:54:00.421170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789643806283822:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:00.421212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:00.500193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:00.500704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:00.500926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789643806283822:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:00.500952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789643806283822:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:00.500984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:00.501034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789643806283822:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:00.500613Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:00.500846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789646338529904:2048] [node 1] ICS01 socket: 21 reason# EndOfStream
2025-11-28T14:54:00.501108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789646338529904:2048] VirtualId# [2:7577789646338529901:0]
2025-11-28T14:54:00.501166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789646338529904:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:00.501249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:00.501281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:00.514010Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789643806283824:2178] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:00.514820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:00.514889Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:00.514913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789643806283822:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:00.515047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789643806283822:2048] VirtualId# [1:7577789643806283815:0]
2025-11-28T14:54:00.515096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:00.512913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789646338529906:2202]) peer tries to resume nonexistent session Self# [1:7577789643806283815:0] Peer# [2:7577789646338529901:0]
2025-11-28T14:54:00.512978Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789646338529906:2202] [node 1] ICH08 Continuation request rejected by proxy
Subscribe
2025-11-28T14:54:00.524598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789646338529907:2203] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:00.525817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:00.531659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789643806283827:2179] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:00.532199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789643806283829:2180] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:00.540114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789643806283827:2179] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:00.539945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789646338529907:2203] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:00.540494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:00.540556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:00.540598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789643806283832:2048]
2025-11-28T14:54:00.540635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789643806283832:2048] [node 2] ICS09 handshake done sender: [1:7577789643806283827:2179] self: [1:7577789643806283826:0] peer: [2:7577789646338529911:0] socket: 16 qp: 4486
2025-11-28T14:54:00.540664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789643806283832:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:00.540697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:00.541757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:00.541804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:00.541838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789646338529913:2048]
2025-11-28T14:54:00.541868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789646338529913:2048] [node 1] ICS09 handshake done sender: [2:7577789646338529907:2203] self: [2:7577789646338529911:0] peer: [1:7577789643806283826:0] socket: 18 qp: 4487
2025-11-28T14:54:00.541892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789646338529913:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:00.541915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:00.843479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:00.843620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:00.843823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789646338529913:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:00.843846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789646338529913:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:00.843873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:00.843909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789646338529913:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:00.846183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:00.846429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789643806283832:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:00.846451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789643806283832:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:00.846511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:00.846543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789643806283832:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:00.852233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789646338529913:2048] [node 1] ICS08 incoming handshake Self# [1:7577789643806283826:0] Peer# [2:7577789646338529911:0] Counter# 1724 LastInputSerial# 45
2025-11-28T14:54:00.852690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789643806283832:2048] [node 2] ICS08 incoming handshake Self# [2:7577789646338529911:0] Peer# [1:7577789643806283826:0] Counter# 45 LastInputSerial# 1724
2025-11-28T14:54:00.853442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789643806283834:2181] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:00.853680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:00.853735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789643806283832:2048] [node 2] ICS09 handshake done sender: [1:7577789643806283834:2181] self: [1:7577789643806283826:0] peer: [2:7577789646338529911:0] socket: 24 qp: 4491
2025-11-28T14:54:00.854090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789643806283832:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:00.854129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:00.856388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789646338529917:2207] held: no
2025-11-28T14:54:00.858074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789646338529917:2207] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:00.858263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:00.858319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789646338529913:2048] [node 1] ICS09 handshake done sender: [2:7577789646338529917:2207] self: [2:7577789646338529911:0] peer: [1:7577789643806283826:0] socket: 25 qp: 4493
2025-11-28T14:54:00.858688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789646338529913:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:00.858727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:01.445371Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:01.445637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789643806283832:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-28T14:54:01.445660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789643806283832:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:01.445722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:01.445750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789643806283832:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:01.444951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:01.446983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:01.447233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789646338529913:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-28T14:54:01.447252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789646338529913:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:01.447283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:01.447336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789646338529913:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:01.453822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789643806283832:2048] [node 2] ICS08 incoming handshake Self# [2:7577789646338529911:0] Peer# [1:7577789643806283826:0] Counter# 159 LastInputSerial# 6302
2025-11-28T14:54:01.454413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789646338529913:2048] [node 1] ICS08 incoming handshake Self# [1:7577789643806283826:0] Peer# [2:7577789646338529911:0] Counter# 6302 LastInputSerial# 159
2025-11-28T14:54:01.455107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789648101251135:2183] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:01.455395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:01.455448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789643806283832:2048] [node 2] ICS09 handshake done sender: [1:7577789648101251135:2183] self: [1:7577789643806283826:0] peer: [2:7577789646338529911:0] socket: 18 qp: 4494
2025-11-28T14:54:01.456046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789643806283832:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:01.456084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:01.460929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789650633497217:2210] held: no
2025-11-28T14:54:01.461002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789650633497217:2210] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:01.461695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:01.461757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789646338529913:2048] [node 1] ICS09 handshake done sender: [2:7577789650633497217:2210] self: [2:7577789646338529911:0] peer: [1:7577789643806283826:0] socket: 20 qp: 4495
2025-11-28T14:54:01.462145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789646338529913:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:01.462184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:01.948520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:01.948683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:01.948933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789643806283832:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:01.948958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789643806283832:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:01.948986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:01.949025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789643806283832:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:01.960978Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:01.961216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789646338529913:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:54:01.961244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789646338529913:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:01.961312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:01.961349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789646338529913:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:01.972824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789646338529913:2048] [node 1] ICS08 incoming handshake Self# [1:7577789643806283826:0] Peer# [2:7577789646338529911:0] Counter# 10239 LastInputSerial# 262
2025-11-28T14:54:01.974336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789643806283832:2048] [node 2] ICS08 incoming handshake Self# [2:7577789646338529911:0] Peer# [1:7577789643806283826:0] Counter# 262 LastInputSerial# 10239
2025-11-28T14:54:01.974699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789648101251140:2185] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:01.974888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:01.974937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789643806283832:2048] [node 2] ICS09 handshake done sender: [1:7577789648101251140:2185] self: [1:7577789643806283826:0] peer: [2:7577789646338529911:0] socket: 16 qp: 4498
2025-11-28T14:54:01.975261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789643806283832:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:01.975300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:01.977851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789650633497222:2213] held: no
2025-11-28T14:54:01.985162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789650633497222:2213] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:01.988741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:01.988802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789646338529913:2048] [node 1] ICS09 handshake done sender: [2:7577789650633497222:2213] self: [2:7577789646338529911:0] peer: [1:7577789643806283826:0] socket: 21 qp: 4497
2025-11-28T14:54:01.989287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789646338529913:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:01.989330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:02.373816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:02.374006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789646338529913:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-28T14:54:02.374028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789646338529913:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:02.374096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:02.374127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789646338529913:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:02.372708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789643806283832:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-28T14:54:02.372800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789643806283832:2048] VirtualId# [1:7577789643806283826:0]
2025-11-28T14:54:02.372829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789643806283832:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:02.372882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:02.372938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:02.377724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789652396218441:2187]) peer tries to resume nonexistent session Self# [2:7577789646338529911:0] Peer# [1:7577789643806283826:0]
2025-11-28T14:54:02.377799Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789652396218441:2187] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:02.378398Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789654928464522:2215] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:02.379139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:02.379203Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:02.379224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789646338529913:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:02.379282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789646338529913:2048] VirtualId# [2:7577789646338529911:0]
2025-11-28T14:54:02.379329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:02.385403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789654928464524:2216] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:02.386799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:02.396832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789652396218443:2188] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:02.398177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789652396218445:2189] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:02.402103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789654928464524:2216] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:02.402356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:02.402408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:02.402448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789654928464530:2048]
2025-11-28T14:54:02.402485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789654928464530:2048] [node 1] ICS09 handshake done sender: [2:7577789654928464524:2216] self: [2:7577789654928464528:0] peer: [1:7577789652396218442:0] socket: 23 qp: 4502
2025-11-28T14:54:02.402516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789654928464530:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:02.402547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:02.403742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789652396218443:2188] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:02.403969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:02.404035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:02.404072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789652396218448:2048]
2025-11-28T14:54:02.404106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789652396218448:2048] [node 2] ICS09 handshake done sender: [1:7577789652396218443:2188] self: [1:7577789652396218442:0] peer: [2:7577789654928464528:0] socket: 22 qp: 4501
2025-11-28T14:54:02.404131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789652396218448:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:02.404158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:02.618863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:02.619150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:02.619363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789654928464530:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-28T14:54:02.619385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789654928464530:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:02.619446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:02.619474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789654928464530:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:02.621817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:02.622068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789652396218448:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-28T14:54:02.622093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789652396218448:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:02.622126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:02.622171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789652396218448:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:02.627561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789654928464530:2048] [node 1] ICS08 incoming handshake Self# [1:7577789652396218442:0] Peer# [2:7577789654928464528:0] Counter# 691 LastInputSerial# 19
2025-11-28T14:54:02.627868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789652396218448:2048] [node 2] ICS08 incoming handshake Self# [2:7577789654928464528:0] Peer# [1:7577789652396218442:0] Counter# 19 LastInputSerial# 691
2025-11-28T14:54:02.628170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789652396218450:2190] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:02.628509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:02.628563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789652396218448:2048] [node 2] ICS09 handshake done sender: [1:7577789652396218450:2190] self: [1:7577789652396218442:0] peer: [2:7577789654928464528:0] socket: 24 qp: 4506
2025-11-28T14:54:02.628970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789652396218448:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:02.629007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:02.630768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789654928464534:2220] held: no
2025-11-28T14:54:02.641313Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789654928464534:2220] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:02.641572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:02.641625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789654928464530:2048] [node 1] ICS09 handshake done sender: [2:7577789654928464534:2220] self: [2:7577789654928464528:0] peer: [1:7577789652396218442:0] socket: 25 qp: 4508
2025-11-28T14:54:02.642022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789654928464530:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:02.642065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:03.073195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:03.073330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:03.073353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789654928464530:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:03.073375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789654928464530:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:03.073396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:03.074171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:03.074435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789652396218448:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-28T14:54:03.074458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789652396218448:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:03.074515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:03.074549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789652396218448:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:03.076084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:03.076287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789654928464530:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:03.076306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789654928464530:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:03.081007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789654928464530:2048] [node 1] ICS08 incoming handshake Self# [1:7577789652396218442:0] Peer# [2:7577789654928464528:0] Counter# 4801 LastInputSerial# 120
2025-11-28T14:54:03.082491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789652396218448:2048] [node 2] ICS08 incoming handshake Self# [2:7577789654928464528:0] Peer# [1:7577789652396218442:0] Counter# 120 LastInputSerial# 4801
2025-11-28T14:54:03.082908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789656691185751:2192] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:03.083111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:03.083164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789652396218448:2048] [node 2] ICS09 handshake done sender: [1:7577789656691185751:2192] self: [1:7577789652396218442:0] peer: [2:7577789654928464528:0] socket: 16 qp: 4510
2025-11-28T14:54:03.083454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789652396218448:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:03.083491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:03.087082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789659223431834:2223] held: no
2025-11-28T14:54:03.087148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789659223431834:2223] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:03.087343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:03.087395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789654928464530:2048] [node 1] ICS09 handshake done sender: [2:7577789659223431834:2223] self: [2:7577789654928464528:0] peer: [1:7577789652396218442:0] socket: 20 qp: 4509
2025-11-28T14:54:03.087733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789654928464530:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:03.087770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:54:03.333646Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:54:03.333937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789654928464530:2048] [node 1] ICS07 socket disconnect 20 reason# Debug
2025-11-28T14:54:03.333961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789654928464530:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:03.333999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:03.334029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789654928464530:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:03.335008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:03.335252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789652396218448:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:03.335272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789652396218448:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:03.335317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:03.335346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789652396218448:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:03.340687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789652396218448:2048] [node 2] ICS08 incoming handshake Self# [2:7577789654928464528:0] Peer# [1:7577789652396218442:0] Counter# 174 LastInputSerial# 7029
2025-11-28T14:54:03.342009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789654928464530:2048] [node 1] ICS08 incoming handshake Self# [1:7577789652396218442:0] Peer# [2:7577789654928464528:0] Counter# 7029 LastInputSerial# 174
2025-11-28T14:54:03.343965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789656691185756:2194] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:03.344208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:03.344264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789652396218448:2048] [node 2] ICS09 handshake done sender: [1:7577789656691185756:2194] self: [1:7577789652396218442:0] peer: [2:7577789654928464528:0] socket: 22 qp: 4516
2025-11-28T14:54:03.344596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789652396218448:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:03.344634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:03.346113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789659223431840:2226] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:03.346298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:03.346356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789654928464530:2048] [node 1] ICS09 handshake done sender: [2:7577789659223431840:2226] self: [2:7577789654928464528:0] peer: [1:7577789652396218442:0] socket: 24 qp: 4515
2025-11-28T14:54:03.346709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789654928464530:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:03.346756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:03.473447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789652396218448:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-28T14:54:03.473537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789652396218448:2048] VirtualId# [1:7577789652396218442:0]
2025-11-28T14:54:03.473566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789652396218448:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:03.473611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:03.473653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:03.474548Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:03.474824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:03.474847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789654928464530:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:03.474866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789654928464530:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:03.474893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:03.474950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789654928464530:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:03.474966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789654928464530:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:03.478533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789656691185761:2196]) peer tries to resume nonexistent session Self# [2:7577789654928464528:0] Peer# [1:7577789652396218442:0]
2025-11-28T14:54:03.478588Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789656691185761:2196] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:03.479214Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789659223431843:2228] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:03.479885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:03.479946Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:03.479967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789654928464530:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:03.480019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789654928464530:2048] VirtualId# [2:7577789654928464528:0]
2025-11-28T14:54:03.480101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 28091
TEvUndelivered Cookie# 28092
TEvUndelivered Cookie# 28093
TEvUndelivered Cookie# 28094
2025-11-28T14:54:03.484684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789656691185762:2197] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:03.484825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:03.485815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789659223431846:2229] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:03.488570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789656691185762:2197] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:03.488744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789659223431846:2229] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:03.488956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:03.489005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:03.489045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789659223431849:2048]
2025-11-28T14:54:03.489081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789659223431849:2048] [node 1] ICS09 handshake done sender: [2:7577789659223431846:2229] self: [2:7577789659223431845:0] peer: [1:7577789656691185763:0] socket: 16 qp: 4519
2025-11-28T14:54:03.489107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789659223431849:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:03.489137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:03.489609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:03.489643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:03.489677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789656691185765:2048]
2025-11-28T14:54:03.489711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789656691185765:2048] [node 2] ICS09 handshake done sender: [1:7577789656691185762:2197] self: [1:7577789656691185763:0] peer: [2:7577789659223431845:0] socket: 18 qp: 4518
2025-11-28T14:54:03.489734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789656691185765:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:03.489757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:54:03.623672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:54:03.626929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789656691185765:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-28T14:54:03.626959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789656691185765:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:03.627298Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:03.627497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789659223431849:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:03.627515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789659223431849:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:03.627564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:03.627591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789659223431849:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:03.630904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:03.630963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789656691185765:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:03.632660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789656691185765:2048] [node 2] ICS08 incoming handshake Self# [2:7577789659223431845:0] Peer# [1:7577789656691185763:0] Counter# 6 LastInputSerial# 204
2025-11-28T14:54:03.642316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789659223431849:2048] [node 1] ICS08 incoming handshake Self# [1:7577789656691185763:0] Peer# [2:7577789659223431845:0] Counter# 204 LastInputSerial# 6
2025-11-28T14:54:03.645081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789656691185767:2199] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:03.645351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:03.645409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789656691185765:2048] [node 2] ICS09 handshake done sender: [1:7577789656691185767:2199] self: [1:7577789656691185763:0] peer: [2:7577789659223431845:0] socket: 24 qp: 4523
2025-11-28T14:54:03.645874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789656691185765:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:03.645918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:03.649275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789659223431853:2231] held: no
2025-11-28T14:54:03.650912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789659223431853:2231] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:03.651038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:03.651079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789659223431849:2048] [node 1] ICS09 handshake done sender: [2:7577789659223431853:2231] self: [2:7577789659223431845:0] peer: [1:7577789656691185763:0] socket: 25 qp: 4522
2025-11-28T14:54:03.651429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789659223431849:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:03.651468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:04.171476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789659223431849:2048] [node 1] ICS01 socket: 25 reason# Debug
2025-11-28T14:54:04.171602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789659223431849:2048] VirtualId# [2:7577789659223431845:0]
2025-11-28T14:54:04.171640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789659223431849:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:04.171706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:04.171775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:04.172514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:04.172806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789656691185765:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-28T14:54:04.172839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789656691185765:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:04.172906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:04.172939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789656691185765:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 29120
TEvUndelivered Cookie# 29121
TEvUndelivered Cookie# 29122
2025-11-28T14:54:04.176609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789663518399152:2233]) peer tries to resume nonexistent session Self# [1:7577789656691185763:0] Peer# [2:7577789659223431845:0]
2025-11-28T14:54:04.176676Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789663518399152:2233] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:04.178144Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789660986153068:2201] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:04.178958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:04.179048Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:04.179073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789656691185765:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:04.179128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789656691185765:2048] VirtualId# [1:7577789656691185763:0]
2025-11-28T14:54:04.179177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:04.185941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789660986153070:2202] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:04.186808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789660986153072:2203] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:04.186332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789663518399156:2235] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:04.186426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:04.190953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789663518399156:2235] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:04.191162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789660986153072:2203] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:04.191388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:04.191445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:04.191486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789660986153076:2048]
2025-11-28T14:54:04.191522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789660986153076:2048] [node 2] ICS09 handshake done sender: [1:7577789660986153072:2203] self: [1:7577789660986153071:0] peer: [2:7577789663518399157:0] socket: 16 qp: 4528
2025-11-28T14:54:04.191554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789660986153076:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:04.191584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:04.192730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:04.192789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:04.192823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789663518399159:2048]
2025-11-28T14:54:04.192876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789663518399159:2048] [node 1] ICS09 handshake done sender: [2:7577789663518399156:2235] self: [2:7577789663518399157:0] peer: [1:7577789660986153071:0] socket: 18 qp: 4527
2025-11-28T14:54:04.192903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789663518399159:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:04.192942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:04.195218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:54:04.493380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:04.493580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789663518399159:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:04.493599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789663518399159:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:04.493661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:04.493689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789663518399159:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:04.492968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:54:04.493214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789660986153076:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:54:04.493239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789660986153076:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:04.493307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:04.493337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789660986153076:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:04.499057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789663518399159:2048] [node 1] ICS08 incoming handshake Self# [1:7577789660986153071:0] Peer# [2:7577789663518399157:0] Counter# 2093 LastInputSerial# 51
2025-11-28T14:54:04.499150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789660986153076:2048] [node 2] ICS08 incoming handshake Self# [2:7577789663518399157:0] Peer# [1:7577789660986153071:0] Counter# 51 LastInputSerial# 2093
2025-11-28T14:54:04.499705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789660986153078:2204] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:04.501158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:04.501220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789660986153076:2048] [node 2] ICS09 handshake done sender: [1:7577789660986153078:2204] self: [1:7577789660986153071:0] peer: [2:7577789663518399157:0] socket: 20 qp: 4529
2025-11-28T14:54:04.501506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789660986153076:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:04.501543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:04.505535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789663518399163:2238] held: no
2025-11-28T14:54:04.505604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789663518399163:2238] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:04.505794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:04.505840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789663518399159:2048] [node 1] ICS09 handshake done sender: [2:7577789663518399163:2238] self: [2:7577789663518399157:0] peer: [1:7577789660986153071:0] socket: 25 qp: 4532
2025-11-28T14:54:04.506183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789663518399159:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:04.506235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:04.642173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:04.642302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:04.642564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789663518399159:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-28T14:54:04.642594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789663518399159:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:04.642627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:04.642678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789663518399159:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:04.643026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:04.643266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789660986153076:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:54:04.643288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789660986153076:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:04.643307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:04.643354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789660986153076:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:04.645833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789663518399159:2048] [node 1] ICS08 incoming handshake Self# [1:7577789660986153071:0] Peer# [2:7577789663518399157:0] Counter# 3436 LastInputSerial# 83
2025-11-28T14:54:04.646269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789660986153076:2048] [node 2] ICS08 incoming handshake Self# [2:7577789663518399157:0] Peer# [1:7577789660986153071:0] Counter# 83 LastInputSerial# 3436
2025-11-28T14:54:04.646703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789660986153083:2206] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:04.648214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:04.648272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789660986153076:2048] [node 2] ICS09 handshake done sender: [1:7577789660986153083:2206] self: [1:7577789660986153071:0] peer: [2:7577789663518399157:0] socket: 18 qp: 4534
2025-11-28T14:54:04.648640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789660986153076:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:04.648679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:04.653956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789663518399168:2241] held: no
2025-11-28T14:54:04.654031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789663518399168:2241] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:04.654220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:04.654263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789663518399159:2048] [node 1] ICS09 handshake done sender: [2:7577789663518399168:2241] self: [2:7577789663518399157:0] peer: [1:7577789660986153071:0] socket: 22 qp: 4535
2025-11-28T14:54:04.654632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789663518399159:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:04.654704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:05.196480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789660986153076:2048] [node 2] ICS01 socket: 18 reason# Debug
2025-11-28T14:54:05.196578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789660986153076:2048] VirtualId# [1:7577789660986153071:0]
2025-11-28T14:54:05.196611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789660986153076:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:05.196644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:05.196720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:05.203028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789667813366467:2243] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:05.203135Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:05.203161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789663518399159:2048] [node 1] ICS01 socket: 22 reason# NewSession
2025-11-28T14:54:05.203217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789663518399159:2048] VirtualId# [2:7577789663518399157:0]
2025-11-28T14:54:05.203257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789663518399159:2048] [node 1] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:54:05.203315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:05.203386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:05.203431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:05.208609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789665281120385:2208] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:05.216609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789667813366467:2243] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:05.216786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789665281120385:2208] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:05.217053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:05.217098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:05.217137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789665281120388:2048]
2025-11-28T14:54:05.217176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789665281120388:2048] [node 2] ICS09 handshake done sender: [1:7577789665281120385:2208] self: [1:7577789665281120384:0] peer: [2:7577789667813366468:0] socket: 16 qp: 4537
2025-11-28T14:54:05.217205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789665281120388:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:05.217235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:05.217822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:05.217889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:05.217922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789667813366470:2048]
2025-11-28T14:54:05.217959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789667813366470:2048] [node 1] ICS09 handshake done sender: [2:7577789667813366467:2243] self: [2:7577789667813366468:0] peer: [1:7577789665281120384:0] socket: 18 qp: 4538
2025-11-28T14:54:05.217982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789667813366470:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:05.218012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:05.516695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:05.516927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:05.517795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:05.517218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789665281120388:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:05.517241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789665281120388:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:05.517273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:05.517316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789665281120388:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:05.518162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789667813366470:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:05.518193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789667813366470:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:05.518221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:05.518261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789667813366470:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:05.520245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789667813366470:2048] [node 1] ICS08 incoming handshake Self# [1:7577789665281120384:0] Peer# [2:7577789667813366468:0] Counter# 1802 LastInputSerial# 44
2025-11-28T14:54:05.521578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789665281120390:2209] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:05.521905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:05.521954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789665281120388:2048] [node 2] ICS09 handshake done sender: [1:7577789665281120390:2209] self: [1:7577789665281120384:0] peer: [2:7577789667813366468:0] socket: 22 qp: 4539
2025-11-28T14:54:05.522489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789665281120388:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:05.522535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:05.522645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789665281120388:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:05.522668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789665281120388:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:54:05.522712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:05.522752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789665281120388:2048] [node 2] ICS08 incoming handshake Self# [2:7577789667813366468:0] Peer# [1:7577789665281120384:0] Counter# 44 LastInputSerial# 1802
2025-11-28T14:54:05.522826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:05.523032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789665281120388:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:54:05.523084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789665281120388:2048] VirtualId# [1:7577789665281120384:0]
2025-11-28T14:54:05.523114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:05.524234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789667813366473:2246] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:05.525250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:05.525303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789667813366470:2048] [node 1] ICS09 handshake done sender: [2:7577789667813366473:2246] self: [2:7577789667813366468:0] peer: [1:7577789665281120384:0] socket: 23 qp: 4540
2025-11-28T14:54:05.525953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789667813366470:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:05.526001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:05.526292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:05.526315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789667813366470:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:05.526334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789667813366470:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:05.526356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:05.526428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:05.526597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789667813366470:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:05.526613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789667813366470:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:05.530666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789665281120395:2211]) peer tries to resume nonexistent session Self# [2:7577789667813366468:0] Peer# [1:7577789665281120384:0]
2025-11-28T14:54:05.530721Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789665281120395:2211] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:05.531162Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789667813366477:2248] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:05.531836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:05.531936Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:05.531966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789667813366470:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:05.532042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789667813366470:2048] VirtualId# [2:7577789667813366468:0]
2025-11-28T14:54:05.532163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 31605
TEvUndelivered Cookie# 31606
TEvUndelivered Cookie# 31607
TEvUndelivered Cookie# 31608
TEvUndelivered Cookie# 31609
TEvUndelivered Cookie# 31610
TEvUndelivered Cookie# 31611
2025-11-28T14:54:05.535803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789665281120396:2212] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:05.535922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:05.536990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789667813366481:2249] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:05.540208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789667813366481:2249] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:05.540441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:05.540500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:05.540538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789667813366484:2048]
2025-11-28T14:54:05.540571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789667813366484:2048] [node 1] ICS09 handshake done sender: [2:7577789667813366481:2249] self: [2:7577789667813366480:0] peer: [1:7577789665281120397:0] socket: 16 qp: 4544
2025-11-28T14:54:05.540599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789667813366484:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:05.540029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789665281120396:2212] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:05.540628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:05.541142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:05.541176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:05.541221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789665281120399:2048]
2025-11-28T14:54:05.541255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789665281120399:2048] [node 2] ICS09 handshake done sender: [1:7577789665281120396:2212] self: [1:7577789665281120397:0] peer: [2:7577789667813366480:0] socket: 18 qp: 4545
2025-11-28T14:54:05.541279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789665281120399:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:05.541304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:05.956759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:05.957330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:05.957619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789665281120399:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:05.957641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789665281120399:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:05.957715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:05.957744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789665281120399:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:05.961021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:05.961325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:05.961346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789667813366484:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:05.961367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789667813366484:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:05.961398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:05.961456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789667813366484:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:05.961472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789667813366484:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:05.963376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789667813366484:2048] [node 1] ICS08 incoming handshake Self# [1:7577789665281120397:0] Peer# [2:7577789667813366480:0] Counter# 2839 LastInputSerial# 67
2025-11-28T14:54:05.965950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789665281120401:2214] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:05.966214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:05.966264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789665281120399:2048] [node 2] ICS09 handshake done sender: [1:7577789665281120401:2214] self: [1:7577789665281120397:0] peer: [2:7577789667813366480:0] socket: 22 qp: 4546
2025-11-28T14:54:05.966558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789665281120399:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:05.966601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:05.969717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789665281120399:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:05.969750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789665281120399:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:54:05.969805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:05.969847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789665281120399:2048] [node 2] ICS08 incoming handshake Self# [2:7577789667813366480:0] Peer# [1:7577789665281120397:0] Counter# 67 LastInputSerial# 2839
2025-11-28T14:54:05.970070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:05.970285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789665281120399:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:54:05.970342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789665281120399:2048] VirtualId# [1:7577789665281120397:0]
2025-11-28T14:54:05.970374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:05.971681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789667813366487:2251] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:05.972031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:05.972082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789667813366484:2048] [node 1] ICS09 handshake done sender: [2:7577789667813366487:2251] self: [2:7577789667813366480:0] peer: [1:7577789665281120397:0] socket: 23 qp: 4547
2025-11-28T14:54:05.972845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789667813366484:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:05.972886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:05.973175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:05.973197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789667813366484:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:05.973219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789667813366484:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:05.973238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:05.973310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:05.973471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789667813366484:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:05.973489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789667813366484:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:05.979857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789665281120407:2217]) peer tries to resume nonexistent session Self# [2:7577789667813366480:0] Peer# [1:7577789665281120397:0]
2025-11-28T14:54:05.979910Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789665281120407:2217] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:05.980601Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789667813366492:2253] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:05.981272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:05.981341Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:05.981381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789667813366484:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:05.981449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789667813366484:2048] VirtualId# [2:7577789667813366480:0]
2025-11-28T14:54:05.981560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 32273
TEvUndelivered Cookie# 32274
TEvUndelivered Cookie# 32275
TEvUndelivered Cookie# 32276
TEvUndelivered Cookie# 32277
TEvUndelivered Cookie# 32278
TEvUndelivered Cookie# 32279
2025-11-28T14:54:05.986072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789665281120408:2218] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:05.986189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:05.987141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789667813366495:2254] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:05.991434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789665281120408:2218] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:05.991642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789667813366495:2254] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:05.991891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:05.991934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:05.991972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789667813366498:2048]
2025-11-28T14:54:05.992008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789667813366498:2048] [node 1] ICS09 handshake done sender: [2:7577789667813366495:2254] self: [2:7577789667813366494:0] peer: [1:7577789665281120409:0] socket: 16 qp: 4552
2025-11-28T14:54:05.992035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789667813366498:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:05.992064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:05.992599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:05.992635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:05.992672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789665281120411:2048]
2025-11-28T14:54:05.992708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789665281120411:2048] [node 2] ICS09 handshake done sender: [1:7577789665281120408:2218] self: [1:7577789665281120409:0] peer: [2:7577789667813366494:0] socket: 18 qp: 4551
2025-11-28T14:54:05.992732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789665281120411:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:05.992765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:06.290663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:06.290853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:06.291119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789665281120411:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:06.291140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789665281120411:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:06.291170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:06.291215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789665281120411:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:06.291576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:06.291895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789667813366498:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:06.291914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789667813366498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:06.291945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:06.291986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789667813366498:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:06.295298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789667813366498:2048] [node 1] ICS08 incoming handshake Self# [1:7577789665281120409:0] Peer# [2:7577789667813366494:0] Counter# 1892 LastInputSerial# 52
2025-11-28T14:54:06.296234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789665281120411:2048] [node 2] ICS08 incoming handshake Self# [2:7577789667813366494:0] Peer# [1:7577789665281120409:0] Counter# 52 LastInputSerial# 1892
2025-11-28T14:54:06.296483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789669576087709:2220] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:06.296741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:06.296799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789665281120411:2048] [node 2] ICS09 handshake done sender: [1:7577789669576087709:2220] self: [1:7577789665281120409:0] peer: [2:7577789667813366494:0] socket: 22 qp: 4553
2025-11-28T14:54:06.297245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789665281120411:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:06.297288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:06.298412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789672108333797:2256] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:06.299076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:06.299114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789667813366498:2048] [node 1] ICS09 handshake done sender: [2:7577789672108333797:2256] self: [2:7577789667813366494:0] peer: [1:7577789665281120409:0] socket: 23 qp: 4554
2025-11-28T14:54:06.299373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789667813366498:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:06.299399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:06.299530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:06.875680Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:06.875909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789667813366498:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-28T14:54:06.875931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789667813366498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:06.875996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:06.876030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789667813366498:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:06.875524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:06.875681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:54:06.875711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789665281120411:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:06.875738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789665281120411:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:06.875770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:06.875832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:06.876062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789665281120411:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:54:06.876139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789665281120411:2048] VirtualId# [1:7577789665281120409:0]
2025-11-28T14:54:06.876180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:06.880522Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789672108333801:2258] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:06.881329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:06.881403Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:06.881428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789667813366498:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:06.881486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789667813366498:2048] VirtualId# [2:7577789667813366494:0]
2025-11-28T14:54:06.881542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:06.880235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789669576087714:2222]) peer tries to resume nonexistent session Self# [2:7577789667813366494:0] Peer# [1:7577789665281120409:0]
2025-11-28T14:54:06.880318Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789669576087714:2222] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
2025-11-28T14:54:06.888661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789672108333805:2260] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:06.888879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:06.892166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789672108333805:2260] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:06.889147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789669576087718:2224] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:06.889536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789669576087716:2223] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:06.892325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789669576087716:2223] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:06.892876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:06.892652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:06.892713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:06.892754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789669576087721:2048]
2025-11-28T14:54:06.892803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789669576087721:2048] [node 2] ICS09 handshake done sender: [1:7577789669576087716:2223] self: [1:7577789669576087715:0] peer: [2:7577789672108333807:0] socket: 16 qp: 4558
2025-11-28T14:54:06.892841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789669576087721:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:06.892873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:06.892946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:06.892983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789672108333809:2048]
2025-11-28T14:54:06.893016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789672108333809:2048] [node 1] ICS09 handshake done sender: [2:7577789672108333805:2260] self: [2:7577789672108333807:0] peer: [1:7577789669576087715:0] socket: 18 qp: 4559
2025-11-28T14:54:06.893042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789672108333809:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:06.893067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:07.225622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789669576087721:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-28T14:54:07.225906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789669576087721:2048] VirtualId# [1:7577789669576087715:0]
2025-11-28T14:54:07.225946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789669576087721:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:07.226005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:07.226052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:07.228117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:07.228515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789672108333809:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:07.228539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789672108333809:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:07.228565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:07.228604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789672108333809:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:07.235051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789673871055022:2226]) peer tries to resume nonexistent session Self# [2:7577789672108333807:0] Peer# [1:7577789669576087715:0]
2025-11-28T14:54:07.235102Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789673871055022:2226] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:07.235049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789676403301108:2263] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:07.235146Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:07.235748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789672108333809:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:07.236102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789672108333809:2048] VirtualId# [2:7577789672108333807:0]
2025-11-28T14:54:07.236210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:07.236261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:07.236992Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789676403301107:2262] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 34468
TEvUndelivered Cookie# 34469
2025-11-28T14:54:07.237967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [2:7577789676403301108:2263] held: no
2025-11-28T14:54:07.238203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789673871055020:2225] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:07.240938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789676403301108:2263] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:07.241089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789673871055020:2225] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:07.241298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:07.241336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:07.241374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789673871055024:2048]
2025-11-28T14:54:07.241411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789673871055024:2048] [node 2] ICS09 handshake done sender: [1:7577789673871055020:2225] self: [1:7577789673871055019:0] peer: [2:7577789676403301110:0] socket: 16 qp: 4563
2025-11-28T14:54:07.241436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789673871055024:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:07.241466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:07.241922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:07.241958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:07.241992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789676403301112:2048]
2025-11-28T14:54:07.242023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789676403301112:2048] [node 1] ICS09 handshake done sender: [2:7577789676403301108:2263] self: [2:7577789676403301110:0] peer: [1:7577789673871055019:0] socket: 20 qp: 4562
2025-11-28T14:54:07.242046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789676403301112:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:07.242073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:07.607362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789676403301112:2048] [node 1] ICS01 socket: 20 reason# Debug
2025-11-28T14:54:07.607529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789676403301112:2048] VirtualId# [2:7577789676403301110:0]
2025-11-28T14:54:07.607563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789676403301112:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:07.607617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:07.607686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 35004
TEvUndelivered Cookie# 35005
TEvUndelivered Cookie# 35006
2025-11-28T14:54:07.610128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:07.610462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789673871055024:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:07.610487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789673871055024:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:07.610582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:07.610626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789673871055024:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:07.613671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789673871055027:2228] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:07.613768Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:07.613792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789673871055024:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:07.613856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789673871055024:2048] VirtualId# [1:7577789673871055019:0]
2025-11-28T14:54:07.613906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:07.613660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789676403301117:2266]) peer tries to resume nonexistent session Self# [1:7577789673871055019:0] Peer# [2:7577789676403301110:0]
2025-11-28T14:54:07.613724Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789676403301117:2266] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:07.615927Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789673871055026:2227] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:07.616554Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [1:7577789673871055027:2228] held: yes
2025-11-28T14:54:07.617299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789676403301115:2265] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:07.620129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789673871055027:2228] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:07.620281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789676403301115:2265] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:07.620508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:07.620546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:07.620585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789676403301119:2048]
2025-11-28T14:54:07.620622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789676403301119:2048] [node 1] ICS09 handshake done sender: [2:7577789676403301115:2265] self: [2:7577789676403301114:0] peer: [1:7577789673871055029:0] socket: 20 qp: 4565
2025-11-28T14:54:07.620649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789676403301119:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:07.620679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:07.621482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:07.621534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:07.621570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789673871055031:2048]
2025-11-28T14:54:07.621602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789673871055031:2048] [node 2] ICS09 handshake done sender: [1:7577789673871055027:2228] self: [1:7577789673871055029:0] peer: [2:7577789676403301114:0] socket: 21 qp: 4566
2025-11-28T14:54:07.621629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789673871055031:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:07.621655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:08.016268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:08.016491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:08.016738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789676403301119:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:54:08.016773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789676403301119:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:08.016807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:08.016850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789676403301119:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:08.019712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:08.019989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789673871055031:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-28T14:54:08.020013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789673871055031:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:08.020078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:08.020113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789673871055031:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:08.022297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789673871055031:2048] [node 2] ICS08 incoming handshake Self# [2:7577789676403301114:0] Peer# [1:7577789673871055029:0] Counter# 69 LastInputSerial# 2513
2025-11-28T14:54:08.023931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789676403301119:2048] [node 1] ICS08 incoming handshake Self# [1:7577789673871055029:0] Peer# [2:7577789676403301114:0] Counter# 2513 LastInputSerial# 69
2025-11-28T14:54:08.024356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789678166022329:2230] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:08.026159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:08.026224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789673871055031:2048] [node 2] ICS09 handshake done sender: [1:7577789678166022329:2230] self: [1:7577789673871055029:0] peer: [2:7577789676403301114:0] socket: 24 qp: 4570
2025-11-28T14:54:08.026796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789673871055031:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:08.026857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:08.026746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789680698268419:2268] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:08.026906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:08.026949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789676403301119:2048] [node 1] ICS09 handshake done sender: [2:7577789680698268419:2268] self: [2:7577789676403301114:0] peer: [1:7577789673871055029:0] socket: 25 qp: 4571
2025-11-28T14:54:08.027378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789676403301119:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:08.027418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:08.600503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:08.600690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:08.600907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789676403301119:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-28T14:54:08.600929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789676403301119:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:08.600954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:08.600997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789676403301119:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:08.600805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:08.601133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789673871055031:2048] [node 2] ICS01 socket: 24 reason# EndOfStream
2025-11-28T14:54:08.601199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789673871055031:2048] VirtualId# [1:7577789673871055029:0]
2025-11-28T14:54:08.601218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789673871055031:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:08.601240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:08.601283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:08.606311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789678166022334:2232]) peer tries to resume nonexistent session Self# [2:7577789676403301114:0] Peer# [1:7577789673871055029:0]
2025-11-28T14:54:08.606383Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789678166022334:2232] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:08.606658Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789680698268422:2270] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:08.607631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:08.607709Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:08.607738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789676403301119:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:08.607796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789676403301119:2048] VirtualId# [2:7577789676403301114:0]
2025-11-28T14:54:08.607852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:08.617501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789680698268426:2272] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:08.617667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:08.617960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789678166022338:2234] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:08.618067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789678166022336:2233] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:08.624875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789680698268426:2272] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:08.624965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789678166022336:2233] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:08.625943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:08.625982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:08.626014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789680698268430:2048]
2025-11-28T14:54:08.626051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789680698268430:2048] [node 1] ICS09 handshake done sender: [2:7577789680698268426:2272] self: [2:7577789680698268428:0] peer: [1:7577789678166022335:0] socket: 16 qp: 4573
2025-11-28T14:54:08.626085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789680698268430:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:08.626116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:08.625234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:08.625292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:08.625333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789678166022341:2048]
2025-11-28T14:54:08.625370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789678166022341:2048] [node 2] ICS09 handshake done sender: [1:7577789678166022336:2233] self: [1:7577789678166022335:0] peer: [2:7577789680698268428:0] socket: 18 qp: 4574
2025-11-28T14:54:08.625400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789678166022341:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:08.625429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:54:08.809655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:54:08.809960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789680698268430:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:54:08.809985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789680698268430:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:08.810034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:08.810063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789680698268430:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:08.812045Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:08.812305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789678166022341:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:08.812325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789678166022341:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:08.812491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:08.812532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789678166022341:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:08.814556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789678166022341:2048] [node 2] ICS08 incoming handshake Self# [2:7577789680698268428:0] Peer# [1:7577789678166022335:0] Counter# 20 LastInputSerial# 766
2025-11-28T14:54:08.818079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789678166022343:2235] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:08.818312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:08.818372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789678166022341:2048] [node 2] ICS09 handshake done sender: [1:7577789678166022343:2235] self: [1:7577789678166022335:0] peer: [2:7577789680698268428:0] socket: 24 qp: 4579
2025-11-28T14:54:08.818779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789678166022341:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:08.818813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:08.817721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789680698268430:2048] [node 1] ICS08 incoming handshake Self# [1:7577789678166022335:0] Peer# [2:7577789680698268428:0] Counter# 766 LastInputSerial# 20
2025-11-28T14:54:08.821581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789680698268434:2275] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:08.823149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:08.823208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789680698268430:2048] [node 1] ICS09 handshake done sender: [2:7577789680698268434:2275] self: [2:7577789680698268428:0] peer: [1:7577789678166022335:0] socket: 25 qp: 4580
2025-11-28T14:54:08.823616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789680698268430:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:08.823661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:08.823844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:09.237082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789680698268430:2048] [node 1] ICS01 socket: 25 reason# Debug
2025-11-28T14:54:09.237167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789680698268430:2048] VirtualId# [2:7577789680698268428:0]
2025-11-28T14:54:09.237195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789680698268430:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:09.237251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:09.237334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 37978
TEvUndelivered Cookie# 37979
TEvUndelivered Cookie# 37980
TEvUndelivered Cookie# 37981
TEvUndelivered Cookie# 37982
TEvUndelivered Cookie# 37983
2025-11-28T14:54:09.242651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789684993235733:2277]) peer tries to resume nonexistent session Self# [1:7577789678166022335:0] Peer# [2:7577789680698268428:0]
2025-11-28T14:54:09.242711Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789684993235733:2277] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:09.238297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:09.238562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789678166022341:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-28T14:54:09.238585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789678166022341:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:09.238648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:09.238677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789678166022341:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:09.243298Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789682460989644:2237] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:09.244080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:09.244138Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:09.244160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789678166022341:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:09.244213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789678166022341:2048] VirtualId# [1:7577789678166022335:0]
2025-11-28T14:54:09.244257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:09.253918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789684993235737:2279] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:09.254007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:09.253600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789682460989646:2238] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:09.254374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789682460989648:2239] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:09.261238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789684993235737:2279] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:09.264294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:09.264348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:09.261478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789682460989648:2239] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:09.261727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:09.261781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:09.261822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789682460989652:2048]
2025-11-28T14:54:09.261863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789682460989652:2048] [node 2] ICS09 handshake done sender: [1:7577789682460989648:2239] self: [1:7577789682460989647:0] peer: [2:7577789684993235738:0] socket: 16 qp: 4585
2025-11-28T14:54:09.261895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789682460989652:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:09.261927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:09.264522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789684993235740:2048]
2025-11-28T14:54:09.264561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789684993235740:2048] [node 1] ICS09 handshake done sender: [2:7577789684993235737:2279] self: [2:7577789684993235738:0] peer: [1:7577789682460989647:0] socket: 18 qp: 4584
2025-11-28T14:54:09.264588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789684993235740:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:09.264620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:09.264782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:09.546619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789682460989652:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-28T14:54:09.546778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789682460989652:2048] VirtualId# [1:7577789682460989647:0]
2025-11-28T14:54:09.546813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789682460989652:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:09.546866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:09.546927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:09.547585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:09.547801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789684993235740:2048] [node 1] ICS01 socket: 18 reason# EndOfStream
2025-11-28T14:54:09.547933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789684993235740:2048] VirtualId# [2:7577789684993235738:0]
2025-11-28T14:54:09.547966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789684993235740:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:09.548053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:09.548092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:09.552314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789684993235742:2281] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:09.552494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:09.552859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789682460989655:2240] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:09.555550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789684993235742:2281] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:09.555631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789682460989655:2240] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:09.555796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:09.555842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:09.555894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789684993235745:2048]
2025-11-28T14:54:09.555944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789684993235745:2048] [node 1] ICS09 handshake done sender: [2:7577789684993235742:2281] self: [2:7577789684993235743:0] peer: [1:7577789682460989654:0] socket: 18 qp: 4587
2025-11-28T14:54:09.555999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789684993235745:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:09.556033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:09.556042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:09.556079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:09.556117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789682460989658:2048]
2025-11-28T14:54:09.556152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789682460989658:2048] [node 2] ICS09 handshake done sender: [1:7577789682460989655:2240] self: [1:7577789682460989654:0] peer: [2:7577789684993235743:0] socket: 16 qp: 4586
2025-11-28T14:54:09.556175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789682460989658:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:09.556206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:09.780656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789682460989658:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-28T14:54:09.781005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789682460989658:2048] VirtualId# [1:7577789682460989654:0]
2025-11-28T14:54:09.781050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789682460989658:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:09.781109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:09.781150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:09.782243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:09.782455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789684993235745:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:09.782482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789684993235745:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:09.782506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:09.782555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789684993235745:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:09.787516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789682460989660:2241]) peer tries to resume nonexistent session Self# [2:7577789684993235743:0] Peer# [1:7577789682460989654:0]
2025-11-28T14:54:09.787575Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789682460989660:2241] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:09.788648Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789684993235747:2283] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:09.789554Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:09.789626Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:09.789657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789684993235745:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:09.789724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789684993235745:2048] VirtualId# [2:7577789684993235743:0]
2025-11-28T14:54:09.789775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:09.794061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789682460989661:2242] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:09.794192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:09.794545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789684993235750:2284] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:09.804679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789682460989661:2242] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:09.804955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789684993235750:2284] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:09.805232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:09.805280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:09.805335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789684993235753:2048]
2025-11-28T14:54:09.805376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789684993235753:2048] [node 1] ICS09 handshake done sender: [2:7577789684993235750:2284] self: [2:7577789684993235749:0] peer: [1:7577789682460989662:0] socket: 16 qp: 4590
2025-11-28T14:54:09.805418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789684993235753:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:09.805455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:09.805846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:09.805891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:09.805931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789682460989664:2048]
2025-11-28T14:54:09.805966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789682460989664:2048] [node 2] ICS09 handshake done sender: [1:7577789682460989661:2242] self: [1:7577789682460989662:0] peer: [2:7577789684993235749:0] socket: 18 qp: 4589
2025-11-28T14:54:09.805996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789682460989664:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:09.806027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:09.898051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:09.898127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:54:09.898151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789682460989664:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:09.898173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789682460989664:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:09.898199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:09.898972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:09.899187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789684993235753:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-28T14:54:09.899491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:09.899776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789682460989664:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:09.899343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789684993235753:2048] VirtualId# [2:7577789684993235749:0]
2025-11-28T14:54:09.899799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789682460989664:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:09.899371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789684993235753:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:09.899438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:09.899479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:09.912486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789684993235755:2285]) peer tries to resume nonexistent session Self# [1:7577789682460989662:0] Peer# [2:7577789684993235749:0]
2025-11-28T14:54:09.912565Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789684993235755:2285] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:09.913320Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789682460989666:2244] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:09.914706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:09.914781Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:09.914803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789682460989664:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:09.914936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789682460989664:2048] VirtualId# [1:7577789682460989662:0]
2025-11-28T14:54:09.915017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:09.919269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789684993235756:2286] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:09.919422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:09.919788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789682460989669:2245] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:09.932645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789684993235756:2286] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:09.932929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:09.932971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:09.933016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789684993235759:2048]
2025-11-28T14:54:09.933053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789684993235759:2048] [node 1] ICS09 handshake done sender: [2:7577789684993235756:2286] self: [2:7577789684993235757:0] peer: [1:7577789682460989668:0] socket: 18 qp: 4592
2025-11-28T14:54:09.933083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789684993235759:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:09.933115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:09.932954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789682460989669:2245] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:09.935082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:09.935126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:09.935181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789682460989672:2048]
2025-11-28T14:54:09.935221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789682460989672:2048] [node 2] ICS09 handshake done sender: [1:7577789682460989669:2245] self: [1:7577789682460989668:0] peer: [2:7577789684993235757:0] socket: 16 qp: 4593
2025-11-28T14:54:09.935249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789682460989672:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:09.935280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:10.321964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:10.322253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789682460989672:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:10.322276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789682460989672:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:10.322349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:10.322383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789682460989672:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 39374
TEvUndelivered Cookie# 39375
TEvUndelivered Cookie# 39376
TEvUndelivered Cookie# 39377
TEvUndelivered Cookie# 39378
TEvUndelivered Cookie# 39379
TEvUndelivered Cookie# 39380
TEvUndelivered Cookie# 39381
2025-11-28T14:54:10.320931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789684993235759:2048] [node 1] ICS01 socket: 18 reason# Debug
TEvUndelivered Cookie# 39382
2025-11-28T14:54:10.321035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789684993235759:2048] VirtualId# [2:7577789684993235757:0]
2025-11-28T14:54:10.321064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789684993235759:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:10.321121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:10.321228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:10.326903Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789686755956970:2246] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:10.326333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789689288203057:2288]) peer tries to resume nonexistent session Self# [1:7577789682460989668:0] Peer# [2:7577789684993235757:0]
2025-11-28T14:54:10.326390Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789689288203057:2288] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:10.327722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:10.327783Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:10.327808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789682460989672:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:10.327935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789682460989672:2048] VirtualId# [1:7577789682460989668:0]
2025-11-28T14:54:10.327979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:10.335655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789686755956972:2247] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:10.336133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789689288203061:2290] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:10.336227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:10.336555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789686755956974:2248] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:10.345878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789686755956974:2248] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:10.346155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:10.346215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:10.346254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789686755956978:2048]
2025-11-28T14:54:10.346292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789686755956978:2048] [node 2] ICS09 handshake done sender: [1:7577789686755956974:2248] self: [1:7577789686755956973:0] peer: [2:7577789689288203062:0] socket: 16 qp: 4598
2025-11-28T14:54:10.346321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789686755956978:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:10.346350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:10.345708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789689288203061:2290] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:10.349282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:10.349332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:10.349375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789689288203064:2048]
2025-11-28T14:54:10.349410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789689288203064:2048] [node 1] ICS09 handshake done sender: [2:7577789689288203061:2290] self: [2:7577789689288203062:0] peer: [1:7577789686755956973:0] socket: 18 qp: 4597
2025-11-28T14:54:10.349439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789689288203064:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:10.349468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:10.349621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:10.818098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:10.818324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789689288203064:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:10.818353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789689288203064:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:10.818417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:10.818452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789689288203064:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:10.817986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:10.818059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:54:10.818082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789686755956978:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:10.818104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789686755956978:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:10.818132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:10.819248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:10.819578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789686755956978:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:10.819602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789686755956978:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:10.821427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789686755956978:2048] [node 2] ICS08 incoming handshake Self# [2:7577789689288203062:0] Peer# [1:7577789686755956973:0] Counter# 91 LastInputSerial# 3688
2025-11-28T14:54:10.823010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789689288203064:2048] [node 1] ICS08 incoming handshake Self# [1:7577789686755956973:0] Peer# [2:7577789689288203062:0] Counter# 3688 LastInputSerial# 91
2025-11-28T14:54:10.823462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789686755956980:2249] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:10.823955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:10.824019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789686755956978:2048] [node 2] ICS09 handshake done sender: [1:7577789686755956980:2249] self: [1:7577789686755956973:0] peer: [2:7577789689288203062:0] socket: 24 qp: 4602
2025-11-28T14:54:10.824337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789686755956978:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:10.824400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:10.826472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789689288203068:2293] held: no
2025-11-28T14:54:10.826542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789689288203068:2293] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:10.826699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:10.826743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789689288203064:2048] [node 1] ICS09 handshake done sender: [2:7577789689288203068:2293] self: [2:7577789689288203062:0] peer: [1:7577789686755956973:0] socket: 25 qp: 4601
2025-11-28T14:54:10.827197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789689288203064:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:10.827241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:11.167385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:11.167519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:11.167548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789689288203064:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:11.167569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789689288203064:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:11.167591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:11.167843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:11.168109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789686755956978:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-28T14:54:11.168136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789686755956978:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:11.168195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:11.168232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789686755956978:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:11.170711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:11.170941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789689288203064:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:11.170959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789689288203064:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:11.172786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789689288203064:2048] [node 1] ICS08 incoming handshake Self# [1:7577789686755956973:0] Peer# [2:7577789689288203062:0] Counter# 6521 LastInputSerial# 167
2025-11-28T14:54:11.177458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789691050924281:2251] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:11.177760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:11.177810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789686755956978:2048] [node 2] ICS09 handshake done sender: [1:7577789691050924281:2251] self: [1:7577789686755956973:0] peer: [2:7577789689288203062:0] socket: 16 qp: 4603
2025-11-28T14:54:11.178251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789686755956978:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:11.178292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:11.178455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789686755956978:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:11.178478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789686755956978:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:54:11.178510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:11.178548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789686755956978:2048] [node 2] ICS08 incoming handshake Self# [2:7577789689288203062:0] Peer# [1:7577789686755956973:0] Counter# 167 LastInputSerial# 6521
2025-11-28T14:54:11.178619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:11.178812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789686755956978:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:11.180080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789693583170368:2296] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:11.180436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:11.180502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789689288203064:2048] [node 1] ICS09 handshake done sender: [2:7577789693583170368:2296] self: [2:7577789689288203062:0] peer: [1:7577789686755956973:0] socket: 18 qp: 4604
2025-11-28T14:54:11.180930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789689288203064:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:11.180965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:11.181231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:11.181252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789689288203064:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:11.181273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789689288203064:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:11.181292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:11.181410Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:11.181576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789689288203064:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:11.181594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789689288203064:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:11.188664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789686755956978:2048] [node 2] ICS08 incoming handshake Self# [2:7577789689288203062:0] Peer# [1:7577789686755956973:0] Counter# 168 LastInputSerial# 6521
2025-11-28T14:54:11.191003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789693583170373:2298] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:11.191235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:11.191282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789689288203064:2048] [node 1] ICS09 handshake done sender: [2:7577789693583170373:2298] self: [2:7577789689288203062:0] peer: [1:7577789686755956973:0] socket: 16 qp: 4608
2025-11-28T14:54:11.191679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789689288203064:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:11.191717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:11.193078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789691050924287:2254] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:11.193387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:11.193437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789686755956978:2048] [node 2] ICS09 handshake done sender: [1:7577789691050924287:2254] self: [1:7577789686755956973:0] peer: [2:7577789689288203062:0] socket: 18 qp: 4607
2025-11-28T14:54:11.193935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789686755956978:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:11.193991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:11.706239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:11.706489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789686755956978:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:11.706509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789686755956978:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:11.706568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:11.706597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789686755956978:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:11.705356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789689288203064:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-28T14:54:11.705439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789689288203064:2048] VirtualId# [2:7577789689288203062:0]
2025-11-28T14:54:11.705467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789689288203064:2048] [node 1] ICS25 shutdown socket, reason# Debug
TEvNodeDisconnected
2025-11-28T14:54:11.705508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:11.705549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:11.714013Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789691050924290:2256] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:11.714872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:11.714944Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:11.714969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789686755956978:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:11.715023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789686755956978:2048] VirtualId# [1:7577789686755956973:0]
2025-11-28T14:54:11.715068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:11.713355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789693583170377:2299]) peer tries to resume nonexistent session Self# [1:7577789686755956973:0] Peer# [2:7577789689288203062:0]
2025-11-28T14:54:11.713416Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789693583170377:2299] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:11.719956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789693583170378:2300] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:11.720095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:11.720430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789691050924293:2257] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:11.725516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789691050924293:2257] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:11.725739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:11.725765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:11.725793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789691050924296:2048]
2025-11-28T14:54:11.725820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789691050924296:2048] [node 2] ICS09 handshake done sender: [1:7577789691050924293:2257] self: [1:7577789691050924292:0] peer: [2:7577789693583170379:0] socket: 16 qp: 4611
2025-11-28T14:54:11.725837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789691050924296:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:11.725858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:11.724615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789693583170378:2300] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:11.724845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:11.724886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:11.724928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789693583170381:2048]
2025-11-28T14:54:11.724968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789693583170381:2048] [node 1] ICS09 handshake done sender: [2:7577789693583170378:2300] self: [2:7577789693583170379:0] peer: [1:7577789691050924292:0] socket: 18 qp: 4610
2025-11-28T14:54:11.724998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789693583170381:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:11.725030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:54:11.814658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:54:11.814911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789691050924296:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:54:11.814940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789691050924296:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:11.814989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:11.815039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789691050924296:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:11.815144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:11.815187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789693583170381:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:11.815209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789693583170381:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:11.815236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:11.815597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:11.815793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789693583170381:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:11.815809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789693583170381:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:11.817614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789693583170381:2048] [node 1] ICS08 incoming handshake Self# [1:7577789691050924292:0] Peer# [2:7577789693583170379:0] Counter# 36 LastInputSerial# 1
2025-11-28T14:54:11.821650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789691050924298:2258] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:11.821995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789691050924296:2048] [node 2] ICS08 incoming handshake Self# [2:7577789693583170379:0] Peer# [1:7577789691050924292:0] Counter# 1 LastInputSerial# 36
2025-11-28T14:54:11.822033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:11.822081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789691050924296:2048] [node 2] ICS09 handshake done sender: [1:7577789691050924298:2258] self: [1:7577789691050924292:0] peer: [2:7577789693583170379:0] socket: 22 qp: 4612
2025-11-28T14:54:11.822607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789691050924296:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:11.822649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:11.825483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789693583170384:2303] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:11.825905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789693583170384:2303] held: no
2025-11-28T14:54:11.825934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:11.825988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789693583170381:2048] [node 1] ICS09 handshake done sender: [2:7577789693583170384:2303] self: [2:7577789693583170379:0] peer: [1:7577789691050924292:0] socket: 23 qp: 4613
2025-11-28T14:54:11.826559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789693583170381:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:11.826614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:12.310423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:12.310739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789691050924296:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-28T14:54:12.310767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789691050924296:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:12.310844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:12.310888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789691050924296:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:12.309562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789693583170381:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-11-28T14:54:12.309658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789693583170381:2048] VirtualId# [2:7577789693583170379:0]
TEvNodeDisconnected
2025-11-28T14:54:12.309689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789693583170381:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:12.309742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:12.309778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:12.315390Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789695345891599:2260] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:12.316841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:12.316916Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:12.316942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789691050924296:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:12.317004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789691050924296:2048] VirtualId# [1:7577789691050924292:0]
2025-11-28T14:54:12.317060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:12.314565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789697878137684:2305]) peer tries to resume nonexistent session Self# [1:7577789691050924292:0] Peer# [2:7577789693583170379:0]
2025-11-28T14:54:12.314760Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789697878137684:2305] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:12.319594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789695345891601:2261] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:12.323632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789697878137688:2307] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:12.323759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:12.324128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789695345891603:2262] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:12.326903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789697878137688:2307] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:12.327006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789695345891603:2262] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:12.327108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:12.327157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:12.327195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789697878137691:2048]
2025-11-28T14:54:12.327239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789697878137691:2048] [node 1] ICS09 handshake done sender: [2:7577789697878137688:2307] self: [2:7577789697878137689:0] peer: [1:7577789695345891602:0] socket: 18 qp: 4620
2025-11-28T14:54:12.327271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789697878137691:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:12.327308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:12.327526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:12.327578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:12.327616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789695345891607:2048]
2025-11-28T14:54:12.327651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789695345891607:2048] [node 2] ICS09 handshake done sender: [1:7577789695345891603:2262] self: [1:7577789695345891602:0] peer: [2:7577789697878137689:0] socket: 16 qp: 4619
2025-11-28T14:54:12.327677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789695345891607:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:12.327704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:12.779462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789695345891607:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-28T14:54:12.779674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789695345891607:2048] VirtualId# [1:7577789695345891602:0]
2025-11-28T14:54:12.779704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789695345891607:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:12.779738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:12.779809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:12.783335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789697878137693:2309] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:12.783419Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:12.783437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789697878137691:2048] [node 1] ICS01 socket: 18 reason# NewSession
2025-11-28T14:54:12.783503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789697878137691:2048] VirtualId# [2:7577789697878137689:0]
2025-11-28T14:54:12.783532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789697878137691:2048] [node 1] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:54:12.783589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:12.783632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:12.783665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:12.785157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789695345891610:2263] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:12.791404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789695345891610:2263] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:12.791677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:12.791720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:12.791765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789695345891613:2048]
2025-11-28T14:54:12.791806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789695345891613:2048] [node 2] ICS09 handshake done sender: [1:7577789695345891610:2263] self: [1:7577789695345891609:0] peer: [2:7577789697878137694:0] socket: 16 qp: 4621
2025-11-28T14:54:12.791838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789695345891613:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:12.791878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:12.791229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789697878137693:2309] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:12.792558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:12.792597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:12.792631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789697878137696:2048]
2025-11-28T14:54:12.792663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789697878137696:2048] [node 1] ICS09 handshake done sender: [2:7577789697878137693:2309] self: [2:7577789697878137694:0] peer: [1:7577789695345891609:0] socket: 20 qp: 4622
2025-11-28T14:54:12.792687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789697878137696:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:12.792713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:13.251427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:13.252387Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:13.252729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:54:13.252754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789695345891613:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:13.252777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789695345891613:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:13.252803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:13.252860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789695345891613:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:54:13.252954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789695345891613:2048] VirtualId# [1:7577789695345891609:0]
2025-11-28T14:54:13.252981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:13.253090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:13.253316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789697878137696:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:54:13.253362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789697878137696:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:13.253449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:13.253483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789697878137696:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:13.261013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789699640858914:2265]) peer tries to resume nonexistent session Self# [2:7577789697878137694:0] Peer# [1:7577789695345891609:0]
2025-11-28T14:54:13.261071Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789699640858914:2265] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:13.261070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789702173104996:2312] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:13.261190Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:13.261220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789697878137696:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:13.261654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789697878137696:2048] VirtualId# [2:7577789697878137694:0]
2025-11-28T14:54:13.261716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:13.261750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:13.262581Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789702173104994:2311] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
TEvNodeDisconnected
2025-11-28T14:54:13.263776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [2:7577789702173104996:2312] held: no
2025-11-28T14:54:13.264123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789699640858912:2264] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:13.269440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789702173104996:2312] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:13.269642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789699640858912:2264] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:13.269897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:13.269941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:13.269984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789699640858916:2048]
2025-11-28T14:54:13.270028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789699640858916:2048] [node 2] ICS09 handshake done sender: [1:7577789699640858912:2264] self: [1:7577789699640858911:0] peer: [2:7577789702173104997:0] socket: 18 qp: 4625
2025-11-28T14:54:13.270057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789699640858916:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:13.270089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:13.270742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:13.270776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:13.270812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789702173104999:2048]
2025-11-28T14:54:13.270846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789702173104999:2048] [node 1] ICS09 handshake done sender: [2:7577789702173104996:2312] self: [2:7577789702173104997:0] peer: [1:7577789699640858911:0] socket: 22 qp: 4624
2025-11-28T14:54:13.270869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789702173104999:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:13.270893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:13.367948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:13.369687Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:13.369948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789699640858916:2048] [node 2] ICS01 socket: 18 reason# EndOfStream
2025-11-28T14:54:13.369899Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:13.370117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789702173104999:2048] [node 1] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-28T14:54:13.370177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789699640858916:2048] VirtualId# [1:7577789699640858911:0]
2025-11-28T14:54:13.370211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789699640858916:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:13.370273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:13.370309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:13.370151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789702173104999:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:13.370188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:13.370234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789702173104999:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:13.375112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789699640858918:2266]) peer tries to resume nonexistent session Self# [2:7577789702173104997:0] Peer# [1:7577789699640858911:0]
2025-11-28T14:54:13.375192Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789699640858918:2266] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:13.375507Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789702173105001:2314] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:13.376208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:13.376282Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:13.376306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789702173104999:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:13.376476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789702173104999:2048] VirtualId# [2:7577789702173104997:0]
2025-11-28T14:54:13.376535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:13.383536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789699640858922:2268] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:13.383909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789702173105006:2316] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:13.384004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:13.384345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789699640858920:2267] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:13.395533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789702173105006:2316] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:13.396621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789699640858920:2267] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:13.396685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:13.396742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:13.396788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789702173105009:2048]
2025-11-28T14:54:13.396838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789702173105009:2048] [node 1] ICS09 handshake done sender: [2:7577789702173105006:2316] self: [2:7577789702173105007:0] peer: [1:7577789699640858919:0] socket: 21 qp: 4629
2025-11-28T14:54:13.396881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789702173105009:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:13.396917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:13.396906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:13.396960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:13.397004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789699640858925:2048]
2025-11-28T14:54:13.397040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789699640858925:2048] [node 2] ICS09 handshake done sender: [1:7577789699640858920:2267] self: [1:7577789699640858919:0] peer: [2:7577789702173105007:0] socket: 16 qp: 4630
2025-11-28T14:54:13.397065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789699640858925:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:13.397092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:13.818737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789702173105009:2048] [node 1] ICS01 socket: 21 reason# Debug
2025-11-28T14:54:13.819035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789702173105009:2048] VirtualId# [2:7577789702173105007:0]
2025-11-28T14:54:13.819072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789702173105009:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:13.819137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:13.819226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 45217
TEvUndelivered Cookie# 45218
TEvUndelivered Cookie# 45219
TEvUndelivered Cookie# 45220
TEvUndelivered Cookie# 45221
TEvUndelivered Cookie# 45222
2025-11-28T14:54:13.820188Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:13.820534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789699640858925:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:13.820559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789699640858925:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:13.820625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:13.820655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789699640858925:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:13.825405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789702173105011:2318]) peer tries to resume nonexistent session Self# [1:7577789699640858919:0] Peer# [2:7577789702173105007:0]
2025-11-28T14:54:13.825481Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789702173105011:2318] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:13.827490Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789699640858927:2269] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:13.828385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:13.828473Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:13.828499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789699640858925:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:13.828644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789699640858925:2048] VirtualId# [1:7577789699640858919:0]
2025-11-28T14:54:13.828698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:13.831980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789699640858929:2270] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:13.834632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789702173105015:2320] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:13.834734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:13.835102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789699640858931:2271] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:13.844653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789702173105015:2320] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:13.844940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789699640858931:2271] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:13.845194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:13.845259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:13.845321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789699640858935:2048]
2025-11-28T14:54:13.845364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789699640858935:2048] [node 2] ICS09 handshake done sender: [1:7577789699640858931:2271] self: [1:7577789699640858930:0] peer: [2:7577789702173105016:0] socket: 16 qp: 4635
2025-11-28T14:54:13.845396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789699640858935:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:13.845431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:13.848798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:13.848858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:13.848902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789702173105018:2048]
2025-11-28T14:54:13.848940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789702173105018:2048] [node 1] ICS09 handshake done sender: [2:7577789702173105015:2320] self: [2:7577789702173105016:0] peer: [1:7577789699640858930:0] socket: 18 qp: 4634
2025-11-28T14:54:13.848973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789702173105018:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:13.849004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:13.849220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:14.147996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789699640858935:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-28T14:54:14.148242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:14.148498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789702173105018:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:14.148519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789702173105018:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:14.148575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:14.148604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789702173105018:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:14.148097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789699640858935:2048] VirtualId# [1:7577789699640858930:0]
2025-11-28T14:54:14.148131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789699640858935:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:14.148180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:14.148215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:14.154055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789703935826233:2272]) peer tries to resume nonexistent session Self# [2:7577789702173105016:0] Peer# [1:7577789699640858930:0]
2025-11-28T14:54:14.154123Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789703935826233:2272] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:14.154278Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789706468072316:2322] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 45698
TEvUndelivered Cookie# 45699
TEvUndelivered Cookie# 45700
TEvUndelivered Cookie# 45701
TEvUndelivered Cookie# 45702
TEvUndelivered Cookie# 45703
TEvUndelivered Cookie# 45704
TEvUndelivered Cookie# 45705
TEvUndelivered Cookie# 45706
TEvUndelivered Cookie# 45707
2025-11-28T14:54:14.155655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:14.155727Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:14.155761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789702173105018:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:14.155845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789702173105018:2048] VirtualId# [2:7577789702173105016:0]
2025-11-28T14:54:14.155987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:14.161769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789706468072319:2323] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:14.161164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789703935826234:2273] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:14.161311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:14.166998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789703935826234:2273] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:14.167230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789706468072319:2323] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:14.167457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:14.167502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:14.167542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789706468072322:2048]
2025-11-28T14:54:14.167587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789706468072322:2048] [node 1] ICS09 handshake done sender: [2:7577789706468072319:2323] self: [2:7577789706468072318:0] peer: [1:7577789703935826235:0] socket: 16 qp: 4638
2025-11-28T14:54:14.167618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789706468072322:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:14.167648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:14.168060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:14.168108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:14.168145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789703935826237:2048]
2025-11-28T14:54:14.168180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789703935826237:2048] [node 2] ICS09 handshake done sender: [1:7577789703935826234:2273] self: [1:7577789703935826235:0] peer: [2:7577789706468072318:0] socket: 18 qp: 4637
2025-11-28T14:54:14.168204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789703935826237:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:14.168243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:14.293248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:14.293394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:54:14.293427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789703935826237:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:14.293450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789703935826237:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:14.293477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:14.294503Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:14.294717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789703935826237:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:14.294740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789703935826237:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:14.293671Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:14.293906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789706468072322:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:14.293927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789706468072322:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:14.293989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:14.294018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789706468072322:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:14.303237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789703935826237:2048] [node 2] ICS08 incoming handshake Self# [2:7577789706468072318:0] Peer# [1:7577789703935826235:0] Counter# 6 LastInputSerial# 288
2025-11-28T14:54:14.305429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789706468072322:2048] [node 1] ICS08 incoming handshake Self# [1:7577789703935826235:0] Peer# [2:7577789706468072318:0] Counter# 288 LastInputSerial# 6
2025-11-28T14:54:14.307795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789703935826239:2275] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:14.308079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:14.308132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789703935826237:2048] [node 2] ICS09 handshake done sender: [1:7577789703935826239:2275] self: [1:7577789703935826235:0] peer: [2:7577789706468072318:0] socket: 24 qp: 4642
2025-11-28T14:54:14.308557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789703935826237:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:14.308588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:14.311365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789706468072326:2325] held: no
2025-11-28T14:54:14.311418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789706468072326:2325] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:14.311554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:14.311585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789706468072322:2048] [node 1] ICS09 handshake done sender: [2:7577789706468072326:2325] self: [2:7577789706468072318:0] peer: [1:7577789703935826235:0] socket: 25 qp: 4641
2025-11-28T14:54:14.312204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789706468072322:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:14.312232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:14.728601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:14.730276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:14.730303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789706468072322:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:14.730326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789706468072322:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:14.730349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:14.730771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:14.731054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789703935826237:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-28T14:54:14.731079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789703935826237:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:14.731139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:14.731172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789703935826237:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:14.733085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:14.733326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789706468072322:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:14.733345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789706468072322:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:14.735209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789706468072322:2048] [node 1] ICS08 incoming handshake Self# [1:7577789703935826235:0] Peer# [2:7577789706468072318:0] Counter# 3650 LastInputSerial# 93
2025-11-28T14:54:14.737772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789703935826244:2277] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:14.738034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:14.738084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789703935826237:2048] [node 2] ICS09 handshake done sender: [1:7577789703935826244:2277] self: [1:7577789703935826235:0] peer: [2:7577789706468072318:0] socket: 16 qp: 4643
2025-11-28T14:54:14.738503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789703935826237:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:14.738543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:14.742016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789703935826237:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:14.742051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789703935826237:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:54:14.742107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:14.742149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789703935826237:2048] [node 2] ICS08 incoming handshake Self# [2:7577789706468072318:0] Peer# [1:7577789703935826235:0] Counter# 93 LastInputSerial# 3650
2025-11-28T14:54:14.742468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:14.742698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789703935826237:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:54:14.742754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789703935826237:2048] VirtualId# [1:7577789703935826235:0]
2025-11-28T14:54:14.742789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:14.744219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789706468072330:2328] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:14.744731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:14.744782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789706468072322:2048] [node 1] ICS09 handshake done sender: [2:7577789706468072330:2328] self: [2:7577789706468072318:0] peer: [1:7577789703935826235:0] socket: 18 qp: 4644
2025-11-28T14:54:14.745365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789706468072322:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:14.745406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:14.745693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:14.745717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789706468072322:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:14.745740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789706468072322:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:14.745760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:14.745824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:14.746015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789706468072322:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:14.746040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789706468072322:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:14.752627Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789703935826250:2280]) peer tries to resume nonexistent session Self# [2:7577789706468072318:0] Peer# [1:7577789703935826235:0]
2025-11-28T14:54:14.752688Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789703935826250:2280] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:14.753393Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789706468072335:2330] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:14.754265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:14.754330Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:14.754354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789706468072322:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:14.754410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789706468072322:2048] VirtualId# [2:7577789706468072318:0]
2025-11-28T14:54:14.754521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 46627
TEvUndelivered Cookie# 46628
TEvUndelivered Cookie# 46629
TEvUndelivered Cookie# 46630
TEvUndelivered Cookie# 46631
TEvUndelivered Cookie# 46632
TEvUndelivered Cookie# 46633
2025-11-28T14:54:14.759806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789703935826251:2281] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:14.759936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:14.761282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789706468072338:2331] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:14.768921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789703935826251:2281] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:14.769172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789706468072338:2331] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:14.769467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:14.769511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:14.769559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789706468072341:2048]
2025-11-28T14:54:14.769601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789706468072341:2048] [node 1] ICS09 handshake done sender: [2:7577789706468072338:2331] self: [2:7577789706468072337:0] peer: [1:7577789703935826252:0] socket: 16 qp: 4649
2025-11-28T14:54:14.769631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789706468072341:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:14.769662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:14.770157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:14.770191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:14.770227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789703935826254:2048]
2025-11-28T14:54:14.770262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789703935826254:2048] [node 2] ICS09 handshake done sender: [1:7577789703935826251:2281] self: [1:7577789703935826252:0] peer: [2:7577789706468072337:0] socket: 18 qp: 4648
2025-11-28T14:54:14.770284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789703935826254:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:14.770309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:15.294617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:15.294997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789706468072341:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:15.295019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789706468072341:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:15.295044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:15.295087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789706468072341:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:15.293418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789703935826254:2048] [node 2] ICS01 socket: 18 reason# Debug
2025-11-28T14:54:15.293524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789703935826254:2048] VirtualId# [1:7577789703935826252:0]
2025-11-28T14:54:15.293557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789703935826254:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:15.293615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:15.293654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:15.300935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789708230793552:2283]) peer tries to resume nonexistent session Self# [2:7577789706468072337:0] Peer# [1:7577789703935826252:0]
2025-11-28T14:54:15.300996Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789708230793552:2283] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:15.301763Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789710763039639:2332] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:15.302904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:15.302977Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:15.303001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789706468072341:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:15.303197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789706468072341:2048] VirtualId# [2:7577789706468072337:0]
2025-11-28T14:54:15.303335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 47299
TEvUndelivered Cookie# 47300
TEvUndelivered Cookie# 47301
TEvUndelivered Cookie# 47302
TEvUndelivered Cookie# 47303
TEvUndelivered Cookie# 47304
TEvUndelivered Cookie# 47305
TEvUndelivered Cookie# 47306
TEvUndelivered Cookie# 47307
2025-11-28T14:54:15.322304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789708230793553:2284] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:15.322472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:15.325732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789710763039642:2333] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:15.332687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789708230793553:2284] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:15.332945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789710763039642:2333] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:15.333269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:15.333318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:15.333366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789710763039645:2048]
2025-11-28T14:54:15.333404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789710763039645:2048] [node 1] ICS09 handshake done sender: [2:7577789710763039642:2333] self: [2:7577789710763039641:0] peer: [1:7577789708230793554:0] socket: 16 qp: 4652
2025-11-28T14:54:15.333436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789710763039645:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:15.333469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:15.333997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:15.334042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:15.334079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789708230793556:2048]
2025-11-28T14:54:15.334116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789708230793556:2048] [node 2] ICS09 handshake done sender: [1:7577789708230793553:2284] self: [1:7577789708230793554:0] peer: [2:7577789710763039641:0] socket: 18 qp: 4651
2025-11-28T14:54:15.334140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789708230793556:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:15.334165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:54:15.720635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:54:15.720936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789710763039645:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:54:15.720960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789710763039645:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:15.721024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:15.721056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789710763039645:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:15.721162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:15.721480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789708230793556:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:15.721503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789708230793556:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:15.721561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:15.721584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789708230793556:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:15.727953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789710763039645:2048] [node 1] ICS08 incoming handshake Self# [1:7577789708230793554:0] Peer# [2:7577789710763039641:0] Counter# 2854 LastInputSerial# 71
2025-11-28T14:54:15.729380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789708230793556:2048] [node 2] ICS08 incoming handshake Self# [2:7577789710763039641:0] Peer# [1:7577789708230793554:0] Counter# 71 LastInputSerial# 2854
2025-11-28T14:54:15.731074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789708230793558:2286] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:15.731284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:15.731331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789708230793556:2048] [node 2] ICS09 handshake done sender: [1:7577789708230793558:2286] self: [1:7577789708230793554:0] peer: [2:7577789710763039641:0] socket: 22 qp: 4653
2025-11-28T14:54:15.731662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789708230793556:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:15.731699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:15.733027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789710763039648:2335] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:15.733166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:15.733196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789710763039645:2048] [node 1] ICS09 handshake done sender: [2:7577789710763039648:2335] self: [2:7577789710763039641:0] peer: [1:7577789708230793554:0] socket: 23 qp: 4654
2025-11-28T14:54:15.733531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789710763039645:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:15.733559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:15.954785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:15.955141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789710763039645:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-28T14:54:15.955161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789710763039645:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:15.955183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:15.955219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789710763039645:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:15.959876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789710763039645:2048] [node 1] ICS08 incoming handshake Self# [1:7577789708230793554:0] Peer# [2:7577789710763039641:0] Counter# 5099 LastInputSerial# 140
2025-11-28T14:54:15.952539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:15.952693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:15.952969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789708230793556:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-28T14:54:15.952995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789708230793556:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:15.953021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:15.953062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789708230793556:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:15.961312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789708230793556:2048] [node 2] ICS08 incoming handshake Self# [2:7577789710763039641:0] Peer# [1:7577789708230793554:0] Counter# 140 LastInputSerial# 5099
2025-11-28T14:54:15.961620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789708230793563:2288] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:15.961789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:15.961838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789708230793556:2048] [node 2] ICS09 handshake done sender: [1:7577789708230793563:2288] self: [1:7577789708230793554:0] peer: [2:7577789710763039641:0] socket: 16 qp: 4658
2025-11-28T14:54:15.962152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789708230793556:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:15.962190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:15.965123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789710763039653:2338] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:15.965277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789710763039653:2338] held: no
2025-11-28T14:54:15.965300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:15.965325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789710763039645:2048] [node 1] ICS09 handshake done sender: [2:7577789710763039653:2338] self: [2:7577789710763039641:0] peer: [1:7577789708230793554:0] socket: 18 qp: 4657
2025-11-28T14:54:15.965743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789710763039645:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:15.965776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:16.173372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:16.174072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:16.174386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:54:16.174409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789708230793556:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:16.174432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789708230793556:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:16.174454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:16.174505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789708230793556:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:54:16.174575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789708230793556:2048] VirtualId# [1:7577789708230793554:0]
2025-11-28T14:54:16.174603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:16.176556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:16.176771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789710763039645:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:16.176792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789710763039645:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:16.176877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:16.176911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789710763039645:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:16.181761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789712525760866:2291]) peer tries to resume nonexistent session Self# [2:7577789710763039641:0] Peer# [1:7577789708230793554:0]
2025-11-28T14:54:16.182914Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789712525760866:2291] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:16.183475Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789715058006953:2340] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:16.184566Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:16.184637Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:16.184660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789710763039645:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:16.184715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789710763039645:2048] VirtualId# [2:7577789710763039641:0]
2025-11-28T14:54:16.184771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:16.185619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789715058006955:2341] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:16.185874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:16.186406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789712525760865:2290] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:16.191592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789712525760868:2292] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:16.192586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789715058006955:2341] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:16.192809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789712525760865:2290] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:16.193488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:16.193537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:16.193572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789712525760871:2048]
2025-11-28T14:54:16.193607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789712525760871:2048] [node 2] ICS09 handshake done sender: [1:7577789712525760865:2290] self: [1:7577789712525760864:0] peer: [2:7577789715058006958:0] socket: 21 qp: 4664
2025-11-28T14:54:16.193629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789712525760871:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:16.193654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:16.193091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:16.193151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:16.193192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789715058006961:2048]
2025-11-28T14:54:16.193230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789715058006961:2048] [node 1] ICS09 handshake done sender: [2:7577789715058006955:2341] self: [2:7577789715058006958:0] peer: [1:7577789712525760864:0] socket: 22 qp: 4663
2025-11-28T14:54:16.193260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789715058006961:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:16.193291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:54:16.703575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:16.700917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:54:16.701183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789715058006961:2048] [node 1] ICS07 socket disconnect 22 reason# Debug
2025-11-28T14:54:16.701209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789715058006961:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:16.701264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:16.701294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789715058006961:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:16.703874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789712525760871:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-28T14:54:16.703905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789712525760871:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:16.703969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:16.704002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789712525760871:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:16.707040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789712525760871:2048] [node 2] ICS08 incoming handshake Self# [2:7577789715058006958:0] Peer# [1:7577789712525760864:0] Counter# 76 LastInputSerial# 3086
2025-11-28T14:54:16.707938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789712525760873:2293] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:16.708123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:16.708170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789712525760871:2048] [node 2] ICS09 handshake done sender: [1:7577789712525760873:2293] self: [1:7577789712525760864:0] peer: [2:7577789715058006958:0] socket: 24 qp: 4669
2025-11-28T14:54:16.708513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789712525760871:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:16.708549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:16.707504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789715058006961:2048] [node 1] ICS08 incoming handshake Self# [1:7577789712525760864:0] Peer# [2:7577789715058006958:0] Counter# 3086 LastInputSerial# 76
2025-11-28T14:54:16.712195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789715058006965:2345] held: no
2025-11-28T14:54:16.712269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789715058006965:2345] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:16.712423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:16.712483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789715058006961:2048] [node 1] ICS09 handshake done sender: [2:7577789715058006965:2345] self: [2:7577789715058006958:0] peer: [1:7577789712525760864:0] socket: 25 qp: 4670
2025-11-28T14:54:16.712846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789715058006961:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:16.712888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:17.119192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:17.119437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789715058006961:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-28T14:54:17.119459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789715058006961:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:17.119539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:17.119574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789715058006961:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:17.116627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:17.117430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:17.117761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:54:17.117785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789712525760871:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:17.117809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789712525760871:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:17.117833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:17.117892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789712525760871:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:54:17.117967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789712525760871:2048] VirtualId# [1:7577789712525760864:0]
2025-11-28T14:54:17.117993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:17.131892Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789719352974264:2347] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:17.130314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789716820728174:2295]) peer tries to resume nonexistent session Self# [2:7577789715058006958:0] Peer# [1:7577789712525760864:0]
2025-11-28T14:54:17.131407Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789716820728174:2295] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:17.132722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:17.132796Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:17.132820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789715058006961:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:17.132874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789715058006961:2048] VirtualId# [2:7577789715058006958:0]
2025-11-28T14:54:17.132925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:17.134186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789719352974266:2348] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:17.134352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:17.136074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789716820728176:2296] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:17.141743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789716820728178:2297] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:17.144686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789719352974266:2348] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:17.144871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789716820728176:2296] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:17.145084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:17.145138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:17.145175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789716820728181:2048]
2025-11-28T14:54:17.145212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789716820728181:2048] [node 2] ICS09 handshake done sender: [1:7577789716820728176:2296] self: [1:7577789716820728175:0] peer: [2:7577789719352974270:0] socket: 20 qp: 4674
2025-11-28T14:54:17.145239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789716820728181:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:17.145270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:17.147329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:17.147376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:17.147415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789719352974272:2048]
2025-11-28T14:54:17.147449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789719352974272:2048] [node 1] ICS09 handshake done sender: [2:7577789719352974266:2348] self: [2:7577789719352974270:0] peer: [1:7577789716820728175:0] socket: 21 qp: 4673
2025-11-28T14:54:17.147474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789719352974272:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:17.147501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:17.147601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-28T14:54:17.484123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577789583676741341:2048] [node 2] ICP34 closed connection by debug command
2025-11-28T14:54:17.484253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-28T14:54:17.484280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789716820728181:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:17.484301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789716820728181:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:17.484334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:17.484428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:17.484269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
TEvNodeDisconnected
2025-11-28T14:54:17.484971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789719352974272:2048] [node 1] ICS01 socket: 21 reason# EndOfStream
2025-11-28T14:54:17.485045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789719352974272:2048] VirtualId# [2:7577789719352974270:0]
2025-11-28T14:54:17.485081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789719352974272:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:17.485145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:17.485193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:17.484723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789716820728181:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:17.484749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789716820728181:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:17.488245Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789716820728183:2298] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:17.487690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789719352974274:2351]) peer tries to resume nonexistent session Self# [1:7577789716820728175:0] Peer# [2:7577789719352974270:0]
2025-11-28T14:54:17.487756Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789719352974274:2351] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:17.488822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:17.488898Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:17.488921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789716820728181:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:17.488997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789716820728181:2048] VirtualId# [1:7577789716820728175:0]
2025-11-28T14:54:17.489038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:17.492345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789716820728186:2299] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:17.491888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789719352974275:2352] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:17.492033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:17.494930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789719352974275:2352] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:17.495048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789716820728186:2299] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:17.495091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:17.495122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:17.495149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789719352974278:2048]
2025-11-28T14:54:17.495173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789719352974278:2048] [node 1] ICS09 handshake done sender: [2:7577789719352974275:2352] self: [2:7577789719352974276:0] peer: [1:7577789716820728185:0] socket: 18 qp: 4678
2025-11-28T14:54:17.495198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789719352974278:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:17.495223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:17.495322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:17.495351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:17.495375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789716820728189:2048]
2025-11-28T14:54:17.495399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789716820728189:2048] [node 2] ICS09 handshake done sender: [1:7577789716820728186:2299] self: [1:7577789716820728185:0] peer: [2:7577789719352974276:0] socket: 16 qp: 4679
2025-11-28T14:54:17.495418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789716820728189:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:17.495446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:18.064169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:18.064325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:18.064350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789719352974278:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:18.064390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789719352974278:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:18.064423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:18.064690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:18.064906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789719352974278:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:18.064924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789719352974278:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:18.064745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:18.065005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789716820728189:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:18.065052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789716820728189:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:18.065126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:18.065159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789716820728189:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:18.071662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789716820728189:2048] [node 2] ICS08 incoming handshake Self# [2:7577789719352974276:0] Peer# [1:7577789716820728185:0] Counter# 97 LastInputSerial# 3941
2025-11-28T14:54:18.071267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789719352974278:2048] [node 1] ICS08 incoming handshake Self# [1:7577789716820728185:0] Peer# [2:7577789719352974276:0] Counter# 3941 LastInputSerial# 97
2025-11-28T14:54:18.072773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789721115695487:2300] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:18.073931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:18.073990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789716820728189:2048] [node 2] ICS09 handshake done sender: [1:7577789721115695487:2300] self: [1:7577789716820728185:0] peer: [2:7577789719352974276:0] socket: 23 qp: 4681
2025-11-28T14:54:18.074355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789716820728189:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:18.074399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:18.075805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789723647941578:2355] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:18.076013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:18.076067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789719352974278:2048] [node 1] ICS09 handshake done sender: [2:7577789723647941578:2355] self: [2:7577789719352974276:0] peer: [1:7577789716820728185:0] socket: 25 qp: 4683
2025-11-28T14:54:18.076484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789719352974278:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:18.076530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:54:18.336542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:54:18.336875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789716820728189:2048] [node 2] ICS07 socket disconnect 23 reason# Debug
2025-11-28T14:54:18.336901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789716820728189:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:18.336936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:18.336974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789716820728189:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:18.339843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Connection reset by peer
2025-11-28T14:54:18.339876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789719352974278:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:18.339898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789719352974278:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:54:18.339921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:18.339985Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:18.340203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789719352974278:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:18.340221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789719352974278:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:18.342130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789719352974278:2048] [node 1] ICS08 incoming handshake Self# [1:7577789716820728185:0] Peer# [2:7577789719352974276:0] Counter# 6202 LastInputSerial# 156
2025-11-28T14:54:18.345521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789721115695492:2302] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:18.345913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:18.345970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789716820728189:2048] [node 2] ICS09 handshake done sender: [1:7577789721115695492:2302] self: [1:7577789716820728185:0] peer: [2:7577789719352974276:0] socket: 16 qp: 4684
2025-11-28T14:54:18.346471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789716820728189:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:18.346514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:18.346615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789716820728189:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:18.346637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789716820728189:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:54:18.346687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:18.346724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789716820728189:2048] [node 2] ICS08 incoming handshake Self# [2:7577789719352974276:0] Peer# [1:7577789716820728185:0] Counter# 156 LastInputSerial# 6202
2025-11-28T14:54:18.346780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:18.346913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789716820728189:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-28T14:54:18.346957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789716820728189:2048] VirtualId# [1:7577789716820728185:0]
2025-11-28T14:54:18.346975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:18.348117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789723647941581:2357] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:18.348312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:18.348379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789719352974278:2048] [node 1] ICS09 handshake done sender: [2:7577789723647941581:2357] self: [2:7577789719352974276:0] peer: [1:7577789716820728185:0] socket: 18 qp: 4685
2025-11-28T14:54:18.348793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789719352974278:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:18.348833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:18.349110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:18.349130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789719352974278:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:18.349152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789719352974278:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:18.349173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:18.349241Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:18.349408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789719352974278:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:18.349434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789719352974278:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:18.353514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789721115695498:2305]) peer tries to resume nonexistent session Self# [2:7577789719352974276:0] Peer# [1:7577789716820728185:0]
2025-11-28T14:54:18.353579Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789721115695498:2305] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:18.354351Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789723647941587:2360] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:18.354918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:18.354984Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:18.355006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789719352974278:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:18.355049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789719352974278:2048] VirtualId# [2:7577789719352974276:0]
2025-11-28T14:54:18.355153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 52725
TEvUndelivered Cookie# 52726
TEvUndelivered Cookie# 52727
TEvUndelivered Cookie# 52728
TEvUndelivered Cookie# 52729
TEvUndelivered Cookie# 52730
TEvUndelivered Cookie# 52731
2025-11-28T14:54:18.358018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789721115695499:2306] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:18.358124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:18.358856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789723647941590:2361] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:18.360907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789721115695499:2306] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:18.360969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789723647941590:2361] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:18.361072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:18.361111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:18.361147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789721115695502:2048]
2025-11-28T14:54:18.361186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789721115695502:2048] [node 2] ICS09 handshake done sender: [1:7577789721115695499:2306] self: [1:7577789721115695500:0] peer: [2:7577789723647941589:0] socket: 18 qp: 4690
2025-11-28T14:54:18.361210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789721115695502:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:18.361240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:18.361446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:18.361487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:18.361519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789723647941593:2048]
2025-11-28T14:54:18.361549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789723647941593:2048] [node 1] ICS09 handshake done sender: [2:7577789723647941590:2361] self: [2:7577789723647941589:0] peer: [1:7577789721115695500:0] socket: 16 qp: 4689
2025-11-28T14:54:18.361570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789723647941593:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:18.361591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:18.840525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:18.841710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Connection reset by peer
2025-11-28T14:54:18.841757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789721115695502:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:18.841780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789721115695502:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:54:18.841811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:18.853258Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] socket: Connection reset by peer
2025-11-28T14:54:18.853517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789723647941593:2048] [node 1] ICS07 socket disconnect 16 reason# ECONNRESET
2025-11-28T14:54:18.853544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789723647941593:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-28T14:54:18.853583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:18.853669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789723647941593:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:18.854441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:18.854705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789721115695502:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:18.854735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789721115695502:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:18.859158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789721115695502:2048] [node 2] ICS08 incoming handshake Self# [2:7577789723647941589:0] Peer# [1:7577789721115695500:0] Counter# 75 LastInputSerial# 2919
2025-11-28T14:54:18.860454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789723647941593:2048] [node 1] ICS08 incoming handshake Self# [1:7577789721115695500:0] Peer# [2:7577789723647941589:0] Counter# 2919 LastInputSerial# 75
2025-11-28T14:54:18.867154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789721115695504:2308] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:18.867451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:18.867515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789721115695502:2048] [node 2] ICS09 handshake done sender: [1:7577789721115695504:2308] self: [1:7577789721115695500:0] peer: [2:7577789723647941589:0] socket: 24 qp: 4694
2025-11-28T14:54:18.867874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789721115695502:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:18.867922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:18.871911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789723647941597:2363] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:18.872132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:18.872190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789723647941593:2048] [node 1] ICS09 handshake done sender: [2:7577789723647941597:2363] self: [2:7577789723647941589:0] peer: [1:7577789721115695500:0] socket: 25 qp: 4693
2025-11-28T14:54:18.873357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789723647941593:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:18.873409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:19.297615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789723647941593:2048] [node 1] ICS01 socket: 25 reason# Debug
2025-11-28T14:54:19.297704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789723647941593:2048] VirtualId# [2:7577789723647941589:0]
2025-11-28T14:54:19.297734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789723647941593:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:19.297813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:19.297868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:19.298676Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:19.298968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789721115695502:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-28T14:54:19.298991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789721115695502:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:19.299064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:19.299096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789721115695502:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-28T14:54:19.309935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789727942908896:2365]) peer tries to resume nonexistent session Self# [1:7577789721115695500:0] Peer# [2:7577789723647941589:0]
2025-11-28T14:54:19.310025Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789727942908896:2365] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:19.310303Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789725410662805:2310] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:19.310981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:19.311045Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:19.311071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789721115695502:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:19.311209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789721115695502:2048] VirtualId# [1:7577789721115695500:0]
2025-11-28T14:54:19.311257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:19.320583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789727942908900:2367] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:19.321820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:19.322506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789725410662808:2311] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:19.322724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789725410662810:2312] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:19.327064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789727942908900:2367] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:19.327301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789725410662808:2311] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:19.327598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:19.327660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:19.327701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789725410662813:2048]
2025-11-28T14:54:19.327739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789725410662813:2048] [node 2] ICS09 handshake done sender: [1:7577789725410662808:2311] self: [1:7577789725410662807:0] peer: [2:7577789727942908901:0] socket: 16 qp: 4696
2025-11-28T14:54:19.327771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789725410662813:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:19.327804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:19.330038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:19.330090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:19.330131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789727942908903:2048]
2025-11-28T14:54:19.330181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789727942908903:2048] [node 1] ICS09 handshake done sender: [2:7577789727942908900:2367] self: [2:7577789727942908901:0] peer: [1:7577789725410662807:0] socket: 20 qp: 4697
2025-11-28T14:54:19.330213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789727942908903:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:19.330246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:19.330438Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-28T14:54:19.453066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-28T14:54:19.453384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789727942908903:2048] [node 1] ICS07 socket disconnect 20 reason# Debug
2025-11-28T14:54:19.453407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789727942908903:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:19.453445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:19.453470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789727942908903:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:19.454382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:19.454616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789725410662813:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:19.454638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789725410662813:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:19.454696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:19.454728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789725410662813:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:19.458685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789725410662813:2048] [node 2] ICS08 incoming handshake Self# [2:7577789727942908901:0] Peer# [1:7577789725410662807:0] Counter# 11 LastInputSerial# 406
2025-11-28T14:54:19.460915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789727942908903:2048] [node 1] ICS08 incoming handshake Self# [1:7577789725410662807:0] Peer# [2:7577789727942908901:0] Counter# 406 LastInputSerial# 11
2025-11-28T14:54:19.462848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789725410662815:2313] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:19.463084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:19.463142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789725410662813:2048] [node 2] ICS09 handshake done sender: [1:7577789725410662815:2313] self: [1:7577789725410662807:0] peer: [2:7577789727942908901:0] socket: 24 qp: 4703
2025-11-28T14:54:19.463518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789725410662813:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:19.463555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:19.467274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789727942908907:2370] held: no
2025-11-28T14:54:19.467342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789727942908907:2370] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:19.467518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:19.467563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789727942908903:2048] [node 1] ICS09 handshake done sender: [2:7577789727942908907:2370] self: [2:7577789727942908901:0] peer: [1:7577789725410662807:0] socket: 25 qp: 4702
2025-11-28T14:54:19.467913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789727942908903:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:19.467950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:19.568766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789725410662813:2048] [node 2] ICS01 socket: 24 reason# Debug
2025-11-28T14:54:19.568856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789725410662813:2048] VirtualId# [1:7577789725410662807:0]
2025-11-28T14:54:19.568885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789725410662813:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:19.568933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:19.568972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:19.569925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:19.569958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789727942908903:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:19.569978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789727942908903:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:19.569998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:19.570054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:19.570257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789727942908903:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:19.570273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789727942908903:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:19.576275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789725410662820:2315]) peer tries to resume nonexistent session Self# [2:7577789727942908901:0] Peer# [1:7577789725410662807:0]
2025-11-28T14:54:19.577508Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789725410662820:2315] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:19.578647Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789727942908910:2372] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:19.579565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:19.579635Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:19.579659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789727942908903:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:19.579714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789727942908903:2048] VirtualId# [2:7577789727942908901:0]
2025-11-28T14:54:19.579838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 54423
TEvUndelivered Cookie# 54424
TEvUndelivered Cookie# 54425
TEvUndelivered Cookie# 54426
TEvUndelivered Cookie# 54427
TEvUndelivered Cookie# 54428
TEvUndelivered Cookie# 54429
TEvUndelivered Cookie# 54430
2025-11-28T14:54:19.581382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789727942908912:2373] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:19.581758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:19.582486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789725410662822:2316] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:19.589142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789727942908912:2373] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:19.589148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789725410662824:2317] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:19.589472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789725410662822:2316] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:19.589478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:19.589525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:19.589567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789727942908918:2048]
2025-11-28T14:54:19.589602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789727942908918:2048] [node 1] ICS09 handshake done sender: [2:7577789727942908912:2373] self: [2:7577789727942908915:0] peer: [1:7577789725410662821:0] socket: 21 qp: 4706
2025-11-28T14:54:19.589630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789727942908918:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:19.589661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:19.589866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:19.589911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:19.589947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789725410662827:2048]
2025-11-28T14:54:19.589979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789725410662827:2048] [node 2] ICS09 handshake done sender: [1:7577789725410662822:2316] self: [1:7577789725410662821:0] peer: [2:7577789727942908915:0] socket: 20 qp: 4707
2025-11-28T14:54:19.590003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789725410662827:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:19.590030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:54:20.113752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:54:20.114114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789725410662827:2048] [node 2] ICS07 socket disconnect 20 reason# Debug
2025-11-28T14:54:20.114139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789725410662827:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:20.114168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:20.114200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789725410662827:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:20.118690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789727942908918:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:20.118738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789727942908918:2048] [node 1] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:54:20.118793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:20.118844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789727942908918:2048] [node 1] ICS08 incoming handshake Self# [1:7577789725410662821:0] Peer# [2:7577789727942908915:0] Counter# 3402 LastInputSerial# 88
2025-11-28T14:54:20.119138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:20.119359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789727942908918:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:20.121166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789729705630125:2318] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:20.121430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:20.121481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789725410662827:2048] [node 2] ICS09 handshake done sender: [1:7577789729705630125:2318] self: [1:7577789725410662821:0] peer: [2:7577789727942908915:0] socket: 16 qp: 4711
2025-11-28T14:54:20.121782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789725410662827:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:20.121818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:20.122950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789732237876216:2376] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:20.123149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:20.123194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789727942908918:2048] [node 1] ICS09 handshake done sender: [2:7577789732237876216:2376] self: [2:7577789727942908915:0] peer: [1:7577789725410662821:0] socket: 18 qp: 4710
2025-11-28T14:54:20.123551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789727942908918:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:20.123595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:20.503629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789727942908918:2048] [node 1] ICS01 socket: 18 reason# Debug
2025-11-28T14:54:20.503725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789727942908918:2048] VirtualId# [2:7577789727942908915:0]
2025-11-28T14:54:20.503760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789727942908918:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:20.503820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:20.503872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:20.504882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:20.505169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789725410662827:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:20.505193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789725410662827:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:20.505258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:20.505289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789725410662827:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-28T14:54:20.510481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789732237876219:2378]) peer tries to resume nonexistent session Self# [1:7577789725410662821:0] Peer# [2:7577789727942908915:0]
2025-11-28T14:54:20.510565Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789732237876219:2378] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:20.512077Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789729705630129:2319] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:20.513077Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:20.513155Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:20.513181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789725410662827:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:20.513239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789725410662827:2048] VirtualId# [1:7577789725410662821:0]
2025-11-28T14:54:20.513288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:20.525118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789732237876223:2380] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:20.525327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:20.527931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789729705630134:2321] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:20.528414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789729705630132:2320] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:20.536857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789732237876223:2380] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:20.537217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789729705630132:2320] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:20.538110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:20.538188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:20.538228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789732237876226:2048]
2025-11-28T14:54:20.538286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789732237876226:2048] [node 1] ICS09 handshake done sender: [2:7577789732237876223:2380] self: [2:7577789732237876224:0] peer: [1:7577789729705630131:0] socket: 20 qp: 4714
2025-11-28T14:54:20.538314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789732237876226:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:20.538344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:20.537743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:20.537853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:20.537901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789729705630137:2048]
2025-11-28T14:54:20.537942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789729705630137:2048] [node 2] ICS09 handshake done sender: [1:7577789729705630132:2320] self: [1:7577789729705630131:0] peer: [2:7577789732237876224:0] socket: 16 qp: 4713
2025-11-28T14:54:20.537977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789729705630137:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:20.538039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-28T14:54:20.676148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789732237876226:2048] [node 1] ICS01 socket: 20 reason# Debug
2025-11-28T14:54:20.676703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789732237876226:2048] VirtualId# [2:7577789732237876224:0]
2025-11-28T14:54:20.676743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789732237876226:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:20.676833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:20.676939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:20.677845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:20.678173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789729705630137:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:20.678198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789729705630137:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:20.678283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:20.678317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789729705630137:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 55882
TEvUndelivered Cookie# 55883
TEvUndelivered Cookie# 55884
TEvUndelivered Cookie# 55885
TEvUndelivered Cookie# 55886
TEvUndelivered Cookie# 55887
TEvUndelivered Cookie# 55888
2025-11-28T14:54:20.691365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577789586208987432:2048] [node 1] ICP12 (actor [2:7577789732237876228:2382]) peer tries to resume nonexistent session Self# [1:7577789729705630131:0] Peer# [2:7577789732237876224:0]
2025-11-28T14:54:20.692757Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577789732237876228:2382] [node 1] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:20.693809Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577789729705630139:2322] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake
2025-11-28T14:54:20.694961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789583676741341:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:6162) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:20.695070Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577789583676741341:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:20.695100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789729705630137:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:20.695337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789729705630137:2048] VirtualId# [1:7577789729705630131:0]
2025-11-28T14:54:20.695403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:20.700830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789729705630141:2323] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:20.705345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789732237876232:2384] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:20.705470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577789586208987432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:20.705960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789729705630143:2324] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:20.719745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789732237876232:2384] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:20.720014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789729705630143:2324] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:20.720300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:20.720377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:20.720423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789729705630147:2048]
2025-11-28T14:54:20.720508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789729705630147:2048] [node 2] ICS09 handshake done sender: [1:7577789729705630143:2324] self: [1:7577789729705630142:0] peer: [2:7577789732237876233:0] socket: 16 qp: 4722
2025-11-28T14:54:20.720727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789729705630147:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:20.720775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:20.722510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:20.722600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:20.722645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789732237876235:2048]
2025-11-28T14:54:20.722709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789732237876235:2048] [node 1] ICS09 handshake done sender: [2:7577789732237876232:2384] self: [2:7577789732237876233:0] peer: [1:7577789729705630142:0] socket: 18 qp: 4721
2025-11-28T14:54:20.722744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789732237876235:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:20.722779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-28T14:54:21.204826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-28T14:54:21.205159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789729705630147:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-28T14:54:21.205186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789729705630147:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:21.205237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:21.205271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789729705630147:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:21.212585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:21.212839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789732237876235:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:21.212863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789732237876235:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:21.212928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:21.212963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789732237876235:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:21.214782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789732237876235:2048] [node 1] ICS08 incoming handshake Self# [1:7577789729705630142:0] Peer# [2:7577789732237876233:0] Counter# 2041 LastInputSerial# 53
2025-11-28T14:54:21.220663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789734000597445:2325] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:21.221294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789729705630147:2048] [node 2] ICS08 incoming handshake Self# [2:7577789732237876233:0] Peer# [1:7577789729705630142:0] Counter# 53 LastInputSerial# 2041
2025-11-28T14:54:21.221334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:21.221381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789729705630147:2048] [node 2] ICS09 handshake done sender: [1:7577789734000597445:2325] self: [1:7577789729705630142:0] peer: [2:7577789732237876233:0] socket: 21 qp: 4723
2025-11-28T14:54:21.221760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789729705630147:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:21.221799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:21.225040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789736532843534:2387] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:21.225253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:14726) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577789736532843534:2387] held: no
2025-11-28T14:54:21.225287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:21.225335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789732237876235:2048] [node 1] ICS09 handshake done sender: [2:7577789736532843534:2387] self: [2:7577789732237876233:0] peer: [1:7577789729705630142:0] socket: 23 qp: 4724
2025-11-28T14:54:21.225797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789732237876235:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:21.225838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:21.792026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:21.792131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:21.792440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:21.792467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789732237876235:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:21.792495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789732237876235:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:21.792523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:21.792574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789732237876235:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:21.792592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789732237876235:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:21.793544Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:21.793823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789729705630147:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-28T14:54:21.793845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789729705630147:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:21.793904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:21.793934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789729705630147:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:21.798699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789729705630147:2048] [node 2] ICS08 incoming handshake Self# [2:7577789732237876233:0] Peer# [1:7577789729705630142:0] Counter# 127 LastInputSerial# 4692
2025-11-28T14:54:21.800147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789732237876235:2048] [node 1] ICS08 incoming handshake Self# [1:7577789729705630142:0] Peer# [2:7577789732237876233:0] Counter# 4692 LastInputSerial# 127
2025-11-28T14:54:21.800694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789734000597450:2327] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:21.800942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:21.801001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789729705630147:2048] [node 2] ICS09 handshake done sender: [1:7577789734000597450:2327] self: [1:7577789729705630142:0] peer: [2:7577789732237876233:0] socket: 20 qp: 4730
2025-11-28T14:54:21.801527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789729705630147:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:21.801571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:21.804857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789736532843540:2390] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:21.806115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:21.806178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789732237876235:2048] [node 1] ICS09 handshake done sender: [2:7577789736532843540:2390] self: [2:7577789732237876233:0] peer: [1:7577789729705630142:0] socket: 22 qp: 4729
2025-11-28T14:54:21.806629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789732237876235:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:21.806672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:21.806871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577789586208987432:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-28T14:54:22.191981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577789586208987432:2048] [node 1] ICP34 closed connection by debug command
2025-11-28T14:54:22.193765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-28T14:54:22.193790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577789732237876235:2048] [node 1] ICS13 reestablish connection
2025-11-28T14:54:22.193813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789732237876235:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-28T14:54:22.193839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:22.194200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:22.194481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789729705630147:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-28T14:54:22.194504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789729705630147:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:22.194564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:22.194597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577789729705630147:2048] [node 2] ICS15 start handshake
2025-11-28T14:54:22.197147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:22.197382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789732237876235:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-28T14:54:22.197402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789732237876235:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:22.199164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577789732237876235:2048] [node 1] ICS08 incoming handshake Self# [1:7577789729705630142:0] Peer# [2:7577789732237876233:0] Counter# 7666 LastInputSerial# 204
2025-11-28T14:54:22.201243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789738295564751:2329] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:22.201506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577789583676741341:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-28T14:54:22.201570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789729705630147:2048] [node 2] ICS09 handshake done sender: [1:7577789738295564751:2329] self: [1:7577789729705630142:0] peer: [2:7577789732237876233:0] socket: 16 qp: 4731
2025-11-28T14:54:22.201889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789729705630147:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:22.201934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:22.205519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789740827810840:2393] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:22.205714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577789586208987432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-28T14:54:22.205767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789732237876235:2048] [node 1] ICS09 handshake done sender: [2:7577789740827810840:2393] self: [2:7577789732237876233:0] peer: [1:7577789729705630142:0] socket: 18 qp: 4732
2025-11-28T14:54:22.206081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789732237876235:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:22.206122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:22.208397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577789729705630147:2048] [node 2] ICS13 reestablish connection
2025-11-28T14:54:22.208448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789729705630147:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-28T14:54:22.208493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:22.208535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789729705630147:2048] [node 2] ICS08 incoming handshake Self# [2:7577789732237876233:0] Peer# [1:7577789729705630142:0] Counter# 204 LastInputSerial# 7686
2025-11-28T14:54:22.208784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577789729705630147:2048] [node 2] ICS07 socket disconnect -1 reason# NewSession
2025-11-28T14:54:22.210252Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:22.210490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789732237876235:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-28T14:54:22.210513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789732237876235:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:22.210582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:22.210612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789732237876235:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:22.214459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577789729705630147:2048] [node 2] ICS08 incoming handshake Self# [2:7577789732237876233:0] Peer# [1:7577789729705630142:0] Counter# 204 LastInputSerial# 7686
2025-11-28T14:54:22.217261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789740827810844:2395] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:22.217556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:22.217607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789732237876235:2048] [node 1] ICS09 handshake done sender: [2:7577789740827810844:2395] self: [2:7577789732237876233:0] peer: [1:7577789729705630142:0] socket: 21 qp: 4736
2025-11-28T14:54:22.217912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789732237876235:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:22.217964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:22.219315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789738295564756:2331] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:22.219545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:22.219592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789729705630147:2048] [node 2] ICS09 handshake done sender: [1:7577789738295564756:2331] self: [1:7577789729705630142:0] peer: [2:7577789732237876233:0] socket: 22 qp: 4735
2025-11-28T14:54:22.220232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789729705630147:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:22.220289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:22.692573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789729705630147:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-28T14:54:22.692663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789729705630147:2048] VirtualId# [1:7577789729705630142:0]
2025-11-28T14:54:22.692692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789729705630147:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:22.692739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:22.692780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:22.693641Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:22.693867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789732237876235:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-28T14:54:22.693889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789732237876235:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:22.693938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:22.693967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789732237876235:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:22.699568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789738295564759:2333]) peer tries to resume nonexistent session Self# [2:7577789732237876233:0] Peer# [1:7577789729705630142:0]
2025-11-28T14:54:22.699624Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789738295564759:2333] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:22.701339Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789740827810848:2396] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:22.702049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:22.702114Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:22.702138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789732237876235:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:22.702194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789732237876235:2048] VirtualId# [2:7577789732237876233:0]
2025-11-28T14:54:22.702340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 58564
TEvUndelivered Cookie# 58565
TEvUndelivered Cookie# 58566
TEvUndelivered Cookie# 58567
TEvUndelivered Cookie# 58568
TEvUndelivered Cookie# 58569
TEvUndelivered Cookie# 58570
TEvUndelivered Cookie# 58571
TEvUndelivered Cookie# 58572
TEvUndelivered Cookie# 58573
2025-11-28T14:54:22.706453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789738295564760:2334] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:22.706598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:22.706973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789740827810851:2397] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:22.709985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789738295564760:2334] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:22.710337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789740827810851:2397] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:22.710547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:22.710590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:22.710629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789740827810854:2048]
2025-11-28T14:54:22.710665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789740827810854:2048] [node 1] ICS09 handshake done sender: [2:7577789740827810851:2397] self: [2:7577789740827810850:0] peer: [1:7577789738295564761:0] socket: 16 qp: 4739
2025-11-28T14:54:22.710693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789740827810854:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:22.710731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:22.711199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:22.711235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:22.711283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789738295564763:2048]
2025-11-28T14:54:22.711317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789738295564763:2048] [node 2] ICS09 handshake done sender: [1:7577789738295564760:2334] self: [1:7577789738295564761:0] peer: [2:7577789740827810850:0] socket: 18 qp: 4738
2025-11-28T14:54:22.711343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789738295564763:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:22.711369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-28T14:54:22.860562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789738295564763:2048] [node 2] ICS01 socket: 18 reason# Debug
2025-11-28T14:54:22.860750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789583676741341:2048] [node 2] ICP30 unregister session Session# [1:7577789738295564763:2048] VirtualId# [1:7577789738295564761:0]
2025-11-28T14:54:22.860788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789738295564763:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-28T14:54:22.860871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:22.860923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-28T14:54:22.861930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:22.862161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577789740827810854:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-28T14:54:22.862184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789740827810854:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:22.862243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:22.862275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577789740827810854:2048] [node 1] ICS15 start handshake
2025-11-28T14:54:22.868499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577789583676741341:2048] [node 2] ICP12 (actor [1:7577789738295564765:2336]) peer tries to resume nonexistent session Self# [2:7577789740827810850:0] Peer# [1:7577789738295564761:0]
2025-11-28T14:54:22.868563Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577789738295564765:2336] [node 2] ICH08 Continuation request rejected by proxy
2025-11-28T14:54:22.876618Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577789740827810856:2398] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake
2025-11-28T14:54:22.877773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577789586208987432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:14726) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-28T14:54:22.877851Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577789586208987432:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-28T14:54:22.877877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789740827810854:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-28T14:54:22.878023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789740827810854:2048] VirtualId# [2:7577789740827810850:0]
2025-11-28T14:54:22.878175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 58634
TEvUndelivered Cookie# 58635
TEvUndelivered Cookie# 58636
TEvUndelivered Cookie# 58637
TEvUndelivered Cookie# 58638
TEvUndelivered Cookie# 58639
TEvUndelivered Cookie# 58640
TEvUndelivered Cookie# 58641
TEvUndelivered Cookie# 58642
TEvUndelivered Cookie# 58643
2025-11-28T14:54:22.882769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789738295564766:2337] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:22.882917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789583676741341:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:22.883337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789740827810859:2399] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:22.902769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789738295564766:2337] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:22.903053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789740827810859:2399] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:22.903317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789586208987432:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:22.903362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:22.903404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789586208987432:2048] [node 1] ICP22 created new session: [2:7577789740827810862:2048]
2025-11-28T14:54:22.903470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789740827810862:2048] [node 1] ICS09 handshake done sender: [2:7577789740827810859:2399] self: [2:7577789740827810858:0] peer: [1:7577789738295564767:0] socket: 16 qp: 4742
2025-11-28T14:54:22.903505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789740827810862:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:22.903535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:22.904073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789583676741341:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:22.904108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:22.904143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789583676741341:2048] [node 2] ICP22 created new session: [1:7577789738295564769:2048]
2025-11-28T14:54:22.904178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789738295564769:2048] [node 2] ICS09 handshake done sender: [1:7577789738295564766:2337] self: [1:7577789738295564767:0] peer: [2:7577789740827810858:0] socket: 18 qp: 4741
2025-11-28T14:54:22.904201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789738295564769:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:22.904227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
2025-11-28T14:54:23.060593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:23.060844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789740827810862:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-28T14:54:23.061082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789586208987432:2048] [node 1] ICP30 unregister session Session# [2:7577789740827810862:2048] VirtualId# [2:7577789740827810858:0]
2025-11-28T14:54:23.061117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789740827810862:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:23.061258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:23.061317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:23.156931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789744392316609:2048] [node 2] ICP01 ready to work
Subscribe
2025-11-28T14:54:23.179711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789744990799820:2048] [node 1] ICP01 ready to work
2025-11-28T14:54:23.221503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789744392316618:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:23.223121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789744392316609:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:23.223596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789744990799830:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:23.234627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789744392316618:2056] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:23.234944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789744990799830:2056] [node 1] ICH04 handshake succeeded
TEvNodeConnected
2025-11-28T14:54:23.235267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789744990799820:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:23.235323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:23.235379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789744990799820:2048] [node 1] ICP22 created new session: [2:7577789744990799833:2048]
2025-11-28T14:54:23.235426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789744990799833:2048] [node 1] ICS09 handshake done sender: [2:7577789744990799830:2056] self: [2:7577789744990799829:0] peer: [1:7577789744392316619:0] socket: 16 qp: 4743
2025-11-28T14:54:23.235460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789744990799833:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:23.235488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:23.239015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789744392316609:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:23.239062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:23.239104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789744392316609:2048] [node 2] ICP22 created new session: [1:7577789744392316621:2048]
2025-11-28T14:54:23.239145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789744392316621:2048] [node 2] ICS09 handshake done sender: [1:7577789744392316618:2056] self: [1:7577789744392316619:0] peer: [2:7577789744990799829:0] socket: 18 qp: 4744
2025-11-28T14:54:23.239173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789744392316621:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:23.239203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:23.277249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:622: Proxy [1:7577789744392316609:2048] [node 2] ICP31 proxy http called
2025-11-28T14:54:23.304834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [1:7577789744392316609:2048] [node 2] ICP35 event processing took too much time StateWork:294(27564361ns) {GenerateHttpInfo:620(27559731ns)}
2025-11-28T14:54:23.305287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:622: Proxy [2:7577789744990799820:2048] [node 1] ICP31 proxy http called
2025-11-28T14:54:23.327782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:23.327996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789744990799833:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-28T14:54:23.328302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789744990799820:2048] [node 1] ICP30 unregister session Session# [2:7577789744990799833:2048] VirtualId# [2:7577789744990799829:0]
2025-11-28T14:54:23.328336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789744990799833:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:23.328473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:23.328534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-28T14:54:23.420695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789742805193581:2048] [node 2] ICP01 ready to work
2025-11-28T14:54:23.434595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789741998513390:2048] [node 1] ICP01 ready to work
2025-11-28T14:54:23.599954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789742805193590:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:23.611991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789741998513400:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:23.611415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789742805193581:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:23.616276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789742805193590:2056] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:23.616511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789741998513400:2056] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:23.616852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789741998513390:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:23.616905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:23.616949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789741998513390:2048] [node 1] ICP22 created new session: [2:7577789741998513403:2048]
2025-11-28T14:54:23.616988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789741998513403:2048] [node 1] ICS09 handshake done sender: [2:7577789741998513400:2056] self: [2:7577789741998513399:0] peer: [1:7577789742805193591:0] socket: 16 qp: 4745
2025-11-28T14:54:23.617019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789741998513403:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:23.617047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-28T14:54:23.620720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789742805193581:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:23.620788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:23.620837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789742805193581:2048] [node 2] ICP22 created new session: [1:7577789742805193593:2048]
2025-11-28T14:54:23.620881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789742805193593:2048] [node 2] ICS09 handshake done sender: [1:7577789742805193590:2056] self: [1:7577789742805193591:0] peer: [2:7577789741998513399:0] socket: 18 qp: 4746
2025-11-28T14:54:23.620912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789742805193593:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:23.620951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
RECEIVED TEvTest
2025-11-28T14:54:25.516517Z :INTERCONNECT CRIT: interconnect_tcp_session.cpp:517: Session [2:7577789741998513403:2048] [node 1] ICS31 serialized event Type# 0x7ff00000 is too large
2025-11-28T14:54:25.516579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789741998513403:2048] [node 1] ICS01 socket: 16 reason# EventTooLarge
2025-11-28T14:54:25.516693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789741998513390:2048] [node 1] ICP30 unregister session Session# [2:7577789741998513403:2048] VirtualId# [2:7577789741998513399:0]
2025-11-28T14:54:25.516748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789741998513403:2048] [node 1] ICS25 shutdown socket, reason# EventTooLarge
2025-11-28T14:54:25.516869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:25.517521Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-28T14:54:25.517735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577789742805193593:2048] [node 2] ICS01 socket: 18 reason# EndOfStream
2025-11-28T14:54:25.517819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577789742805193581:2048] [node 2] ICP30 unregister session Session# [1:7577789742805193593:2048] VirtualId# [1:7577789742805193591:0]
2025-11-28T14:54:25.517846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577789742805193593:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:25.517919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-28T14:54:25.517950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
TEvUndelivered
2025-11-28T14:54:25.677527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-28T14:54:25.724653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577789751395128187:2058] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:25.724819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577789742805193581:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-28T14:54:25.728638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577789750588447998:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-28T14:54:25.739001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577789751395128187:2058] [node 2] ICH04 handshake succeeded
2025-11-28T14:54:25.739265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577789750588447998:2057] [node 1] ICH04 handshake succeeded
2025-11-28T14:54:25.739584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577789741998513390:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-28T14:54:25.739625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-28T14:54:25.739662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577789741998513390:2048] [node 1] ICP22 created new session: [2:7577789750588448001:2048]
2025-11-28T14:54:25.739698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577789750588448001:2048] [node 1] ICS09 handshake done sender: [2:7577789750588447998:2057] self: [2:7577789750588447997:0] peer: [1:7577789751395128188:0] socket: 16 qp: 4747
2025-11-28T14:54:25.739726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577789750588448001:2048] [node 1] ICS10 traffic start
2025-11-28T14:54:25.739754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
RECEIVED TEvTest
2025-11-28T14:54:25.740546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577789742805193581:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-28T14:54:25.740582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-28T14:54:25.740615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577789742805193581:2048] [node 2] ICP22 created new session: [1:7577789751395128190:2048]
2025-11-28T14:54:25.740651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577789751395128190:2048] [node 2] ICS09 handshake done sender: [1:7577789751395128187:2058] self: [1:7577789751395128188:0] peer: [2:7577789750588447997:0] socket: 18 qp: 4748
2025-11-28T14:54:25.740673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577789751395128190:2048] [node 2] ICS10 traffic start
2025-11-28T14:54:25.740695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-28T14:54:25.768264Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-28T14:54:25.768496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577789750588448001:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-28T14:54:25.768813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577789741998513390:2048] [node 1] ICP30 unregister session Session# [2:7577789750588448001:2048] VirtualId# [2:7577789750588447997:0]
2025-11-28T14:54:25.768849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577789750588448001:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-28T14:54:25.768940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-28T14:54:25.768977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
written 2560000 bytes
written 3437884 bytes
written 4133621 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4191124 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4229896 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4229472 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195326 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4224051 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195697 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4184522 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4246333 bytes
written 4195061 bytes
written 4195061 bytes
written 4196757 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4183409 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4228041 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
2025-11-28T14:55:12.352608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577789956090757939:2048] [node 2] ICP01 ready to work
2025-11-28T14:55:12.369483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577789956090757939:2048] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5023) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-28T14:55:12.369618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7577789956090757939:2048] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5023) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:55:12.369643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5023) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-28T14:55:12.380560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [1:7577789956090757939:2048] [node 2] ICP33 wake up from error state
2025-11-28T14:55:12.383198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577789955027737433:2048] [node 1] ICP01 ready to work
